From 75bd0ebafe13149d39237a1f0cf8f7bebc24ef10 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 19 Mar 2020 21:12:09 +0800 Subject: [PATCH 01/85] [TD-15] move sdb to mnode module --- src/CMakeLists.txt | 1 - src/dnode/CMakeLists.txt | 2 +- src/mnode/CMakeLists.txt | 2 +- src/{sdb/inc/sdbint.h => mnode/inc/mgmtSdb.h} | 0 .../src/sdbEngine.c => mnode/src/mgmtSdb.c} | 47 ++++++++++++++- src/sdb/CMakeLists.txt | 16 ----- src/sdb/src/sdbstr.c | 58 ------------------- src/{sdb => util}/inc/hashint.h | 0 src/{sdb => util}/inc/hashstr.h | 0 src/{sdb => util}/src/hashint.c | 0 src/{sdb => util}/src/hashstr.c | 0 11 files changed, 47 insertions(+), 79 deletions(-) rename src/{sdb/inc/sdbint.h => mnode/inc/mgmtSdb.h} (100%) rename src/{sdb/src/sdbEngine.c => mnode/src/mgmtSdb.c} (97%) delete mode 100644 src/sdb/CMakeLists.txt delete mode 100644 src/sdb/src/sdbstr.c rename src/{sdb => util}/inc/hashint.h (100%) rename src/{sdb => util}/inc/hashstr.h (100%) rename src/{sdb => util}/src/hashint.c (100%) rename src/{sdb => util}/src/hashstr.c (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a05cc0224a..326f00dbbd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,6 @@ ADD_SUBDIRECTORY(client) ADD_SUBDIRECTORY(query) ADD_SUBDIRECTORY(kit) ADD_SUBDIRECTORY(plugins) -ADD_SUBDIRECTORY(sdb) ADD_SUBDIRECTORY(mnode) ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(dnode) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 298eb6110c..d85782b463 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode sdb taos_static monitor http tsdb) + TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http tsdb) #IF (TD_CLUSTER) # TARGET_LINK_LIBRARIES(taosd dcluster) diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 5bf4cfd604..bacd7497ad 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -13,7 +13,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mnode ${SRC}) - TARGET_LINK_LIBRARIES(mnode trpc tutil sdb pthread) + TARGET_LINK_LIBRARIES(mnode trpc tutil pthread) IF (TD_CLUSTER) TARGET_LINK_LIBRARIES(mnode) diff --git a/src/sdb/inc/sdbint.h b/src/mnode/inc/mgmtSdb.h similarity index 100% rename from src/sdb/inc/sdbint.h rename to src/mnode/inc/mgmtSdb.h diff --git a/src/sdb/src/sdbEngine.c b/src/mnode/src/mgmtSdb.c similarity index 97% rename from src/sdb/src/sdbEngine.c rename to src/mnode/src/mgmtSdb.c index fbc41089d1..04802e77da 100644 --- a/src/sdb/src/sdbEngine.c +++ b/src/mnode/src/mgmtSdb.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" - #include "sdb.h" -#include "sdbint.h" #include "tutil.h" +#include "mgmtSdb.h" #define abs(x) (((x) < 0) ? -(x) : (x)) @@ -1015,3 +1015,46 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } + + +int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; +void (*mpeerCleanUpMnodesFp)() = NULL; +int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; + +char *sdbStatusStr[] = { + "offline", + "unsynced", + "syncing", + "serving", + "null" +}; + +char *sdbRoleStr[] = { + "unauthed", + "undecided", + "master", + "slave", + "null" +}; + +int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { + if (mpeerForwardRequestFp) { + return mpeerForwardRequestFp(pTable, type, data, dataLen); + } else { + return 0; + } +} + +int32_t sdbInitPeers(char *directory) { + if (mpeerInitMnodesFp) { + return (*mpeerInitMnodesFp)(directory); + } else { + return 0; + } +} + +void sdbCleanUpPeers() { + if (mpeerCleanUpMnodesFp) { + (*mpeerCleanUpMnodesFp)(); + } +} diff --git a/src/sdb/CMakeLists.txt b/src/sdb/CMakeLists.txt deleted file mode 100644 index 47ea6e15b8..0000000000 --- a/src/sdb/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) -INCLUDE_DIRECTORIES(inc) - -IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(sdb ${SRC}) - TARGET_LINK_LIBRARIES(sdb trpc) - IF (TD_CLUSTER) - TARGET_LINK_LIBRARIES(sdb) - ENDIF() -ENDIF () diff --git a/src/sdb/src/sdbstr.c b/src/sdb/src/sdbstr.c deleted file mode 100644 index 59c01eb15a..0000000000 --- a/src/sdb/src/sdbstr.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#define _DEFAULT_SOURCE -#include "sdbint.h" - -int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; - -char *sdbStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -char *sdbRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - -int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); - } else { - return 0; - } -} - -int32_t sdbInitPeers(char *directory) { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(directory); - } else { - return 0; - } -} - -void sdbCleanUpPeers() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} diff --git a/src/sdb/inc/hashint.h b/src/util/inc/hashint.h similarity index 100% rename from src/sdb/inc/hashint.h rename to src/util/inc/hashint.h diff --git a/src/sdb/inc/hashstr.h b/src/util/inc/hashstr.h similarity index 100% rename from src/sdb/inc/hashstr.h rename to src/util/inc/hashstr.h diff --git a/src/sdb/src/hashint.c b/src/util/src/hashint.c similarity index 100% rename from src/sdb/src/hashint.c rename to src/util/src/hashint.c diff --git a/src/sdb/src/hashstr.c b/src/util/src/hashstr.c similarity index 100% rename from src/sdb/src/hashstr.c rename to src/util/src/hashstr.c From 4b7e463be83f9ef6b8a88fdb9a9877d839ac7acf Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 19 Mar 2020 23:29:29 +0800 Subject: [PATCH 02/85] [TD-15] refactor sdb --- src/inc/mnode.h | 1 - src/inc/sdb.h | 131 ----------- src/inc/taosdef.h | 10 + src/kit/CMakeLists.txt | 4 +- src/mnode/inc/mgmtMnode.h | 7 +- src/mnode/inc/mgmtSdb.h | 139 ++++-------- src/mnode/src/mgmtChildTable.c | 16 +- src/mnode/src/mgmtDClient.c | 2 +- src/mnode/src/mgmtDServer.c | 2 +- src/mnode/src/mgmtDb.c | 12 +- src/mnode/src/mgmtDnode.c | 2 +- src/mnode/src/mgmtMain.c | 12 +- src/mnode/src/mgmtMnode.c | 70 ++++-- src/mnode/src/mgmtNormalTable.c | 15 +- src/mnode/src/mgmtSdb.c | 371 +++++++++++--------------------- src/mnode/src/mgmtShell.c | 42 ++-- src/mnode/src/mgmtSuperTable.c | 12 +- src/mnode/src/mgmtTable.c | 1 + src/mnode/src/mgmtUser.c | 10 +- src/mnode/src/mgmtVgroup.c | 4 +- src/util/inc/tglobalcfg.h | 6 - src/util/inc/tlog.h | 19 ++ src/util/src/tglobalcfg.c | 28 +-- 23 files changed, 294 insertions(+), 622 deletions(-) delete mode 100644 src/inc/sdb.h diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 35902df617..b7631674b3 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "taosmsg.h" #include "taoserror.h" -#include "sdb.h" #include "tglobalcfg.h" #include "thash.h" #include "tidpool.h" diff --git a/src/inc/sdb.h b/src/inc/sdb.h deleted file mode 100644 index 4b4de1ac4b..0000000000 --- a/src/inc/sdb.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SDB_H -#define TDENGINE_SDB_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "taosmsg.h" -#include "taosdef.h" - -extern uint16_t tsMgmtMgmtPort; -extern uint16_t tsMgmtSyncPort; -extern int tsMgmtPeerHBTimer; // seconds -extern char * sdbStatusStr[]; -extern char * sdbRoleStr[]; -extern int sdbMaster; -extern SRpcIpSet *pSdbIpList; -extern SRpcIpSet *pSdbPublicIpList; - -extern void (*sdbWorkAsMasterCallback)(); // this function pointer will be set by taosd - -enum _keytype { - SDB_KEYTYPE_STRING, SDB_KEYTYPE_UINT32, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_RECYCLE, SDB_KEYTYPE_MAX -}; - -#define SDB_ROLE_UNAPPROVED 0 -#define SDB_ROLE_UNDECIDED 1 -#define SDB_ROLE_MASTER 2 -#define SDB_ROLE_SLAVE 3 - -#define SDB_STATUS_OFFLINE 0 -#define SDB_STATUS_UNSYNCED 1 -#define SDB_STATUS_SYNCING 2 -#define SDB_STATUS_SERVING 3 -#define SDB_STATUS_DELETED 4 - -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, - SDB_TYPE_DECODE, - SDB_TYPE_ENCODE, - SDB_TYPE_BEFORE_BATCH_UPDATE, - SDB_TYPE_BATCH_UPDATE, - SDB_TYPE_AFTER_BATCH_UPDATE, - SDB_TYPE_RESET, - SDB_TYPE_DESTROY, - SDB_MAX_ACTION_TYPES -}; - -#define SDB_MAX_PEERS 4 -typedef struct { - uint32_t ip; - uint32_t publicIp; - char ipstr[20]; - char zone[12]; - char role; - int64_t createdTime; - uint64_t dbVersion; - int64_t lostTime; - char status; - char numOfMnodes; - int numOfDnodes; - char updateEnd[1]; - - // internal - int syncFd; - void *hbTimer; - void *pSync; -} SSdbPeer; - -extern SSdbPeer *sdbPeer[]; -#define sdbInited (sdbPeer[0]) -#define sdbStatus (sdbPeer[0]->status) - -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)); - -void *sdbGetRow(void *handle, void *key); - -int64_t sdbInsertRow(void *handle, void *row, int rowSize); - -int sdbDeleteRow(void *handle, void *key); - -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated); - -void *sdbFetchRow(void *handle, void *pNode, void **ppRow); - -int sdbBatchUpdateRow(void *handle, void *row, int rowSize); - -int64_t sdbGetId(void *handle); - -int64_t sdbGetNumOfRows(void *handle); - -void sdbSaveSnapShot(void *handle); - -void sdbCloseTable(void *handle); - -int sdbRemovePeerByIp(uint32_t ip); - -int sdbInitPeers(char *directory); - -void sdbCleanUpPeers(); - -int64_t sdbGetVersion(); - -int32_t sdbGetRunStatus(); - -#define TSDB_MAX_NORMAL_TABLES 10000 -#define TSDB_MAX_SUPER_TABLES 1000 - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SDB_H diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7308715a87..49ed45559f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -309,6 +309,16 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_SESSIONS_PER_VNODE (300) #define TSDB_SESSIONS_PER_DNODE (TSDB_SESSIONS_PER_VNODE * TSDB_MAX_VNODES) +#define TSDB_MAX_MNODES 5 +#define TSDB_MAX_DNODES 10 +#define TSDB_MAX_ACCOUNTS 10 +#define TSDB_MAX_USERS 20 +#define TSDB_MAX_DBS 100 +#define TSDB_MAX_VGROUPS 1000 +#define TSDB_MAX_SUPER_TABLES 100 +#define TSDB_MAX_NORMAL_TABLES 1000 +#define TSDB_MAX_CHILD_TABLES 100000 + enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf7398..386c8a92f7 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdump) +#ADD_SUBDIRECTORY(taosdemo) +#ADD_SUBDIRECTORY(taosdump) diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index d768d2dd7c..d381b09055 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -20,9 +20,12 @@ extern "C" { #endif -bool mgmtCheckRedirect(void *handle); +int32_t mgmtInitMnodes(); +void mgmtCleanupMnodes(); -void mgmtGetMnodeIpList(SRpcIpSet *ipSet); +bool mgmtCheckRedirect(void *handle); +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp); int32_t mgmtRemoveMnode(uint32_t privateIp); diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 30ebe0909c..c64db286c6 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -13,8 +13,12 @@ * along with this program. If not, see . */ -#ifndef _sdbint_header_ -#define _sdbint_header_ +#ifndef TDENGINE_MNODE_SDB_H +#define TDENGINE_MNODE_SDB_H + +#ifdef __cplusplus +extern "C" { +#endif #include #include @@ -27,115 +31,46 @@ #include "hashint.h" #include "hashstr.h" -#include "sdb.h" #include "tchecksum.h" #include "tlog.h" #include "trpc.h" #include "tutil.h" -#define sdbError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ - } -#define sdbWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ - } -#define sdbPrint(...) \ - { tprintf("MND-SDB ", 255, __VA_ARGS__); } +enum _keytype { + SDB_KEYTYPE_STRING, + SDB_KEYTYPE_AUTO, + SDB_KEYTYPE_MAX +}; -#define mpeerError(...) \ - if (sdbDebugFlag & DEBUG_ERROR) { \ - tprintf("ERROR MND-MPEER ", 255, __VA_ARGS__); \ - } -#define mpeerWarn(...) \ - if (sdbDebugFlag & DEBUG_WARN) { \ - tprintf("WARN MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerTrace(...) \ - if (sdbDebugFlag & DEBUG_TRACE) { \ - tprintf("MND-MPEER ", sdbDebugFlag, __VA_ARGS__); \ - } -#define mpeerPrint(...) \ - { tprintf("MND-MPEER ", 255, __VA_ARGS__); } +enum _sdbaction { + SDB_TYPE_INSERT, + SDB_TYPE_DELETE, + SDB_TYPE_UPDATE, + SDB_TYPE_DECODE, + SDB_TYPE_ENCODE, + SDB_TYPE_DESTROY, + SDB_MAX_ACTION_TYPES +}; -#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) -#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) -#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) +uint64_t sdbGetVersion(); +bool sdbInServerState(); +bool sdbIsMaster(); -#define SDB_MAX_PEERS 4 -#define SDB_DELIMITER 0xFFF00F00 -#define SDB_ENDCOMMIT 0xAFFFAAAF +void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, + void *(*appTool)(char, void *, char *, int32_t, int32_t *)); +void sdbCloseTable(void *handle); -typedef struct { - uint64_t swVersion; - int16_t sdbFileVersion; - char reserved[6]; - TSCKSUM checkSum; -} SSdbHeader; - -typedef struct { - char type; - // short rowSize; - char *row; -} SSdbUpdate; - -typedef struct _SSdbTable { - SSdbHeader header; - int maxRows; - int dbId; - int32_t maxRowSize; - char name[TSDB_DB_NAME_LEN]; - char fn[128]; - int keyType; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; - int fd; - void *(*appTool)(char, void *, char *, int, int *); - pthread_mutex_t mutex; - SSdbUpdate * update; - int numOfUpdates; - int updatePos; -} SSdbTable; - -typedef struct { - int64_t id; - int64_t offset; - int rowSize; - void * row; -} SRowMeta; - -typedef struct { - int32_t delimiter; - int32_t rowSize; - int64_t id; - char data[]; -} SRowHead; - -typedef struct { - uint8_t dbId; - char type; - uint64_t version; - short dataLen; - char data[]; -} SForwardMsg; - -extern SSdbTable *tableList[]; -extern int sdbMaxPeers; -extern int sdbNumOfTables; -extern int64_t sdbVersion; - -int sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int dataLen); -int mpeerRetrieveRows(int fd, SSdbTable *pTable, uint64_t version); -void sdbResetTable(SSdbTable *pTable); -extern const int16_t sdbFileVersion; +void *sdbGetRow(void *handle, void *key); +void *sdbFetchRow(void *handle, void *pNode, void **ppRow); +int64_t sdbGetId(void *handle); +int64_t sdbGetNumOfRows(void *handle); +int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize); +int32_t sdbDeleteRow(void *handle, void *key); +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated); +#ifdef __cplusplus +} #endif + +#endif \ No newline at end of file diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 53304c7565..589beeaaba 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" +#include "taosdef.h" #include "tschemautil.h" #include "tscompression.h" #include "tskiplist.h" @@ -26,10 +27,11 @@ #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" +#include "mgmtDClient.h" #include "mgmtGrant.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" -#include "mgmtDClient.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" @@ -56,7 +58,6 @@ static void mgmtChildTableActionInit() { mgmtChildTableActionFp[SDB_TYPE_UPDATE] = mgmtChildTableActionUpdate; mgmtChildTableActionFp[SDB_TYPE_ENCODE] = mgmtChildTableActionEncode; mgmtChildTableActionFp[SDB_TYPE_DECODE] = mgmtChildTableActionDecode; - mgmtChildTableActionFp[SDB_TYPE_RESET] = mgmtChildTableActionReset; mgmtChildTableActionFp[SDB_TYPE_DESTROY] = mgmtChildTableActionDestroy; } @@ -93,7 +94,7 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss return NULL; } - if (!sdbMaster) { + if (!sdbIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); @@ -311,13 +312,6 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTab } void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - int32_t numOfTables = sdbGetNumOfRows(tsChildTableSdb); - if (numOfTables >= tsMaxTables) { - mError("ctable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, tsMaxTables); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - char *pTagData = (char *) pCreate->schema; // it is a tag key SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); if (pSuperTable == NULL) { @@ -338,7 +332,7 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_CHILD_TABLE; pTable->createdTime = taosGetTimestampMs(); pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + - ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = tid; pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index da11ad2081..cbdcb8ff5d 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -42,7 +42,7 @@ int32_t mgmtInitDClient() { rpcInit.label = "MND-DC"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessRspFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "mgmtDClient"; diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 177a45764c..b1d01b91f1 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -45,7 +45,7 @@ int32_t mgmtInitDServer() { rpcInit.label = "MND-DS"; rpcInit.numOfThreads = 1; rpcInit.cfp = mgmtProcessMsgFromDnode; - rpcInit.sessions = tsMaxDnodes * 5; + rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = mgmtDServerRetrieveAuth; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index caf84737b0..5cd1259486 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -24,11 +24,12 @@ #include "mgmtBalance.h" #include "mgmtDb.h" #include "mgmtDnode.h" -#include "mgmtMnode.h" #include "mgmtGrant.h" #include "mgmtShell.h" +#include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtChildTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtUser.h" @@ -62,7 +63,6 @@ static void mgmtDbActionInit() { mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate; mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode; mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode; - mgmtDbActionFp[SDB_TYPE_RESET] = mgmtDbActionReset; mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy; } @@ -83,7 +83,7 @@ int32_t mgmtInitDbs() { SDbObj tObj; tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; - tsDbSdb = sdbOpenTable(tsMaxDbs, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); + tsDbSdb = sdbOpenTable(TSDB_MAX_DBS, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); if (tsDbSdb == NULL) { mError("failed to init db data"); return -1; @@ -252,12 +252,6 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { } static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { - int32_t numOfDbs = sdbGetNumOfRows(tsDbSdb); - if (numOfDbs >= tsMaxDbs) { - mWarn("numOfDbs:%d, exceed tsMaxDbs:%d", numOfDbs, tsMaxDbs); - return TSDB_CODE_TOO_MANY_DATABASES; - } - int32_t code = mgmtCheckDbLimit(pAcct); if (code != 0) { return code; diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index c7dcefa5f4..77c314c56f 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -597,7 +597,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return; } - mgmtGetMnodeIpList(&pRsp->ipList); + mgmtGetMnodePrivateIpList(&pRsp->ipList); pRsp->dnodeState.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeState.moduleStatus = htonl(pDnode->moduleStatus); diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 66200e5a14..6aec06970e 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -25,6 +25,8 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtDServer.h" +#include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtVgroup.h" #include "mgmtUser.h" #include "mgmtTable.h" @@ -65,7 +67,7 @@ int32_t mgmtStartSystem() { return 0; } - tsMgmtTmr = taosTmrInit((tsMaxDnodes + tsMaxShellConns) * 3, 200, 3600000, "MND"); + tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND"); if (tsMgmtTmr == NULL) { mError("failed to init timer"); return -1; @@ -109,8 +111,8 @@ int32_t mgmtStartSystem() { return -1; } - if (sdbInitPeers(tsMnodeDir) < 0) { - mError("failed to init peers"); + if (mgmtInitMnodes() < 0) { + mError("failed to init mnodes"); return -1; } @@ -125,7 +127,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (sdbMaster) { + if (sdbIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } @@ -136,7 +138,7 @@ void mgmtStopSystem() { void mgmtCleanUpSystem() { mPrint("starting to clean up mgmt"); - sdbCleanUpPeers(); + mgmtCleanupMnodes(); mgmtCleanupBalance(); mgmtCleanUpShell(); mgmtCleanupDClient(); diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index dc322e742f..ac18af0158 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -18,48 +18,81 @@ #include "trpc.h" #include "tschemautil.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtUser.h" -int32_t (*mgmtAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; -int32_t (*mgmtRemoveMnodeFp)(uint32_t privateIp) = NULL; -int32_t (*mgmtGetMnodesNumFp)() = NULL; -void * (*mgmtGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerAddMnodeFp)(uint32_t privateIp, uint32_t publicIp) = NULL; +int32_t (*mpeerRemoveMnodeFp)(uint32_t privateIp) = NULL; +int32_t (*mpeerGetMnodesNumFp)() = NULL; +void * (*mpeerGetNextMnodeFp)(SShowObj *pShow, SMnodeObj **pMnode) = NULL; +int32_t (*mpeerInitMnodesFp)() = NULL; +void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; static int32_t mgmtGetMnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static char *mgmtMnodeStatusStr[] = { + "offline", + "unsynced", + "syncing", + "serving", + "null" +}; + +static char *mgmtMnodeRoleStr[] = { + "unauthed", + "undecided", + "master", + "slave", + "null" +}; + +int32_t mgmtInitMnodes() { + if (mpeerInitMnodesFp) { + return (*mpeerInitMnodesFp)(); + } else { + return 0; + } +} + +void mgmtCleanupMnodes() { + if (mpeerCleanUpMnodesFp) { + (*mpeerCleanUpMnodesFp)(); + } +} + bool mgmtCheckRedirect(void *handle) { return false; } int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp) { - if (mgmtAddMnodeFp) { - return (*mgmtAddMnodeFp)(privateIp, publicIp); + if (mpeerAddMnodeFp) { + return (*mpeerAddMnodeFp)(privateIp, publicIp); } else { return 0; } } int32_t mgmtRemoveMnode(uint32_t privateIp) { - if (mgmtRemoveMnodeFp) { - return (*mgmtRemoveMnodeFp)(privateIp); + if (mpeerRemoveMnodeFp) { + return (*mpeerRemoveMnodeFp)(privateIp); } else { return 0; } } static int32_t mgmtGetMnodesNum() { - if (mgmtGetMnodesNumFp) { - return (*mgmtGetMnodesNumFp)(); + if (mpeerGetMnodesNumFp) { + return (*mpeerGetMnodesNumFp)(); } else { return 1; } } static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { - if (mgmtGetNextMnodeFp) { - return (*mgmtGetNextMnodeFp)(pShow, pMnode); + if (mpeerGetNextMnodeFp) { + return (*mpeerGetNextMnodeFp)(pShow, pMnode); } else { if (*pMnode == NULL) { *pMnode = &tsMnodeObj; @@ -149,11 +182,11 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbStatusStr[(uint8_t)pMnode->status]); + strcpy(pWrite, mgmtMnodeStatusStr[pMnode->status]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, sdbRoleStr[(uint8_t)pMnode->role]); + strcpy(pWrite, mgmtMnodeRoleStr[pMnode->role]); cols++; tinet_ntoa(ipstr, pMnode->publicIp); @@ -168,7 +201,14 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi return numOfRows; } -void mgmtGetMnodeIpList(SRpcIpSet *ipSet) { +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet) { + ipSet->inUse = 0; + ipSet->port = htons(tsMnodeDnodePort); + ipSet->numOfIps = 1; + ipSet->ip[0] = htonl(inet_addr(tsMasterIp)); +} + +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet) { ipSet->inUse = 0; ipSet->port = htons(tsMnodeDnodePort); ipSet->numOfIps = 1; diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index ca138bf42d..d42a596008 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -27,6 +27,7 @@ #include "mgmtDClient.h" #include "mgmtGrant.h" #include "mgmtNormalTable.h" +#include "mgmtSdb.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" @@ -55,7 +56,6 @@ static void mgmtNormalTableActionInit() { mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate; mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode; mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode; - mgmtNormalTableActionFp[SDB_TYPE_RESET] = mgmtNormalTableActionReset; mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy; } @@ -98,7 +98,7 @@ void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *s return NULL; } - if (!sdbMaster) { + if (!sdbIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("sid:%d is not matched from the master:%d", sid, pTable->sid); @@ -222,7 +222,7 @@ int32_t mgmtInitNormalTables() { SNormalTableObj tObj; tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsNormalTableSdb = sdbOpenTable(tsMaxTables, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + tsNormalTableSdb = sdbOpenTable(TSDB_MAX_NORMAL_TABLES, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction); if (tsNormalTableSdb == NULL) { mError("failed to init ntables data"); @@ -323,13 +323,6 @@ void *mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable) { } void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid) { - int32_t numOfTables = sdbGetNumOfRows(tsNormalTableSdb); - if (numOfTables >= TSDB_MAX_NORMAL_TABLES) { - mError("table:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_NORMAL_TABLES); - terrno = TSDB_CODE_TOO_MANY_TABLES; - return NULL; - } - SNormalTableObj *pTable = (SNormalTableObj *) calloc(sizeof(SNormalTableObj), 1); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); @@ -341,7 +334,7 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->type = TSDB_NORMAL_TABLE; pTable->vgId = pVgroup->vgId; pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = sid; pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 04802e77da..3564392ed1 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -15,60 +15,107 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "sdb.h" +#include "taosdef.h" #include "tutil.h" #include "mgmtSdb.h" #define abs(x) (((x) < 0) ? -(x) : (x)) +#define SDB_MAX_PEERS 4 +#define SDB_DELIMITER 0xFFF00F00 +#define SDB_ENDCOMMIT 0xAFFFAAAF +#define SDB_STATUS_OFFLINE 0 +#define SDB_STATUS_SERVING 1 + + +typedef struct { + uint64_t swVersion; + int16_t sdbFileVersion; + char reserved[6]; + TSCKSUM checkSum; +} SSdbHeader; + +typedef struct _SSdbTable { + SSdbHeader header; + int maxRows; + int dbId; + int32_t maxRowSize; + char name[TSDB_DB_NAME_LEN]; + char fn[128]; + int keyType; + uint32_t autoIndex; + int64_t numOfRows; + int64_t id; + int64_t size; + void * iHandle; + int fd; + void *(*appTool)(char, void *, char *, int, int *); + pthread_mutex_t mutex; +} SSdbTable; + +typedef struct { + int64_t id; + int64_t offset; + int rowSize; + void * row; +} SRowMeta; + +typedef struct { + int32_t delimiter; + int32_t rowSize; + int64_t id; + char data[]; +} SRowHead; + +typedef struct { + uint8_t dbId; + char type; + uint64_t version; + short dataLen; + char data[]; +} SForwardMsg; extern char version[]; -const int16_t sdbFileVersion = 0; -SRpcIpSet *pSdbIpList = NULL; -SRpcIpSet *pSdbPublicIpList = NULL; -SSdbPeer * sdbPeer[SDB_MAX_PEERS]; // first slot for self +const int16_t sdbFileVersion = 2; +int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; -#ifdef CLUSTER -int sdbMaster = 0; -#else -int sdbMaster = 1; -#endif +static SSdbTable *sdbTableList[10] = {0}; +static int32_t sdbNumOfTables = 0; +static uint64_t sdbVersion = 0; +static int32_t sdbMaster = 0; +static int32_t sdbStatus = SDB_STATUS_OFFLINE; -void *(*sdbInitIndexFp[])(int maxRows, int dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; -void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash}; +// #ifdef CLUSTER +// int32_t sdbMaster = 0; +// #else +// int32_t sdbMaster = 1; +// #endif -void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash}; +static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; +static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; +static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash}; +static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData}; +static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; +static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; -void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData}; +void sdbResetTable(SSdbTable *pTable); +void sdbSaveSnapShot(void *handle); -void (*sdbCleanUpIndexFp[])(void *handle) = { - sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash, -}; +uint64_t sdbGetVersion() { return sdbVersion; } +bool sdbInServerState() { return sdbStatus == SDB_STATUS_SERVING; } +bool sdbIsMaster() { return sdbMaster; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } +int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } -void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = { - sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData, -}; - -SSdbTable *tableList[20]; -int sdbNumOfTables; -int64_t sdbVersion; - -int64_t sdbGetVersion() { - return sdbVersion; -}; - -int32_t sdbGetRunStatus() { - if (!tsIsCluster) { - return SDB_STATUS_SERVING; +static int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { + if (mpeerForwardRequestFp) { + return mpeerForwardRequestFp(pTable, type, data, dataLen); + } else { + return 0; } - - if (sdbInited == NULL) { - return SDB_STATUS_OFFLINE; - } - return sdbStatus; } -void sdbFinishCommit(void *handle) { +static void sdbFinishCommit(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; uint32_t sdbEcommit = SDB_ENDCOMMIT; @@ -78,7 +125,7 @@ void sdbFinishCommit(void *handle) { pTable->size += sizeof(sdbEcommit); } -int sdbOpenSdbFile(SSdbTable *pTable) { +static int32_t sdbOpenSdbFile(SSdbTable *pTable) { struct stat fstat, ofstat; uint64_t size; char * dirc = NULL; @@ -91,7 +138,7 @@ int sdbOpenSdbFile(SSdbTable *pTable) { memcpy(swVersion.cversion, version, sizeof(uint64_t)); // check sdb.db and .sdb.db status - char fn[128] = "\0"; + char fn[TSDB_FILENAME_LEN] = "\0"; dirc = strdup(pTable->fn); basec = strdup(pTable->fn); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); @@ -168,27 +215,15 @@ int sdbOpenSdbFile(SSdbTable *pTable) { return pTable->fd; } -// TODO: Change here -void sdbAddIntoUpdateList(SSdbTable *pTable, char type, char *row) { - pTable->numOfUpdates++; - pTable->updatePos = pTable->numOfUpdates % pTable->maxRows; - - if (pTable->update[pTable->updatePos].type == SDB_TYPE_DELETE) - (*(pTable->appTool))(SDB_TYPE_DESTROY, pTable->update[pTable->updatePos].row, NULL, 0, NULL); - - pTable->update[pTable->updatePos].type = type; - pTable->update[pTable->updatePos].row = row; -} - -int sdbInitTableByFile(SSdbTable *pTable) { +static int32_t sdbInitTableByFile(SSdbTable *pTable) { SRowMeta rowMeta; - int numOfDels = 0; - int bytes = 0; + int32_t numOfDels = 0; + int32_t bytes = 0; int64_t oldId = 0; void * pMetaRow = NULL; - int total_size = 0; - int real_size = 0; - int maxAutoIndex = 0; + int32_t total_size = 0; + int32_t real_size = 0; + int32_t maxAutoIndex = 0; oldId = pTable->id; if (sdbOpenSdbFile(pTable) < 0) return -1; @@ -277,7 +312,7 @@ int sdbInitTableByFile(SSdbTable *pTable) { numOfDels++; } else { // Reset the object TODO: is it possible to merge reset and // update ?? - (*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); + //(*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); } numOfDels++; } @@ -293,9 +328,6 @@ int sdbInitTableByFile(SSdbTable *pTable) { sdbVersion += (pTable->id - oldId); if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable); - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - tfree(rowHead); return 0; @@ -304,20 +336,12 @@ sdb_exit1: return -1; } -void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int, int *)) { +void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, + void *(*appTool)(char, void *, char *, int32_t, int32_t *)) { SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); if (pTable == NULL) return NULL; memset(pTable, 0, sizeof(SSdbTable)); - int size = sizeof(SSdbUpdate) * maxRows; - pTable->update = (SSdbUpdate *)malloc(size); - if (pTable->update == NULL) { - free(pTable); - return NULL; - }; - memset(pTable->update, 0, size); - strcpy(pTable->name, name); pTable->keyType = keyType; pTable->maxRows = maxRows; @@ -332,14 +356,14 @@ void *sdbOpenTable(int maxRows, int32_t maxRowSize, char *name, uint8_t keyType, if (sdbInitTableByFile(pTable) < 0) return NULL; pTable->dbId = sdbNumOfTables++; - tableList[pTable->dbId] = pTable; + sdbTableList[pTable->dbId] = pTable; sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d", pTable->name, pTable->numOfRows, sdbNumOfTables); return pTable; } -SRowMeta *sdbGetRowMeta(void *handle, void *key) { +static SRowMeta *sdbGetRowMeta(void *handle, void *key) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; @@ -365,15 +389,14 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } -// row here must be encoded string (rowSize > 0) or the object it self (rowSize -// = 0) -int64_t sdbInsertRow(void *handle, void *row, int rowSize) { +// row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) +int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; int64_t id = -1; void * pObj = NULL; - int total_size = 0; - int real_size = 0; + int32_t total_size = 0; + int32_t real_size = 0; /* char action = SDB_TYPE_INSERT; */ if (pTable == NULL) { @@ -398,9 +421,6 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) { case SDB_KEYTYPE_STRING: sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - break; case SDB_KEYTYPE_AUTO: sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); break; @@ -464,18 +484,12 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) { pTable->size += real_size; sdbFinishCommit(pTable); - sdbAddIntoUpdateList(pTable, SDB_TYPE_INSERT, rowMeta.row); - pTable->numOfRows++; switch (pTable->keyType) { case SDB_KEYTYPE_STRING: sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; case SDB_KEYTYPE_AUTO: sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); @@ -502,14 +516,14 @@ int64_t sdbInsertRow(void *handle, void *row, int rowSize) { } // row here can be object or null-terminated string -int sdbDeleteRow(void *handle, void *row) { +int32_t sdbDeleteRow(void *handle, void *row) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int code = -1; + int32_t code = -1; void * pMetaRow = NULL; SRowHead * rowHead = NULL; - int rowSize = 0; - int total_size = 0; + int32_t rowSize = 0; + int32_t total_size = 0; /* char action = SDB_TYPE_DELETE; */ if (pTable == NULL) return -1; @@ -527,9 +541,6 @@ int sdbDeleteRow(void *handle, void *row) { case SDB_KEYTYPE_STRING: rowSize = strlen((char *)row) + 1; break; - case SDB_KEYTYPE_UINT32: - rowSize = sizeof(uint32_t); - break; case SDB_KEYTYPE_AUTO: rowSize = sizeof(uint64_t); break; @@ -568,17 +579,12 @@ int sdbDeleteRow(void *handle, void *row) { sdbFinishCommit(pTable); pTable->numOfRows--; - // TODO:Change the update list here - sdbAddIntoUpdateList(pTable, SDB_TYPE_DELETE, pMetaRow); + switch (pTable->keyType) { case SDB_KEYTYPE_STRING: sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; case SDB_KEYTYPE_AUTO: sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); @@ -606,12 +612,12 @@ int sdbDeleteRow(void *handle, void *row) { } // row here can be the object or the string info (encoded string) -int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int code = -1; - int total_size = 0; - int real_size = 0; + int32_t code = -1; + int32_t total_size = 0; + int32_t real_size = 0; /* char action = SDB_TYPE_UPDATE; */ if (pTable == NULL || row == NULL) return -1; @@ -622,10 +628,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, (char *) row, sdbVersion, pTable->id); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *) row), sdbVersion, pTable->id); - break; case SDB_KEYTYPE_AUTO: sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *) row, sdbVersion, pTable->id); @@ -694,10 +696,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); break; - case SDB_KEYTYPE_UINT32: //dnodes or mnodes - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id, pTable->numOfRows); - break; case SDB_KEYTYPE_AUTO: sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); @@ -708,7 +706,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { break; } - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, pMetaRow); code = 0; } @@ -719,79 +716,6 @@ int sdbUpdateRow(void *handle, void *row, int updateSize, char isUpdated) { return code; } -// row here must be the instruction string -int sdbBatchUpdateRow(void *handle, void *row, int rowSize) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int total_size = 0; - /* char action = SDB_TYPE_BATCH_UPDATE; */ - - if (pTable == NULL || row == NULL || rowSize <= 0) return -1; - pMeta = sdbGetRowMeta(handle, row); - if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, batch update failed", pTable->name); - return -1; - } - - void *pMetaRow = pMeta->row; - assert(pMetaRow != NULL); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - - pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_BATCH_UPDATE, row, rowSize) == 0) { - /* // write action */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - - (*(pTable->appTool))(SDB_TYPE_BEFORE_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - - void *next_row = pMetaRow; - while (next_row != NULL) { - pTable->id++; - sdbVersion++; - - void *last_row = next_row; - next_row = (*(pTable->appTool))(SDB_TYPE_BATCH_UPDATE, last_row, (char *)row, rowSize, 0); - memset(rowHead, 0, sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM)); - - // update in current layer - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - - // write to disk - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, last_row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - taosCalcChecksumAppend(0, (uint8_t *)rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pMeta->rowSize = rowHead->rowSize; - lseek(pTable->fd, pTable->size, SEEK_SET); - twrite(pTable->fd, rowHead, sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - pTable->size += (sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM)); - - sdbAddIntoUpdateList(pTable, SDB_TYPE_UPDATE, last_row); - - if (next_row != NULL) { - pMeta = sdbGetRowMeta(handle, next_row); - } - } - - sdbFinishCommit(pTable); - - (*(pTable->appTool))(SDB_TYPE_AFTER_BATCH_UPDATE, pMetaRow, NULL, 0, NULL); - } - pthread_mutex_unlock(&pTable->mutex); - - tfree(rowHead); - - return 0; -} - void sdbCloseTable(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; void * pNode = NULL; @@ -814,20 +738,19 @@ void sdbCloseTable(void *handle) { sdbNumOfTables--; sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->name, pTable->id, sdbNumOfTables); - tfree(pTable->update); tfree(pTable); } void sdbResetTable(SSdbTable *pTable) { /* SRowHead rowHead; */ SRowMeta rowMeta; - int bytes; - int total_size = 0; - int real_size = 0; + int32_t bytes; + int32_t total_size = 0; + int32_t real_size = 0; SRowHead *rowHead = NULL; void * pMetaRow = NULL; int64_t oldId = pTable->id; - int oldNumOfRows = pTable->numOfRows; + int32_t oldNumOfRows = pTable->numOfRows; if (sdbOpenSdbFile(pTable) < 0) return; pTable->numOfRows = oldNumOfRows; @@ -911,24 +834,21 @@ void sdbResetTable(SSdbTable *pTable) { } sdbVersion += (pTable->id - oldId); - pTable->numOfUpdates = 0; - pTable->updatePos = 0; - + tfree(rowHead); sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); } -// TODO:A problem here :use snapshot file to sync another node will cause -// problem +// TODO:A problem here :use snapshot file to sync another node will cause problem void sdbSaveSnapShot(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; void * pNode = NULL; - int total_size = 0; - int real_size = 0; - int size = 0; - int numOfRows = 0; + int32_t total_size = 0; + int32_t real_size = 0; + int32_t size = 0; + int32_t numOfRows = 0; uint32_t sdbEcommit = SDB_ENDCOMMIT; char * dirc = NULL; char * basec = NULL; @@ -942,7 +862,7 @@ void sdbSaveSnapShot(void *handle) { dirc = strdup(pTable->fn); basec = strdup(pTable->fn); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - int fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); tfree(dirc); tfree(basec); @@ -1011,50 +931,3 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { return pNode; } - -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } - -int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } - - -int32_t (*mpeerInitMnodesFp)(char *directory) = NULL; -void (*mpeerCleanUpMnodesFp)() = NULL; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; - -char *sdbStatusStr[] = { - "offline", - "unsynced", - "syncing", - "serving", - "null" -}; - -char *sdbRoleStr[] = { - "unauthed", - "undecided", - "master", - "slave", - "null" -}; - -int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { - if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); - } else { - return 0; - } -} - -int32_t sdbInitPeers(char *directory) { - if (mpeerInitMnodesFp) { - return (*mpeerInitMnodesFp)(directory); - } else { - return 0; - } -} - -void sdbCleanUpPeers() { - if (mpeerCleanUpMnodesFp) { - (*mpeerCleanUpMnodesFp)(); - } -} diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index e87c964496..c50ecc01c7 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -32,6 +32,7 @@ #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtTable.h" @@ -63,7 +64,7 @@ int32_t mgmtInitShell() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_HEARTBEAT, mgmtProcessHeartBeatMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg); - tsMgmtTranQhandle = taosInitScheduler(tsMaxDnodes + tsMaxShellConns, 1, "mnodeT"); + tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT"); int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0; if (numOfThreads < 1) { @@ -131,7 +132,7 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { } static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { - if (sdbGetRunStatus() != SDB_STATUS_SERVING) { + if (!sdbInServerState()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; @@ -309,20 +310,10 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { return; } - pHBRsp->ipList.inUse = 0; - pHBRsp->ipList.port = htons(tsMnodeShellPort); - pHBRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pHBRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pHBRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pHBRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pHBRsp->ipList); } /* @@ -411,20 +402,11 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { strcpy(pConnectRsp->serverVersion, version); pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - pConnectRsp->ipList.inUse = 0; - pConnectRsp->ipList.port = htons(tsMnodeShellPort); - pConnectRsp->ipList.numOfIps = 0; - if (pSdbPublicIpList != NULL && pSdbIpList != NULL) { - pConnectRsp->ipList.numOfIps = htons(pSdbPublicIpList->numOfIps); - if (connInfo.serverIp == tsPublicIpInt) { - for (int i = 0; i < pSdbPublicIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbPublicIpList->ip[i]); - } - } else { - for (int i = 0; i < pSdbIpList->numOfIps; ++i) { - pConnectRsp->ipList.ip[i] = htonl(pSdbIpList->ip[i]); - } - } + + if (connInfo.serverIp == tsPublicIpInt) { + mgmtGetMnodePublicIpList(&pConnectRsp->ipList); + } else { + mgmtGetMnodePrivateIpList(&pConnectRsp->ipList); } connect_over: diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index b0d506980a..075633cd4d 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -31,6 +31,7 @@ #include "mgmtGrant.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" +#include "mgmtSdb.h" #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" @@ -63,7 +64,6 @@ static void mgmtSuperTableActionInit() { mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate; mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode; mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode; - mgmtSuperTableActionFp[SDB_TYPE_RESET] = mgmtSuperTableActionReset; mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy; } @@ -164,7 +164,7 @@ int32_t mgmtInitSuperTables() { mgmtSuperTableActionInit(); - tsSuperTableSdb = sdbOpenTable(tsMaxTables, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + tsSuperTableSdb = sdbOpenTable(TSDB_MAX_SUPER_TABLES, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); @@ -201,12 +201,6 @@ void mgmtCleanUpSuperTables() { } int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { - int32_t numOfTables = sdbGetNumOfRows(tsSuperTableSdb); - if (numOfTables >= TSDB_MAX_SUPER_TABLES) { - mError("stable:%s, numOfTables:%d exceed maxTables:%d", pCreate->tableId, numOfTables, TSDB_MAX_SUPER_TABLES); - return TSDB_CODE_TOO_MANY_TABLES; - } - SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1); if (pStable == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; @@ -217,7 +211,7 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { pStable->createdTime = taosGetTimestampMs(); pStable->vgId = 0; pStable->sid = 0; - pStable->uid = (((uint64_t) pStable->createdTime) << 16) + ((uint64_t) sdbGetVersion() & ((1ul << 16) - 1ul)); + pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pStable->sversion = 0; pStable->numOfColumns = htons(pCreate->numOfColumns); pStable->numOfTags = htons(pCreate->numOfTags); diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 6001c97a92..34c1a69cb1 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -24,6 +24,7 @@ #include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" #include "mgmtUser.h" diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 22c6cbc1dc..33caa75988 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -21,6 +21,7 @@ #include "mgmtAcct.h" #include "mgmtGrant.h" #include "mgmtMnode.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtUser.h" @@ -59,7 +60,7 @@ int32_t mgmtInitUsers() { SUserObj tObj; tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsUserSdb = sdbOpenTable(tsMaxUsers, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); + tsUserSdb = sdbOpenTable(TSDB_MAX_USERS, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); if (tsUserSdb == NULL) { mError("failed to init user data"); return -1; @@ -106,12 +107,6 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) { } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { - int32_t numOfUsers = sdbGetNumOfRows(tsUserSdb); - if (numOfUsers >= tsMaxUsers) { - mWarn("numOfUsers:%d, exceed tsMaxUsers:%d", numOfUsers, tsMaxUsers); - return TSDB_CODE_TOO_MANY_USERS; - } - int32_t code = mgmtCheckUserLimit(pAcct); if (code != 0) { return code; @@ -257,7 +252,6 @@ static void mgmtUserActionInit() { mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate; mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode; mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode; - mgmtUserActionFp[SDB_TYPE_RESET] = mgmtUserActionReset; mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy; } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index be99148deb..62a2abd17d 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -25,6 +25,7 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtProfile.h" +#include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtTable.h" #include "mgmtVgroup.h" @@ -58,7 +59,6 @@ static void mgmtVgroupActionInit() { mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate; mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode; mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode; - mgmtVgroupActionFp[SDB_TYPE_RESET] = mgmtVgroupActionReset; mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy; } @@ -75,7 +75,7 @@ int32_t mgmtInitVgroups() { mgmtVgroupActionInit(); - tsVgroupSdb = sdbOpenTable(tsMaxVGroups, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); + tsVgroupSdb = sdbOpenTable(TSDB_MAX_VGROUPS, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); if (tsVgroupSdb == NULL) { mError("failed to init vgroups data"); return -1; diff --git a/src/util/inc/tglobalcfg.h b/src/util/inc/tglobalcfg.h index 993992ffcb..664eb16373 100644 --- a/src/util/inc/tglobalcfg.h +++ b/src/util/inc/tglobalcfg.h @@ -103,13 +103,7 @@ extern int tsReplications; extern int tsNumOfMPeers; extern int tsMaxShellConns; -extern int tsMaxAccounts; -extern int tsMaxUsers; -extern int tsMaxDbs; extern int tsMaxTables; -extern int tsMaxDnodes; -extern int tsMaxVGroups; -extern char tsMgmtZone[]; extern char tsLocalIp[]; extern char tsDefaultDB[]; diff --git a/src/util/inc/tlog.h b/src/util/inc/tlog.h index 350a2c700e..3a327b0a47 100644 --- a/src/util/inc/tlog.h +++ b/src/util/inc/tlog.h @@ -239,6 +239,25 @@ extern uint32_t cdebugFlag; #define monitorLWarn(...) taosLogWarn(__VA_ARGS__) monitorWarn(__VA_ARGS__) #define monitorLPrint(...) taosLogPrint(__VA_ARGS__) monitorPrint(__VA_ARGS__) +#define sdbError(...) \ + if (sdbDebugFlag & DEBUG_ERROR) { \ + tprintf("ERROR MND-SDB ", 255, __VA_ARGS__); \ + } +#define sdbWarn(...) \ + if (sdbDebugFlag & DEBUG_WARN) { \ + tprintf("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbTrace(...) \ + if (sdbDebugFlag & DEBUG_TRACE) { \ + tprintf("MND-SDB ", sdbDebugFlag, __VA_ARGS__); \ + } +#define sdbPrint(...) \ + { tprintf("MND-SDB ", 255, __VA_ARGS__); } + +#define sdbLError(...) taosLogError(__VA_ARGS__) sdbError(__VA_ARGS__) +#define sdbLWarn(...) taosLogWarn(__VA_ARGS__) sdbWarn(__VA_ARGS__) +#define sdbLPrint(...) taosLogPrint(__VA_ARGS__) sdbPrint(__VA_ARGS__) + #ifdef __cplusplus } #endif diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index bbea30207c..02de88881d 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -112,13 +112,7 @@ int tsReplications = TSDB_REPLICA_MIN_NUM; int tsNumOfMPeers = 3; int tsMaxShellConns = 2000; -int tsMaxAccounts = 100; -int tsMaxUsers = 1000; -int tsMaxDbs = 1000; -int tsMaxTables = 650000; -int tsMaxDnodes = 1000; -int tsMaxVGroups = 1000; -char tsMgmtZone[16] = "rzone"; +int tsMaxTables = 100000; char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsDefaultDB[TSDB_DB_NAME_LEN] = {0}; @@ -612,28 +606,10 @@ static void doInitGlobalConfig() { 1, 8640000, 0, TSDB_CFG_UTYPE_SECOND); // mgmt configs - tsInitConfigOption(cfg++, "mgmtZone", tsMgmtZone, TSDB_CFG_VTYPE_STRING, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 0, 0, 16, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxAccounts", &tsMaxAccounts, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxUsers", &tsMaxUsers, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDbs", &tsMaxDbs, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 10000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "maxTables", &tsMaxTables, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 1, 100000000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxDnodes", &tsMaxDnodes, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLUSTER, - 1, 1000, 0, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "maxVGroups", &tsMaxVGroups, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, - 1, 1000000, 0, TSDB_CFG_UTYPE_NONE); - + tsInitConfigOption(cfg++, "minSlidingTime", &tsMinSlidingTime, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 10, 1000000, 0, TSDB_CFG_UTYPE_MS); From e9a07987fdd123cde221064372a0cc2fc1d7f8ce Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 20 Mar 2020 15:09:01 +0800 Subject: [PATCH 03/85] [TD-15] refactor sdb --- src/mnode/inc/mgmtMnode.h | 3 + src/mnode/inc/mgmtSdb.h | 34 ++-- src/mnode/src/mgmtChildTable.c | 17 +- src/mnode/src/mgmtDb.c | 4 +- src/mnode/src/mgmtMain.c | 2 +- src/mnode/src/mgmtMnode.c | 12 ++ src/mnode/src/mgmtNormalTable.c | 19 +- src/mnode/src/mgmtSdb.c | 326 +++++++++++++++----------------- src/mnode/src/mgmtShell.c | 2 +- src/mnode/src/mgmtSuperTable.c | 16 +- src/mnode/src/mgmtTable.c | 8 +- src/mnode/src/mgmtUser.c | 6 +- src/mnode/src/mgmtVgroup.c | 14 +- 13 files changed, 224 insertions(+), 239 deletions(-) diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index d381b09055..729d31544f 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -23,6 +23,9 @@ extern "C" { int32_t mgmtInitMnodes(); void mgmtCleanupMnodes(); +bool mgmtInServerStatus(); +bool mgmtIsMaster(); + bool mgmtCheckRedirect(void *handle); void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index c64db286c6..7381ec6669 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -20,27 +20,11 @@ extern "C" { #endif -#include -#include -#include -#include -#include -#include -#include -#include - -#include "hashint.h" -#include "hashstr.h" -#include "tchecksum.h" -#include "tlog.h" -#include "trpc.h" -#include "tutil.h" - enum _keytype { SDB_KEYTYPE_STRING, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_MAX -}; +} ESdbKeyType; enum _sdbaction { SDB_TYPE_INSERT, @@ -50,11 +34,15 @@ enum _sdbaction { SDB_TYPE_ENCODE, SDB_TYPE_DESTROY, SDB_MAX_ACTION_TYPES -}; +} ESdbType; + +typedef enum { + SDB_OPER_GLOBAL, + SDB_OPER_LOCAL, + SDB_OPER_DISK +} ESdbOper; uint64_t sdbGetVersion(); -bool sdbInServerState(); -bool sdbIsMaster(); void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, void *(*appTool)(char, void *, char *, int32_t, int32_t *)); @@ -65,9 +53,9 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow); int64_t sdbGetId(void *handle); int64_t sdbGetNumOfRows(void *handle); -int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize); -int32_t sdbDeleteRow(void *handle, void *key); -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated); +int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper); +int32_t sdbDeleteRow(void *handle, void *key, ESdbOper oper); +int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOper oper); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 589beeaaba..be78983c91 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -29,6 +29,7 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -94,7 +95,7 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss return NULL; } - if (!sdbIsMaster()) { + if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); @@ -214,7 +215,7 @@ int32_t mgmtInitChildTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -223,7 +224,7 @@ int32_t mgmtInitChildTables() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -232,7 +233,7 @@ int32_t mgmtInitChildTables() { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -240,7 +241,7 @@ int32_t mgmtInitChildTables() { if (pVgroup->tableList == NULL) { mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -252,7 +253,7 @@ int32_t mgmtInitChildTables() { if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -337,7 +338,7 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; - if (sdbInsertRow(tsChildTableSdb, pTable, 0) < 0) { + if (sdbInsertRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; @@ -481,7 +482,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_LOCAL); pNode = pLastNode; numOfTables ++; continue; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 5cd1259486..e889d200f5 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -279,7 +279,7 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->createdTime = taosGetTimestampMs(); pDb->cfg = *pCreate; - if (sdbInsertRow(tsDbSdb, pDb, 0) < 0) { + if (sdbInsertRow(tsDbSdb, pDb, SDB_OPER_GLOBAL) < 0) { code = TSDB_CODE_SDB_ERROR; tfree(pDb); } @@ -819,7 +819,7 @@ static void mgmtDropDb(void *handle, void *tmrId) { SDbObj *pDb = newMsg->ahandle; mPrint("db:%s, drop db from sdb", pDb->name); - int32_t code = sdbDeleteRow(tsDbSdb, pDb); + int32_t code = sdbDeleteRow(tsDbSdb, pDb, SDB_OPER_GLOBAL); if (code != 0) { code = TSDB_CODE_SDB_ERROR; } diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index 6aec06970e..ab0b99fa58 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -127,7 +127,7 @@ int32_t mgmtStartSystem() { void mgmtStopSystem() { - if (sdbIsMaster()) { + if (mgmtIsMaster()) { mTrace("it is a master mgmt node, it could not be stopped"); return; } diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index ac18af0158..8e459bf5af 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -29,6 +29,8 @@ int32_t (*mpeerInitMnodesFp)() = NULL; void (*mpeerCleanUpMnodesFp)() = NULL; static SMnodeObj tsMnodeObj = {0}; +static bool tsMnodeIsMaster = false; +static bool tsMnodeIsServing = false; static int32_t mgmtGetMnodeMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -52,6 +54,8 @@ int32_t mgmtInitMnodes() { if (mpeerInitMnodesFp) { return (*mpeerInitMnodesFp)(); } else { + tsMnodeIsServing = true; + tsMnodeIsMaster = true; return 0; } } @@ -62,6 +66,14 @@ void mgmtCleanupMnodes() { } } +bool mgmtInServerStatus() { + return tsMnodeIsServing; +} + +bool mgmtIsMaster() { + return tsMnodeIsMaster; +} + bool mgmtCheckRedirect(void *handle) { return false; } diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index d42a596008..df9fe0f439 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -26,6 +26,7 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtGrant.h" +#include "mgmtMnode.h" #include "mgmtNormalTable.h" #include "mgmtSdb.h" #include "mgmtSuperTable.h" @@ -98,7 +99,7 @@ void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *s return NULL; } - if (!sdbIsMaster()) { + if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("sid:%d is not matched from the master:%d", sid, pTable->sid); @@ -237,7 +238,7 @@ int32_t mgmtInitNormalTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ntable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -246,7 +247,7 @@ int32_t mgmtInitNormalTables() { if (pVgroup == NULL) { mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -255,7 +256,7 @@ int32_t mgmtInitNormalTables() { mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -263,7 +264,7 @@ int32_t mgmtInitNormalTables() { if (pVgroup->tableList == NULL) { mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -370,7 +371,7 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); } - if (sdbInsertRow(tsNormalTableSdb, pTable, 0) < 0) { + if (sdbInsertRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { mError("table:%s, update sdb error", pTable->tableId); free(pTable); terrno = TSDB_CODE_SDB_ERROR; @@ -467,7 +468,7 @@ int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int3 pTable->sversion++; pAcct->acctInfo.numOfTimeSeries += ncols; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -496,7 +497,7 @@ int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) pTable->sversion++; pAcct->acctInfo.numOfTimeSeries--; - sdbUpdateRow(tsNormalTableSdb, pTable, 0, 1); + sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -555,7 +556,7 @@ void mgmtDropAllNormalTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_LOCAL); pNode = pLastNode; numOfTables ++; continue; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 3564392ed1..33bfe4622b 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -17,15 +17,18 @@ #include "os.h" #include "taosdef.h" #include "tutil.h" +#include "tchecksum.h" +#include "tlog.h" +#include "trpc.h" +#include "tutil.h" +#include "hashint.h" +#include "hashstr.h" #include "mgmtSdb.h" #define abs(x) (((x) < 0) ? -(x) : (x)) #define SDB_MAX_PEERS 4 #define SDB_DELIMITER 0xFFF00F00 #define SDB_ENDCOMMIT 0xAFFFAAAF -#define SDB_STATUS_OFFLINE 0 -#define SDB_STATUS_SERVING 1 - typedef struct { uint64_t swVersion; @@ -81,15 +84,6 @@ int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32 static SSdbTable *sdbTableList[10] = {0}; static int32_t sdbNumOfTables = 0; static uint64_t sdbVersion = 0; -static int32_t sdbMaster = 0; -static int32_t sdbStatus = SDB_STATUS_OFFLINE; - - -// #ifdef CLUSTER -// int32_t sdbMaster = 0; -// #else -// int32_t sdbMaster = 1; -// #endif static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash}; static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash}; @@ -102,8 +96,6 @@ void sdbResetTable(SSdbTable *pTable); void sdbSaveSnapShot(void *handle); uint64_t sdbGetVersion() { return sdbVersion; } -bool sdbInServerState() { return sdbStatus == SDB_STATUS_SERVING; } -bool sdbIsMaster() { return sdbMaster; } int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } @@ -319,6 +311,10 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->size += real_size; if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); + + //TODO: check this valid + pTable->size += 4; + lseek(pTable->fd, 4, SEEK_CUR); } if (pTable->keyType == SDB_KEYTYPE_AUTO) { @@ -390,81 +386,67 @@ void *sdbGetRow(void *handle, void *key) { } // row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) -int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { +int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; - int64_t id = -1; void * pObj = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - /* char action = SDB_TYPE_INSERT; */ + int32_t total_size = 0; + int32_t real_size = 0; if (pTable == NULL) { sdbError("sdb tables is null"); return -1; } - - if ((pTable->keyType != SDB_KEYTYPE_AUTO) || *((int64_t *)row)) - if (sdbGetRow(handle, row)) { - if (strcmp(pTable->name, "mnode") == 0) { - /* - * The first mnode created when the system just start, so the insert action may failed - * see sdbPeer.c : sdbInitPeers - */ - pTable->id++; - sdbVersion++; - sdbPrint("table:%s, record:%s already exist, think it successed, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, taosIpStr(*(int32_t *)row), sdbVersion, pTable->id); - return 0; - } else { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); - break; - } - return -1; - } + + if (sdbGetRow(handle, row)) { + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); + break; + case SDB_KEYTYPE_AUTO: + sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); + break; + default: + sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); + break; } + return -1; + } total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb: %s", pTable->name); + sdbError("table:%s, failed to allocate row head memory", pTable->name); return -1; } memset(rowHead, 0, total_size); - if (rowSize == 0) { // object is created already + if (oper == SDB_OPER_GLOBAL) { pObj = row; - } else { // encoded string, to create object - pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, rowSize, NULL); - } - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); - + } else { + pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, 0, NULL); + } + pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - // TODO:here need to change - *((uint32_t *)pObj) = ++pTable->autoIndex; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); - } + if (oper == SDB_OPER_GLOBAL) { + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) != 0) { + sdbError("table:%s, failed to insert record", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + } + + if (oper == SDB_OPER_GLOBAL || oper == SDB_OPER_LOCAL) { + (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; + rowHead->id = pTable->id + 1; if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + sdbError("table:%s, failed to get checksum while inserting", pTable->name); pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); return -1; @@ -477,14 +459,10 @@ int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { rowMeta.row = pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - /* Update the disk content */ - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ twrite(pTable->fd, rowHead, real_size); pTable->size += real_size; sdbFinishCommit(pTable); - pTable->numOfRows++; switch (pTable->keyType) { case SDB_KEYTYPE_STRING: sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, @@ -499,32 +477,33 @@ int64_t sdbInsertRow(void *handle, void *row, int32_t rowSize) { pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); break; } + } - id = rowMeta.id; - } else { - sdbError("table:%s, failed to insert record", pTable->name); + if (pTable->keyType == SDB_KEYTYPE_AUTO) { + *((uint32_t *)pObj) = ++pTable->autoIndex; } - tfree(rowHead); + pTable->numOfRows++; + pTable->id++; + sdbVersion++; pthread_mutex_unlock(&pTable->mutex); - /* callback function to update the MGMT layer */ - if (id >= 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); + (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); - return id; + tfree(rowHead); + + return 0; } // row here can be object or null-terminated string -int32_t sdbDeleteRow(void *handle, void *row) { +int32_t sdbDeleteRow(void *handle, void *row, ESdbOper oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int32_t code = -1; void * pMetaRow = NULL; SRowHead * rowHead = NULL; - int32_t rowSize = 0; - int32_t total_size = 0; - /* char action = SDB_TYPE_DELETE; */ + int32_t rowSize = 0; + int32_t total_size = 0; if (pTable == NULL) return -1; @@ -558,67 +537,67 @@ int32_t sdbDeleteRow(void *handle, void *row) { pthread_mutex_lock(&pTable->mutex); - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { - pTable->id++; - sdbVersion++; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + if (oper == SDB_OPER_GLOBAL) { + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { + sdbError("table:%s, failed to delete record", pTable->name); pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - - pTable->numOfRows--; - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, sdbVersion, pTable->id, pTable->numOfRows); - break; - } - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - - code = 0; + } } + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->id = -(pTable->id); + memcpy(rowHead->data, row, rowSize); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, total_size); + pTable->size += total_size; + sdbFinishCommit(pTable); + + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + case SDB_KEYTYPE_AUTO: + sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + default: + sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->name, sdbVersion, pTable->id, pTable->numOfRows); + break; + } + + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); + + pTable->numOfRows--; + pTable->id++; + sdbVersion++; + pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); - // callback function of the delete - if (code == 0 && pTable->appTool) (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); + (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); - return code; + return 0; } // row here can be the object or the string info (encoded string) -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated) { +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; - int32_t code = -1; - int32_t total_size = 0; - int32_t real_size = 0; - /* char action = SDB_TYPE_UPDATE; */ + int32_t total_size = 0; + int32_t real_size = 0; if (pTable == NULL || row == NULL) return -1; pMeta = sdbGetRowMeta(handle, row); @@ -651,8 +630,15 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated } memset(rowHead, 0, total_size); - if (!isUpdated) { - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer + pthread_mutex_lock(&pTable->mutex); + + if (oper == SDB_OPER_GLOBAL) { + if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { + sdbError("table:%s, failed to update record", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } } if (pMetaRow != row) { @@ -663,57 +649,51 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, char isUpdated } real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - ; - pthread_mutex_lock(&pTable->mutex); - - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { - pTable->id++; - sdbVersion++; - - // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - /* write(pTable->fd, &action, sizeof(action)); */ - /* pTable->size += sizeof(action); */ - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } - - code = 0; + // write to the new position + rowHead->delimiter = SDB_DELIMITER; + rowHead->id = pTable->id; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; } + + twrite(pTable->fd, rowHead, real_size); + + pMeta->id = pTable->id; + pMeta->offset = pTable->size; + pMeta->rowSize = rowHead->rowSize; + pTable->size += real_size; + + sdbFinishCommit(pTable); + + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + case SDB_KEYTYPE_AUTO: + sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); + break; + default: + sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, + pTable->id, pTable->numOfRows); + break; + } + + pTable->id++; + sdbVersion++; pthread_mutex_unlock(&pTable->mutex); + (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer + tfree(rowHead); - return code; + return 0; } void sdbCloseTable(void *handle) { diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index c50ecc01c7..501faaab97 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -132,7 +132,7 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg) { } static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { - if (!sdbInServerState()) { + if (!mgmtInServerStatus()) { mgmtProcessMsgWhileNotReady(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index 075633cd4d..9120854e47 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -181,7 +181,7 @@ int32_t mgmtInitSuperTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("super table:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsSuperTableSdb, pTable); + sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_DISK); pNode = pLastNode; continue; } @@ -233,7 +233,7 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { tschema[col].bytes = htons(tschema[col].bytes); } - if (sdbInsertRow(tsSuperTableSdb, pStable, 0) < 0) { + if (sdbInsertRow(tsSuperTableSdb, pStable, SDB_OPER_GLOBAL) < 0) { mError("stable:%s, update sdb error", pStable->tableId); return TSDB_CODE_SDB_ERROR; } @@ -319,7 +319,7 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); return TSDB_CODE_SUCCESS; @@ -352,7 +352,7 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -384,7 +384,7 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); - int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, rowSize, 1); + int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); if (ret < 0) { @@ -446,7 +446,7 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -479,7 +479,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { pStable->schema = realloc(pStable->schema, schemaSize); pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, 0, 1); + sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -618,7 +618,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsSuperTableSdb, pTable); + sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_GLOBAL); pNode = pLastNode; numOfTables ++; continue; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 34c1a69cb1..a907cd85f5 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -757,9 +757,9 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - sdbDeleteRow(tsChildTableSdb, pTable); + sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); } else if (pTable->type == TSDB_NORMAL_TABLE){ - sdbDeleteRow(tsNormalTableSdb, pTable); + sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -813,14 +813,14 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - if (sdbDeleteRow(tsChildTableSdb, pTable) < 0) { + if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { mError("table:%s, update ctables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); return; } } else if (pTable->type == TSDB_NORMAL_TABLE){ - if (sdbDeleteRow(tsNormalTableSdb, pTable) < 0) { + if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { mError("table:%s, update ntables sdb error", pTable->tableId); mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); free(queueMsg); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 33caa75988..174976f4c0 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -103,7 +103,7 @@ SUserObj *mgmtGetUser(char *name) { } static int32_t mgmtUpdateUser(SUserObj *pUser) { - return sdbUpdateRow(tsUserSdb, pUser, 0, 1); + return sdbUpdateRow(tsUserSdb, pUser, tsUserUpdateSize, SDB_OPER_GLOBAL); } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { @@ -140,7 +140,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { } code = TSDB_CODE_SUCCESS; - if (sdbInsertRow(tsUserSdb, pUser, 0) < 0) { + if (sdbInsertRow(tsUserSdb, pUser, SDB_OPER_GLOBAL) < 0) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; } @@ -161,7 +161,7 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return TSDB_CODE_NO_RIGHTS; } - sdbDeleteRow(tsUserSdb, pUser); + sdbDeleteRow(tsUserSdb, pUser, SDB_OPER_GLOBAL); return 0; } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 62a2abd17d..d4df83c25c 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -112,7 +112,7 @@ int32_t mgmtInitVgroups() { if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) { pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp); - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); } // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); @@ -161,7 +161,7 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) { mgmtAddVgroupIntoDb(pDb, pVgroup); // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); - sdbInsertRow(tsVgroupSdb, pVgroup, 0); + sdbInsertRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { @@ -179,7 +179,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); - sdbDeleteRow(tsVgroupSdb, pVgroup); + sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); } } @@ -474,7 +474,7 @@ static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t } void mgmtUpdateVgroup(SVgObj *pVgroup) { - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 0); + sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_LOCAL); } void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { @@ -607,7 +607,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); mgmtAddToShellQueue(newMsg); } else { - sdbDeleteRow(tsVgroupSdb, pVgroup); + sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); } @@ -661,7 +661,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; - sdbDeleteRow(tsVgroupSdb, pVgroup); + sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); newMsg->msgType = queueMsg->msgType; @@ -691,7 +691,7 @@ void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); vnodeGid->publicIp = pDnode->publicIp; vnodeGid->privateIp = pDnode->privateIp; - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, 1); + sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); } } } From eb6764be24d27d7a3a93f6268cdfcd137c4bb4e6 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 20 Mar 2020 18:13:03 +0800 Subject: [PATCH 04/85] [TD-15] refactor sdb --- src/mnode/inc/mgmtSdb.h | 45 ++++---- src/mnode/src/mgmtChildTable.c | 110 +++++++----------- src/mnode/src/mgmtDb.c | 147 +++++++++++------------- src/mnode/src/mgmtNormalTable.c | 143 ++++++++++-------------- src/mnode/src/mgmtSdb.c | 114 ++++++++++--------- src/mnode/src/mgmtSuperTable.c | 128 +++++++++------------ src/mnode/src/mgmtUser.c | 151 +++++++++++-------------- src/mnode/src/mgmtVgroup.c | 191 ++++++++++++++------------------ 8 files changed, 446 insertions(+), 583 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 7381ec6669..bf5ccb2634 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -20,42 +20,49 @@ extern "C" { #endif -enum _keytype { +typedef enum { SDB_KEYTYPE_STRING, SDB_KEYTYPE_AUTO, SDB_KEYTYPE_MAX } ESdbKeyType; -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, - SDB_TYPE_DECODE, - SDB_TYPE_ENCODE, - SDB_TYPE_DESTROY, - SDB_MAX_ACTION_TYPES -} ESdbType; - typedef enum { SDB_OPER_GLOBAL, SDB_OPER_LOCAL, SDB_OPER_DISK -} ESdbOper; +} ESdbOperType; -uint64_t sdbGetVersion(); +enum _sdbaction { + SDB_TYPE_INSERT, + SDB_TYPE_DELETE, + SDB_TYPE_UPDATE, +} ESdbForwardType; -void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int32_t, int32_t *)); +typedef struct { + char *tableName; + int32_t hashSessions; + int32_t maxRowSize; + ESdbKeyType keyType; + int32_t (*insertFp)(void *pObj); + int32_t (*deleteFp)(void *pObj); + int32_t (*updateFp)(void *pObj); + int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); + void * (*decodeFp)(void *pData); + int32_t (*destroyFp)(void *pObj); +} SSdbTableDesc; + +void *sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); +int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper); +int32_t sdbDeleteRow(void *handle, void *key, ESdbOperType oper); +int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOperType oper); + void *sdbGetRow(void *handle, void *key); void *sdbFetchRow(void *handle, void *pNode, void **ppRow); int64_t sdbGetId(void *handle); int64_t sdbGetNumOfRows(void *handle); - -int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper); -int32_t sdbDeleteRow(void *handle, void *key, ESdbOper oper); -int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOper oper); +uint64_t sdbGetVersion(); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index be78983c91..5b27acf1c4 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -39,67 +39,43 @@ void *tsChildTableSdb; int32_t tsChildTableUpdateSize; -void *(*mgmtChildTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyChildTable(SChildTableObj *pTable) { - free(pTable); + tfree(pTable); } -static void mgmtChildTableActionInit() { - mgmtChildTableActionFp[SDB_TYPE_INSERT] = mgmtChildTableActionInsert; - mgmtChildTableActionFp[SDB_TYPE_DELETE] = mgmtChildTableActionDelete; - mgmtChildTableActionFp[SDB_TYPE_UPDATE] = mgmtChildTableActionUpdate; - mgmtChildTableActionFp[SDB_TYPE_ENCODE] = mgmtChildTableActionEncode; - mgmtChildTableActionFp[SDB_TYPE_DECODE] = mgmtChildTableActionDecode; - mgmtChildTableActionFp[SDB_TYPE_DESTROY] = mgmtChildTableActionDestroy; -} - -void *mgmtChildTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - memcpy(pTable, str, tsChildTableUpdateSize); - return NULL; -} - -void *mgmtChildTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *)row; +static int32_t mgmtChildTableActionDestroy(void *pObj) { + SChildTableObj *pTable = (SChildTableObj *)pObj; mgmtDestroyChildTable(pTable); - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionInsert(void *pObj) { + SChildTableObj *pTable = (SChildTableObj *) pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("ctable:%s, not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in db:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + return TSDB_CODE_INVALID_ACCT; } if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); - return NULL; + return TSDB_CODE_INVALID_SESSION_ID; } } @@ -114,30 +90,30 @@ void *mgmtChildTableActionInsert(void *row, char *str, int32_t size, int32_t *ss mgmtMoveVgroupToTail(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; +static int32_t mgmtChildTableActionDelete(void *pObj) { + SChildTableObj *pTable = (SChildTableObj *) pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); @@ -150,56 +126,54 @@ void *mgmtChildTableActionDelete(void *row, char *str, int32_t size, int32_t *ss mgmtMoveVgroupToHead(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtChildTableActionReset(row, str, size, NULL); +static int32_t mgmtChildTableActionUpdate(void *pObj) { + return TSDB_CODE_SUCCESS; } -void *mgmtChildTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SChildTableObj *pTable = (SChildTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtChildTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SChildTableObj *pTable = (SChildTableObj *) pObj; + assert(pObj != NULL && pData != NULL); - memcpy(str, pTable, tsChildTableUpdateSize); - *ssize = tsChildTableUpdateSize; - - return NULL; + memcpy(pData, pTable, tsChildTableUpdateSize); + return tsChildTableUpdateSize; } -void *mgmtChildTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static void *mgmtChildTableActionDecode(void *pData) { + assert(pData != NULL); SChildTableObj *pTable = (SChildTableObj *)calloc(sizeof(SChildTableObj), 1); if (pTable == NULL) return NULL; - if (size < tsChildTableUpdateSize) { - mgmtDestroyChildTable(pTable); - return NULL; - } - memcpy(pTable, str, tsChildTableUpdateSize); + memcpy(pTable, pData, tsChildTableUpdateSize); return (void *)pTable; } -void *mgmtChildTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtChildTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtChildTableActionFp[(uint8_t)action]))(row, str, size, ssize); - } - return NULL; -} - int32_t mgmtInitChildTables() { void *pNode = NULL; void *pLastNode = NULL; SChildTableObj *pTable = NULL; - mgmtChildTableActionInit(); SChildTableObj tObj; tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsChildTableSdb = sdbOpenTable(tsMaxTables, tsChildTableUpdateSize, - "ctables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtChildTableAction); + SSdbTableDesc tableDesc = { + .tableName = "ctables", + .hashSessions = tsMaxTables, + .maxRowSize = tsChildTableUpdateSize, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtChildTableActionInsert, + .deleteFp = mgmtChildTableActionDelete, + .updateFp = mgmtChildTableActionUpdate, + .encodeFp = mgmtChildTableActionEncode, + .decodeFp = mgmtChildTableActionDecode, + .destroyFp = mgmtChildTableActionDestroy, + }; + + tsChildTableSdb = sdbOpenTable(&tableDesc); if (tsChildTableSdb == NULL) { mError("failed to init child table data"); return -1; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index e889d200f5..2549dd4c64 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -48,29 +48,57 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg); -static void *(*mgmtDbActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - -static void mgmtDbActionInit() { - mgmtDbActionFp[SDB_TYPE_INSERT] = mgmtDbActionInsert; - mgmtDbActionFp[SDB_TYPE_DELETE] = mgmtDbActionDelete; - mgmtDbActionFp[SDB_TYPE_UPDATE] = mgmtDbActionUpdate; - mgmtDbActionFp[SDB_TYPE_ENCODE] = mgmtDbActionEncode; - mgmtDbActionFp[SDB_TYPE_DECODE] = mgmtDbActionDecode; - mgmtDbActionFp[SDB_TYPE_DESTROY] = mgmtDbActionDestroy; +static int32_t mgmtDbActionDestroy(void *pObj) { + tfree(pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtDbAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtDbActionFp[(uint8_t)action] != NULL) { - return (*(mgmtDbActionFp[(uint8_t)action]))(row, str, size, ssize); +static int32_t mgmtDbActionInsert(void *pObj) { + SDbObj *pDb = (SDbObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + + pDb->pHead = NULL; + pDb->pTail = NULL; + pDb->numOfVgroups = 0; + pDb->numOfTables = 0; + mgmtAddDbIntoAcct(pAcct, pDb); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDbActionDelete(void *pObj) { + SDbObj *pDb = (SDbObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + mgmtRemoveDbFromAcct(pAcct, pDb); + + mgmtDropAllNormalTables(pDb); + mgmtDropAllChildTables(pDb); + mgmtDropAllSuperTables(pDb); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDbActionUpdate(void *pObj) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDbActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SDbObj *pDb = (SDbObj *)pObj; + if (maxRowSize < tsDbUpdateSize) { + return -1; + } else { + memcpy(pData, pDb, tsDbUpdateSize); + return tsDbUpdateSize; } - return NULL; +} + +static void *mgmtDbActionDecode(void *pData) { + SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); + if (pDb == NULL) return NULL; + memset(pDb, 0, sizeof(SDbObj)); + memcpy(pDb, pData, tsDbUpdateSize); + + return (void *)pDb; } int32_t mgmtInitDbs() { @@ -78,12 +106,23 @@ int32_t mgmtInitDbs() { SDbObj * pDb = NULL; SAcctObj *pAcct = NULL; - mgmtDbActionInit(); - SDbObj tObj; tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; - tsDbSdb = sdbOpenTable(TSDB_MAX_DBS, tsDbUpdateSize, "dbs", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtDbAction); + SSdbTableDesc tableDesc = { + .tableName = "dbs", + .hashSessions = TSDB_MAX_DBS, + .maxRowSize = tsDbUpdateSize, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtDbActionInsert, + .deleteFp = mgmtDbActionDelete, + .updateFp = mgmtDbActionUpdate, + .encodeFp = mgmtDbActionEncode, + .decodeFp = mgmtDbActionDecode, + .destroyFp = mgmtDbActionDestroy, + }; + + tsDbSdb = sdbOpenTable(&tableDesc); if (tsDbSdb == NULL) { mError("failed to init db data"); return -1; @@ -683,68 +722,6 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * return numOfRows; } -void *mgmtDbActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->numOfVgroups = 0; - pDb->numOfTables = 0; - mgmtAddDbIntoAcct(pAcct, pDb); - - return NULL; -} - -void *mgmtDbActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtRemoveDbFromAcct(pAcct, pDb); - - mgmtDropAllNormalTables(pDb); - mgmtDropAllChildTables(pDb); - mgmtDropAllSuperTables(pDb); - - return NULL; -} - -void *mgmtDbActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtDbActionReset(row, str, size, ssize); -} - -void *mgmtDbActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - if (size < tsDbUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pDb, tsDbUpdateSize); - *ssize = tsDbUpdateSize; - } - - return NULL; -} -void *mgmtDbActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); - if (pDb == NULL) return NULL; - memset(pDb, 0, sizeof(SDbObj)); - - memcpy(pDb, str, tsDbUpdateSize); - - return (void *)pDb; -} - -void *mgmtDbActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SDbObj *pDb = (SDbObj *) row; - memcpy(pDb, str, tsDbUpdateSize); - - return NULL; -} - -void *mgmtDbActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - return NULL; -} - void mgmtAddSuperTableIntoDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfSuperTables, 1); } diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index df9fe0f439..58578faf9a 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -35,75 +35,45 @@ void *tsNormalTableSdb; int32_t tsNormalTableUpdateSize; -void *(*mgmtNormalTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); - -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { - free(pTable->schema); - free(pTable->sql); - free(pTable); + tfree(pTable->schema); + tfree(pTable->sql); + tfree(pTable); } -static void mgmtNormalTableActionInit() { - mgmtNormalTableActionFp[SDB_TYPE_INSERT] = mgmtNormalTableActionInsert; - mgmtNormalTableActionFp[SDB_TYPE_DELETE] = mgmtNormalTableActionDelete; - mgmtNormalTableActionFp[SDB_TYPE_UPDATE] = mgmtNormalTableActionUpdate; - mgmtNormalTableActionFp[SDB_TYPE_ENCODE] = mgmtNormalTableActionEncode; - mgmtNormalTableActionFp[SDB_TYPE_DECODE] = mgmtNormalTableActionDecode; - mgmtNormalTableActionFp[SDB_TYPE_DESTROY] = mgmtNormalTableActionDestroy; -} - -void *mgmtNormalTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - memcpy(pTable, str, tsNormalTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; - pTable->schema = realloc(pTable->schema, schemaSize); - pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - - return NULL; -} - -void *mgmtNormalTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *)row; +static int32_t mgmtNormalTableActionDestroy(void *pObj) { + SNormalTableObj *pTable = (SNormalTableObj *)pObj; mgmtDestroyNormalTable(pTable); - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionInsert(void *pObj) { + SNormalTableObj *pTable = (SNormalTableObj *) pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId); - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("account not exists"); - return NULL; + return TSDB_CODE_INVALID_ACCT; } if (!mgmtIsMaster()) { int32_t sid = taosAllocateId(pVgroup->idPool); if (sid != pTable->sid) { mError("sid:%d is not matched from the master:%d", sid, pTable->sid); - return NULL; + return TSDB_CODE_INVALID_SESSION_ID; } } @@ -115,30 +85,30 @@ void *mgmtNormalTableActionInsert(void *row, char *str, int32_t size, int32_t *s mgmtMoveVgroupToTail(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; +static int32_t mgmtNormalTableActionDelete(void *pObj) { + SNormalTableObj *pTable = (SNormalTableObj *) pObj; if (pTable->vgId == 0) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return NULL; + return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); - return NULL; + return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { mError("account not exists"); - return NULL; + return TSDB_CODE_INVALID_ACCT; } mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); @@ -149,45 +119,46 @@ void *mgmtNormalTableActionDelete(void *row, char *str, int32_t size, int32_t *s mgmtMoveVgroupToHead(pDb, pVgroup); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtNormalTableActionReset(row, str, size, NULL); +static int32_t mgmtNormalTableActionUpdate(void *pObj) { + // SNormalTableObj *pTable = (SNormalTableObj *) pObj; + // memcpy(pTable, str, tsNormalTableUpdateSize); + + // int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; + // pTable->schema = realloc(pTable->schema, schemaSize); + // pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); + // memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); + + return TSDB_CODE_SUCCESS; } -void *mgmtNormalTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SNormalTableObj *pTable = (SNormalTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtNormalTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SNormalTableObj *pTable = (SNormalTableObj *) pObj; + assert(pObj != NULL && pData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (size < tsNormalTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { + return -1; } - memcpy(str, pTable, tsNormalTableUpdateSize); - memcpy(str + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(str + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - *ssize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; - - return NULL; + memcpy(pData, pTable, tsNormalTableUpdateSize); + memcpy(pData + tsNormalTableUpdateSize, pTable->schema, schemaSize); + memcpy(pData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); + + return tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; } -void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static void *mgmtNormalTableActionDecode(void *pData) { + assert(pData != NULL); SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); if (pTable == NULL) { return NULL; } memset(pTable, 0, sizeof(SNormalTableObj)); - - if (size < tsNormalTableUpdateSize) { - mgmtDestroyNormalTable(pTable); - return NULL; - } - memcpy(pTable, str, tsNormalTableUpdateSize); + memcpy(pTable, pData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); @@ -196,35 +167,39 @@ void *mgmtNormalTableActionDecode(void *row, char *str, int32_t size, int32_t *s return NULL; } - memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); + memcpy(pTable->schema, pData + tsNormalTableUpdateSize, schemaSize); pTable->sql = (char *)malloc(pTable->sqlLen); if (pTable->sql == NULL) { mgmtDestroyNormalTable(pTable); return NULL; } - memcpy(pTable->sql, str + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + memcpy(pTable->sql, pData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); return (void *)pTable; } -void *mgmtNormalTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtNormalTableActionFp[(uint8_t)action] != NULL) { - return (*(mgmtNormalTableActionFp[(uint8_t)action]))(row, str, size, ssize); - } - return NULL; -} - int32_t mgmtInitNormalTables() { void *pNode = NULL; void *pLastNode = NULL; SNormalTableObj *pTable = NULL; - mgmtNormalTableActionInit(); SNormalTableObj tObj; tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsNormalTableSdb = sdbOpenTable(TSDB_MAX_NORMAL_TABLES, sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "ntables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtNormalTableAction); + SSdbTableDesc tableDesc = { + .tableName = "ntables", + .hashSessions = TSDB_MAX_NORMAL_TABLES, + .maxRowSize = sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtNormalTableActionInsert, + .deleteFp = mgmtNormalTableActionDelete, + .updateFp = mgmtNormalTableActionUpdate, + .encodeFp = mgmtNormalTableActionEncode, + .decodeFp = mgmtNormalTableActionDecode, + .destroyFp = mgmtNormalTableActionDestroy, + }; + + tsNormalTableSdb = sdbOpenTable(&tableDesc); if (tsNormalTableSdb == NULL) { mError("failed to init ntables data"); return -1; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 33bfe4622b..8703c63541 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" -#include "tutil.h" #include "tchecksum.h" +#include "tglobalcfg.h" #include "tlog.h" #include "trpc.h" #include "tutil.h" @@ -33,32 +33,37 @@ typedef struct { uint64_t swVersion; int16_t sdbFileVersion; - char reserved[6]; + char reserved[2]; TSCKSUM checkSum; } SSdbHeader; typedef struct _SSdbTable { - SSdbHeader header; - int maxRows; - int dbId; - int32_t maxRowSize; - char name[TSDB_DB_NAME_LEN]; - char fn[128]; - int keyType; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; - int fd; - void *(*appTool)(char, void *, char *, int, int *); + SSdbHeader header; + char name[TSDB_DB_NAME_LEN]; + char fn[TSDB_FILENAME_LEN]; + ESdbKeyType keyType; + int32_t dbId; + int32_t hashSessions; + int32_t maxRowSize; + uint32_t autoIndex; + int64_t numOfRows; + int64_t id; + int64_t size; + void * iHandle; + int32_t fd; + int32_t (*insertFp)(void *pObj); + int32_t (*deleteFp)(void *pObj); + int32_t (*updateFp)(void *pObj); + void * (*decodeFp)(void *pData); // return pObj + int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); // return size of pData + int32_t (*destroyFp)(void *pObj); pthread_mutex_t mutex; } SSdbTable; typedef struct { int64_t id; int64_t offset; - int rowSize; + int32_t rowSize; void * row; } SRowMeta; @@ -71,9 +76,9 @@ typedef struct { typedef struct { uint8_t dbId; - char type; + int8_t type; + int16_t dataLen; uint64_t version; - short dataLen; char data[]; } SForwardMsg; @@ -283,7 +288,7 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { // TODO: Get rid of the rowMeta.offset and rowSize rowMeta.offset = pTable->size; rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); + rowMeta.row = (*pTable->decodeFp)(rowHead->data); (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); if (pTable->keyType == SDB_KEYTYPE_AUTO) { pTable->autoIndex++; @@ -299,7 +304,7 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { if (rowHead->id < 0) { // Delete the object (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); + (*pTable->destroyFp)(pMetaRow); pTable->numOfRows--; numOfDels++; } else { // Reset the object TODO: is it possible to merge reset and @@ -322,7 +327,7 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { } sdbVersion += (pTable->id - oldId); - if (numOfDels > pTable->maxRows / 4) sdbSaveSnapShot(pTable); + if (numOfDels > pTable->hashSessions / 4) sdbSaveSnapShot(pTable); tfree(rowHead); return 0; @@ -332,20 +337,25 @@ sdb_exit1: return -1; } -void *sdbOpenTable(int32_t maxRows, int32_t maxRowSize, char *name, uint8_t keyType, char *directory, - void *(*appTool)(char, void *, char *, int32_t, int32_t *)) { - SSdbTable *pTable = (SSdbTable *)malloc(sizeof(SSdbTable)); +void *sdbOpenTable(SSdbTableDesc *pDesc) { + SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); if (pTable == NULL) return NULL; - memset(pTable, 0, sizeof(SSdbTable)); - strcpy(pTable->name, name); - pTable->keyType = keyType; - pTable->maxRows = maxRows; - pTable->maxRowSize = maxRowSize; - pTable->appTool = appTool; - sprintf(pTable->fn, "%s/%s.db", directory, pTable->name); + pTable->keyType = pDesc->keyType; + pTable->hashSessions = pDesc->hashSessions; + pTable->maxRowSize = pDesc->maxRowSize; + pTable->insertFp = pDesc->insertFp; + pTable->deleteFp = pDesc->deleteFp; + pTable->updateFp = pDesc->updateFp; + pTable->encodeFp = pDesc->encodeFp; + pTable->decodeFp = pDesc->decodeFp; + pTable->destroyFp = pDesc->destroyFp; + strcpy(pTable->name, pDesc->tableName); + sprintf(pTable->fn, "%s/%s.db", tsMnodeDir, pTable->name); - if (sdbInitIndexFp[keyType] != NULL) pTable->iHandle = (*sdbInitIndexFp[keyType])(maxRows, sizeof(SRowMeta)); + if (sdbInitIndexFp[pTable->keyType] != NULL) { + pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); + } pthread_mutex_init(&pTable->mutex, NULL); @@ -386,7 +396,7 @@ void *sdbGetRow(void *handle, void *key) { } // row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) -int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { +int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; void * pObj = NULL; @@ -424,7 +434,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { if (oper == SDB_OPER_GLOBAL) { pObj = row; } else { - pObj = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, row, 0, NULL); + pObj = (*pTable->decodeFp)(row); } pthread_mutex_lock(&pTable->mutex); @@ -439,7 +449,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { } if (oper == SDB_OPER_GLOBAL || oper == SDB_OPER_LOCAL) { - (*(pTable->appTool))(SDB_TYPE_ENCODE, pObj, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + rowHead->rowSize = (*pTable->encodeFp)(pObj, rowHead->data, pTable->maxRowSize); assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); @@ -489,7 +499,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { pthread_mutex_unlock(&pTable->mutex); - (*pTable->appTool)(SDB_TYPE_INSERT, pObj, NULL, 0, NULL); + (*pTable->insertFp)(pObj); tfree(rowHead); @@ -497,7 +507,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOper oper) { } // row here can be object or null-terminated string -int32_t sdbDeleteRow(void *handle, void *row, ESdbOper oper) { +int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; void * pMetaRow = NULL; @@ -587,13 +597,13 @@ int32_t sdbDeleteRow(void *handle, void *row, ESdbOper oper) { tfree(rowHead); - (*pTable->appTool)(SDB_TYPE_DELETE, pMetaRow, NULL, 0, NULL); + (*pTable->deleteFp)(pMetaRow); return 0; } // row here can be the object or the string info (encoded string) -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) { +int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta = NULL; int32_t total_size = 0; @@ -645,7 +655,7 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) memcpy(rowHead->data, row, updateSize); rowHead->rowSize = updateSize; } else { - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMetaRow, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + rowHead->rowSize = (*pTable->encodeFp)(pMetaRow, rowHead->data, pTable->maxRowSize); } real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); @@ -689,7 +699,7 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOper oper) pthread_mutex_unlock(&pTable->mutex); - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, row, updateSize, NULL); // update in upper layer + (*pTable->updateFp)(pMetaRow); // update in upper layer tfree(rowHead); @@ -706,7 +716,7 @@ void sdbCloseTable(void *handle) { while (1) { pNode = sdbFetchRow(handle, pNode, &row); if (row == NULL) break; - (*(pTable->appTool))(SDB_TYPE_DESTROY, row, NULL, 0, NULL); + (*pTable->destroyFp)(row); } if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); @@ -724,13 +734,13 @@ void sdbCloseTable(void *handle) { void sdbResetTable(SSdbTable *pTable) { /* SRowHead rowHead; */ SRowMeta rowMeta; - int32_t bytes; - int32_t total_size = 0; - int32_t real_size = 0; + int32_t bytes; + int32_t total_size = 0; + int32_t real_size = 0; SRowHead *rowHead = NULL; void * pMetaRow = NULL; int64_t oldId = pTable->id; - int32_t oldNumOfRows = pTable->numOfRows; + int32_t oldNumOfRows = pTable->numOfRows; if (sdbOpenSdbFile(pTable) < 0) return; pTable->numOfRows = oldNumOfRows; @@ -792,19 +802,19 @@ void sdbResetTable(SSdbTable *pTable) { // TODO:Get rid of the rowMeta.offset and rowSize rowMeta.offset = pTable->size; rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*(pTable->appTool))(SDB_TYPE_DECODE, NULL, rowHead->data, rowHead->rowSize, NULL); + rowMeta.row = (*pTable->decodeFp)(rowHead->data); (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); pTable->numOfRows++; - (*pTable->appTool)(SDB_TYPE_INSERT, rowMeta.row, NULL, 0, NULL); + (*pTable->insertFp)(rowMeta.row); } } else { // already exists if (rowHead->id < 0) { // Delete the object (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*(pTable->appTool))(SDB_TYPE_DESTROY, pMetaRow, NULL, 0, NULL); + (*pTable->destroyFp)(pMetaRow); pTable->numOfRows--; } else { // update the object - (*(pTable->appTool))(SDB_TYPE_UPDATE, pMetaRow, rowHead->data, rowHead->rowSize, NULL); + (*pTable->updateFp)(pMetaRow); } } } @@ -866,7 +876,7 @@ void sdbSaveSnapShot(void *handle) { rowHead->delimiter = SDB_DELIMITER; rowHead->id = pMeta->id; - (*(pTable->appTool))(SDB_TYPE_ENCODE, pMeta->row, rowHead->data, pTable->maxRowSize, &(rowHead->rowSize)); + rowHead->rowSize = (*pTable->encodeFp)(pMeta->row, rowHead->data, pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { sdbError("failed to get checksum while save sdb %s snapshot", pTable->name); diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index 9120854e47..de64f9f54e 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -39,105 +39,73 @@ static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; -static void *(*mgmtSuperTableActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static void mgmtDestroySuperTable(SSuperTableObj *pTable) { - free(pTable->schema); - free(pTable); + tfree(pTable->schema); + tfree(pTable); } -static void mgmtSuperTableActionInit() { - SSuperTableObj tObj; - tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtSuperTableActionFp[SDB_TYPE_INSERT] = mgmtSuperTableActionInsert; - mgmtSuperTableActionFp[SDB_TYPE_DELETE] = mgmtSuperTableActionDelete; - mgmtSuperTableActionFp[SDB_TYPE_UPDATE] = mgmtSuperTableActionUpdate; - mgmtSuperTableActionFp[SDB_TYPE_ENCODE] = mgmtSuperTableActionEncode; - mgmtSuperTableActionFp[SDB_TYPE_DECODE] = mgmtSuperTableActionDecode; - mgmtSuperTableActionFp[SDB_TYPE_DESTROY] = mgmtSuperTableActionDestroy; -} - -void *mgmtSuperTableActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - memcpy(pTable, str, tsSuperTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = realloc(pTable->schema, schemaSize); - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); - - return NULL; -} - -void *mgmtSuperTableActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; +static int32_t mgmtSuperTableActionDestroy(void *pObj) { + SSuperTableObj *pTable = (SSuperTableObj *) pObj; mgmtDestroySuperTable(pTable); - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; +static int32_t mgmtSuperTableActionInsert(void *pObj) { + STableInfo *pTable = (STableInfo *) pObj; SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb) { mgmtAddSuperTableIntoDb(pDb); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - STableInfo *pTable = (STableInfo *) row; +static int32_t mgmtSuperTableActionDelete(void *pObj) { + STableInfo *pTable = (STableInfo *) pObj; SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb) { mgmtRemoveSuperTableFromDb(pDb); } - return NULL; + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtSuperTableActionReset(row, str, size, NULL); +static int32_t mgmtSuperTableActionUpdate(void *pObj) { + SSuperTableObj *pTable = (SSuperTableObj *) pObj; + memcpy(pTable, pObj, tsSuperTableUpdateSize); + + int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); + pTable->schema = realloc(pTable->schema, schemaSize); + memcpy(pTable->schema, pObj + tsSuperTableUpdateSize, schemaSize); + + return TSDB_CODE_SUCCESS; } -void *mgmtSuperTableActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SSuperTableObj *pTable = (SSuperTableObj *) row; - assert(row != NULL && str != NULL); +static int32_t mgmtSuperTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SSuperTableObj *pTable = (SSuperTableObj *) pObj; + assert(pObj != NULL && pData != NULL); int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - if (size < tsSuperTableUpdateSize + schemaSize + 1) { - *ssize = -1; - return NULL; + if (maxRowSize < tsSuperTableUpdateSize + schemaSize + 1) { + return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(str, pTable, tsSuperTableUpdateSize); - memcpy(str + tsSuperTableUpdateSize, pTable->schema, schemaSize); - *ssize = tsSuperTableUpdateSize + schemaSize; - - return NULL; + memcpy(pData, pTable, tsSuperTableUpdateSize); + memcpy(pData + tsSuperTableUpdateSize, pTable->schema, schemaSize); + return tsSuperTableUpdateSize + schemaSize; } -void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - assert(str != NULL); +static void *mgmtSuperTableActionDecode(void *pData) { + assert(pData != NULL); SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj)); if (pTable == NULL) { return NULL; } memset(pTable, 0, sizeof(SSuperTableObj)); - - if (size < tsSuperTableUpdateSize) { - mgmtDestroySuperTable(pTable); - return NULL; - } - memcpy(pTable, str, tsSuperTableUpdateSize); + memcpy(pTable, pData, tsSuperTableUpdateSize); int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); pTable->schema = malloc(schemaSize); @@ -146,26 +114,32 @@ void *mgmtSuperTableActionDecode(void *row, char *str, int32_t size, int32_t *ss return NULL; } - memcpy(pTable->schema, str + tsSuperTableUpdateSize, schemaSize); + memcpy(pTable->schema, pData + tsSuperTableUpdateSize, schemaSize); return (void *) pTable; } -void *mgmtSuperTableAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtSuperTableActionFp[(uint8_t) action] != NULL) { - return (*(mgmtSuperTableActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; -} - int32_t mgmtInitSuperTables() { void *pNode = NULL; void *pLastNode = NULL; SSuperTableObj *pTable = NULL; - mgmtSuperTableActionInit(); + SSuperTableObj tObj; + tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsSuperTableSdb = sdbOpenTable(TSDB_MAX_SUPER_TABLES, tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, - "stables", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtSuperTableAction); + SSdbTableDesc tableDesc = { + .tableName = "stables", + .hashSessions = TSDB_MAX_SUPER_TABLES, + .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtSuperTableActionInsert, + .deleteFp = mgmtSuperTableActionDelete, + .updateFp = mgmtSuperTableActionUpdate, + .encodeFp = mgmtSuperTableActionEncode, + .decodeFp = mgmtSuperTableActionDecode, + .destroyFp = mgmtSuperTableActionDestroy, + }; + + tsSuperTableSdb = sdbOpenTable(&tableDesc); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); return -1; @@ -365,8 +339,8 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN return TSDB_CODE_INVALID_MSG_TYPE; } - int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + int32_t rowSize = 0; + uint32_t len = strlen(newTagName); if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_APP_ERROR; @@ -382,7 +356,7 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN if (msg == NULL) return TSDB_CODE_APP_ERROR; memset(msg, 0, size); - mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); + // mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 174976f4c0..2ca37a4572 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -18,6 +18,7 @@ #include "trpc.h" #include "tschemautil.h" #include "ttime.h" +#include "tutil.h" #include "mgmtAcct.h" #include "mgmtGrant.h" #include "mgmtMnode.h" @@ -38,16 +39,53 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); -static void *(*mgmtUserActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize); -static void mgmtUserActionInit(); +static int32_t mgmtUserActionDestroy(void *pObj) { + tfree(pObj); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionInsert(void *pObj) { + SUserObj *pUser = (SUserObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); + + pUser->pAcct = pAcct; + mgmtAddUserIntoAcct(pAcct, pUser); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionDelete(void *pObj) { + SUserObj *pUser = (SUserObj *) pObj; + SAcctObj *pAcct = mgmtGetAcct(pUser->acct); + + mgmtRemoveUserFromAcct(pAcct, pUser); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionUpdate(void *pObj) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtUserActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SUserObj *pUser = (SUserObj *) pObj; + + if (maxRowSize < tsUserUpdateSize) { + return -1; + } else { + memcpy(pData, pUser, tsUserUpdateSize); + return tsUserUpdateSize; + } +} + +static void *mgmtUserActionDecode(void *pData) { + SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); + if (pUser == NULL) return NULL; + memset(pUser, 0, sizeof(SUserObj)); + memcpy(pUser, pData, tsUserUpdateSize); + + return pUser; +} int32_t mgmtInitUsers() { void *pNode = NULL; @@ -55,12 +93,23 @@ int32_t mgmtInitUsers() { SAcctObj *pAcct = NULL; int32_t numOfUsers = 0; - mgmtUserActionInit(); - SUserObj tObj; tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsUserSdb = sdbOpenTable(TSDB_MAX_USERS, tsUserUpdateSize, "users", SDB_KEYTYPE_STRING, tsMnodeDir, mgmtUserAction); + SSdbTableDesc tableDesc = { + .tableName = "users", + .hashSessions = TSDB_MAX_USERS, + .maxRowSize = tsUserUpdateSize, + .keyType = SDB_KEYTYPE_STRING, + .insertFp = mgmtUserActionInsert, + .deleteFp = mgmtUserActionDelete, + .updateFp = mgmtUserActionUpdate, + .encodeFp = mgmtUserActionEncode, + .decodeFp = mgmtUserActionDecode, + .destroyFp = mgmtUserActionDestroy, + }; + + tsUserSdb = sdbOpenTable(&tableDesc); if (tsUserSdb == NULL) { mError("failed to init user data"); return -1; @@ -246,82 +295,6 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void return numOfRows; } -static void mgmtUserActionInit() { - mgmtUserActionFp[SDB_TYPE_INSERT] = mgmtUserActionInsert; - mgmtUserActionFp[SDB_TYPE_DELETE] = mgmtUserActionDelete; - mgmtUserActionFp[SDB_TYPE_UPDATE] = mgmtUserActionUpdate; - mgmtUserActionFp[SDB_TYPE_ENCODE] = mgmtUserActionEncode; - mgmtUserActionFp[SDB_TYPE_DECODE] = mgmtUserActionDecode; - mgmtUserActionFp[SDB_TYPE_DESTROY] = mgmtUserActionDestroy; -} - -static void *mgmtUserAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtUserActionFp[(uint8_t) action] != NULL) { - return (*(mgmtUserActionFp[(uint8_t) action]))(row, str, size, ssize); - } - return NULL; -} - -static void *mgmtUserActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - pUser->pAcct = pAcct; - mgmtAddUserIntoAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - mgmtRemoveUserFromAcct(pAcct, pUser); - - return NULL; -} - -static void *mgmtUserActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - return mgmtUserActionReset(row, str, size, ssize); -} - -static void *mgmtUserActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) row; - - if (size < tsUserUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pUser, tsUserUpdateSize); - *ssize = tsUserUpdateSize; - } - - return NULL; -} - -static void *mgmtUserActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); - if (pUser == NULL) return NULL; - memset(pUser, 0, sizeof(SUserObj)); - - memcpy(pUser, str, tsUserUpdateSize); - - return (void *)pUser; -} - -static void *mgmtUserActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SUserObj *pUser = (SUserObj *)row; - - memcpy(pUser, str, tsUserUpdateSize); - - return NULL; -} - -static void *mgmtUserActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - tfree(row); - - return NULL; -} - SUserObj *mgmtGetUserFromConn(void *pConn) { SRpcConnInfo connInfo; if (rpcGetConnInfo(pConn, &connInfo) == 0) { diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index d4df83c25c..12ac6fef57 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -33,15 +33,6 @@ static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; -static void *(*mgmtVgroupActionFp[SDB_MAX_ACTION_TYPES])(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize); -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize); - static int32_t mgmtGetVgroupMeta(STableMeta *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); @@ -50,32 +41,98 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); -static void mgmtVgroupActionInit() { - SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - - mgmtVgroupActionFp[SDB_TYPE_INSERT] = mgmtVgroupActionInsert; - mgmtVgroupActionFp[SDB_TYPE_DELETE] = mgmtVgroupActionDelete; - mgmtVgroupActionFp[SDB_TYPE_UPDATE] = mgmtVgroupActionUpdate; - mgmtVgroupActionFp[SDB_TYPE_ENCODE] = mgmtVgroupActionEncode; - mgmtVgroupActionFp[SDB_TYPE_DECODE] = mgmtVgroupActionDecode; - mgmtVgroupActionFp[SDB_TYPE_DESTROY] = mgmtVgroupActionDestroy; +static int32_t mgmtVgroupActionDestroy(void *pObj) { + SVgObj *pVgroup = (SVgObj *) pObj; + if (pVgroup->idPool) { + taosIdPoolCleanUp(pVgroup->idPool); + pVgroup->idPool = NULL; + } + if (pVgroup->tableList) tfree(pVgroup->tableList); + tfree(pObj); + return TSDB_CODE_SUCCESS; } -static void *mgmtVgroupAction(char action, void *row, char *str, int32_t size, int32_t *ssize) { - if (mgmtVgroupActionFp[(uint8_t) action] != NULL) { - return (*(mgmtVgroupActionFp[(uint8_t) action]))(row, str, size, ssize); +static int32_t mgmtVgroupActionInsert(void *pObj) { + SVgObj *pVgroup = pObj; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + pVgroup->vnodeGid[i].vnode = pVgroup->vgId; } - return NULL; + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionDelete(void *pObj) { + SVgObj *pVgroup = pObj; + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + + if (pDb != NULL) { + mgmtRemoveVgroupFromDb(pDb, pVgroup); + } + + // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); + tfree(pVgroup->tableList); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionUpdate(void *pObj) { + SVgObj *pVgroup = (SVgObj *) pObj; + int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); + + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb != NULL) { + if (pDb->cfg.maxSessions != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); + } + } + + mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtVgroupActionEncode(void *pObj, void *pData, int32_t maxRowSize) { + SVgObj *pVgroup = (SVgObj *) pObj; + if (maxRowSize < tsVgUpdateSize) { + return -1; + } else { + memcpy(pData, pVgroup, tsVgUpdateSize); + return tsVgUpdateSize; + } +} + +static void *mgmtVgroupActionDecode(void *pObj) { + SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); + if (pVgroup == NULL) return NULL; + memset(pVgroup, 0, sizeof(SVgObj)); + memcpy(pVgroup, pObj, tsVgUpdateSize); + + return pVgroup; } int32_t mgmtInitVgroups() { void *pNode = NULL; SVgObj *pVgroup = NULL; - mgmtVgroupActionInit(); + SVgObj tObj; + tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; - tsVgroupSdb = sdbOpenTable(TSDB_MAX_VGROUPS, tsVgUpdateSize, "vgroups", SDB_KEYTYPE_AUTO, tsMnodeDir, mgmtVgroupAction); + SSdbTableDesc tableDesc = { + .tableName = "vgroups", + .hashSessions = TSDB_MAX_VGROUPS, + .maxRowSize = tsVgUpdateSize, + .keyType = SDB_KEYTYPE_AUTO, + .insertFp = mgmtVgroupActionInsert, + .deleteFp = mgmtVgroupActionDelete, + .updateFp = mgmtVgroupActionUpdate, + .encodeFp = mgmtVgroupActionEncode, + .decodeFp = mgmtVgroupActionDecode, + .destroyFp = mgmtVgroupActionDestroy, + }; + + tsVgroupSdb = sdbOpenTable(&tableDesc); if (tsVgroupSdb == NULL) { mError("failed to init vgroups data"); return -1; @@ -389,90 +446,6 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -static void *mgmtVgroupActionInsert(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - pVgroup->vnodeGid[i].vnode = pVgroup->vgId; - } - - return NULL; -} - -static void *mgmtVgroupActionDelete(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = row; - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - - if (pDb != NULL) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - } - - // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); - tfree(pVgroup->tableList); - - return NULL; -} - -static void *mgmtVgroupActionUpdate(void *row, char *str, int32_t size, int32_t *ssize) { - mgmtVgroupActionReset(row, str, size, ssize); - - SVgObj *pVgroup = (SVgObj *) row; - int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb != NULL) { - if (pDb->cfg.maxSessions != oldTables) { - mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); - } - } - - mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); - - return NULL; -} - -static void *mgmtVgroupActionEncode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (size < tsVgUpdateSize) { - *ssize = -1; - } else { - memcpy(str, pVgroup, tsVgUpdateSize); - *ssize = tsVgUpdateSize; - } - - return NULL; -} - -static void *mgmtVgroupActionDecode(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); - if (pVgroup == NULL) return NULL; - memset(pVgroup, 0, sizeof(SVgObj)); - - int32_t tsVgUpdateSize = pVgroup->updateEnd - (int8_t *) pVgroup; - memcpy(pVgroup, str, tsVgUpdateSize); - - return (void *) pVgroup; -} - -static void *mgmtVgroupActionReset(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - memcpy(pVgroup, str, tsVgUpdateSize); - return NULL; -} - -static void *mgmtVgroupActionDestroy(void *row, char *str, int32_t size, int32_t *ssize) { - SVgObj *pVgroup = (SVgObj *) row; - if (pVgroup->idPool) { - taosIdPoolCleanUp(pVgroup->idPool); - pVgroup->idPool = NULL; - } - if (pVgroup->tableList) tfree(pVgroup->tableList); - tfree(row); - return NULL; -} - void mgmtUpdateVgroup(SVgObj *pVgroup) { sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_LOCAL); } From d0b0ac956e5c30a44fbd8630a57ca57fe13e2b26 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 20 Mar 2020 23:18:00 +0800 Subject: [PATCH 05/85] [TD-15] fix bug in sdbInsertRow --- src/mnode/src/mgmtSdb.c | 184 +++++++++++++++++++-------------------- src/mnode/src/mgmtUser.c | 3 +- 2 files changed, 89 insertions(+), 98 deletions(-) diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 8703c63541..ebdbba5605 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -222,7 +222,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { int32_t real_size = 0; int32_t maxAutoIndex = 0; - oldId = pTable->id; if (sdbOpenSdbFile(pTable) < 0) return -1; total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); @@ -234,7 +233,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { sdbTrace("open sdb file:%s for read", pTable->fn); - // Loop to read sdb file row by row while (1) { memset(rowHead, 0, total_size); @@ -259,9 +257,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { continue; } - // sdbTrace("%s id:%ld rowSize:%d", pTable->name, rowHead->id, - // rowHead->rowSize); - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { // TODO: Here may cause pTable->size not end of the file @@ -276,40 +271,22 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { continue; } - // Check if the the object exists already + if (pTable->keyType == SDB_KEYTYPE_AUTO) { + maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); + } pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object + if (pMetaRow == NULL) { if (rowHead->id < 0) { - /* assert(0); */ sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); } else { - rowMeta.id = rowHead->id; - // TODO: Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*pTable->decodeFp)(rowHead->data); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t*)rowHead->data); - } - pTable->numOfRows++; + sdbInsertRow(pTable, rowHead->data, SDB_OPER_DISK); } - } else { // already exists - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - pTable->autoIndex++; - maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); - } - - if (rowHead->id < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*pTable->destroyFp)(pMetaRow); - pTable->numOfRows--; - numOfDels++; - } else { // Reset the object TODO: is it possible to merge reset and - // update ?? - //(*(pTable->appTool))(SDB_TYPE_RESET, pMetaRow, rowHead->data, rowHead->rowSize, NULL); + } else { + if (rowHead->id < 0) { + sdbDeleteRow(pTable, rowHead->data, SDB_OPER_DISK); + } else { + sdbUpdateRow(pTable, rowHead->data, rowHead->rowSize, SDB_OPER_DISK); } numOfDels++; } @@ -317,7 +294,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->size += real_size; if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - //TODO: check this valid pTable->size += 4; lseek(pTable->fd, 4, SEEK_CUR); } @@ -326,8 +302,10 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->autoIndex = maxAutoIndex; } - sdbVersion += (pTable->id - oldId); - if (numOfDels > pTable->hashSessions / 4) sdbSaveSnapShot(pTable); + sdbVersion += pTable->id; + if (numOfDels > pTable->hashSessions / 4) { + sdbSaveSnapShot(pTable); + } tfree(rowHead); return 0; @@ -390,12 +368,13 @@ void *sdbGetRow(void *handle, void *key) { pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); pthread_mutex_unlock(&pTable->mutex); - if (pMeta == NULL) return NULL; + if (pMeta == NULL) { + return NULL; + } return pMeta->row; } -// row here must be encoded string (rowSize > 0) or the object it self (rowSize = 0) int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta rowMeta; @@ -448,7 +427,7 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { } } - if (oper == SDB_OPER_GLOBAL || oper == SDB_OPER_LOCAL) { + if (oper != SDB_OPER_DISK) { rowHead->rowSize = (*pTable->encodeFp)(pObj, rowHead->data, pTable->maxRowSize); assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); @@ -462,43 +441,46 @@ int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { return -1; } - // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - twrite(pTable->fd, rowHead, real_size); pTable->size += real_size; sdbFinishCommit(pTable); + } - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - } - } + // update in SDB layer + rowMeta.id = pTable->id; + rowMeta.offset = pTable->size; + rowMeta.rowSize = rowHead->rowSize; + rowMeta.row = pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); if (pTable->keyType == SDB_KEYTYPE_AUTO) { *((uint32_t *)pObj) = ++pTable->autoIndex; } pTable->numOfRows++; - pTable->id++; - sdbVersion++; + if (oper != SDB_OPER_DISK) { + pTable->id++; + sdbVersion++; + } + pthread_mutex_unlock(&pTable->mutex); + switch (pTable->keyType) { + case SDB_KEYTYPE_STRING: + sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + case SDB_KEYTYPE_AUTO: + sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + default: + sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); + break; + } + (*pTable->insertFp)(pObj); tfree(rowHead); @@ -556,20 +538,22 @@ int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { } } - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } + if (oper != SDB_OPER_DISK) { + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->id = -(pTable->id); + memcpy(rowHead->data, row, rowSize); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); + twrite(pTable->fd, rowHead, total_size); + pTable->size += total_size; + sdbFinishCommit(pTable); + } switch (pTable->keyType) { case SDB_KEYTYPE_STRING: @@ -590,14 +574,18 @@ int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); pTable->numOfRows--; - pTable->id++; - sdbVersion++; + if (oper != SDB_OPER_DISK) { + pTable->id++; + sdbVersion++; + } + pthread_mutex_unlock(&pTable->mutex); tfree(rowHead); (*pTable->deleteFp)(pMetaRow); + (*pTable->destroyFp)(pMetaRow); return 0; } @@ -661,23 +649,25 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType o real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); // write to the new position - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; + if (oper != SDB_OPER_DISK) { + rowHead->delimiter = SDB_DELIMITER; + rowHead->id = pTable->id; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; + } + + twrite(pTable->fd, rowHead, real_size); + + pMeta->id = pTable->id; + pMeta->offset = pTable->size; + pMeta->rowSize = rowHead->rowSize; + pTable->size += real_size; + + sdbFinishCommit(pTable); } - - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); switch (pTable->keyType) { case SDB_KEYTYPE_STRING: @@ -694,8 +684,10 @@ int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType o break; } - pTable->id++; - sdbVersion++; + if (oper != SDB_OPER_DISK) { + pTable->id++; + sdbVersion++; + } pthread_mutex_unlock(&pTable->mutex); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 2ca37a4572..3321c41513 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -176,8 +176,7 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { return code; } - pUser = malloc(sizeof(SUserObj)); - memset(pUser, 0, sizeof(SUserObj)); + pUser = calloc(1, sizeof(SUserObj)); strcpy(pUser->user, name); taosEncryptPass((uint8_t*) pass, strlen(pass), pUser->pass); strcpy(pUser->acct, pAcct->user); From 35b52a5f2770d27a00eb8dfabf93198df563a8f0 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 21 Mar 2020 22:23:35 +0800 Subject: [PATCH 06/85] TD-34 --- src/common/inc/dataformat.h | 2 ++ src/vnode/tsdb/inc/tsdbMeta.h | 32 ++++++++++++++++++------------ src/vnode/tsdb/src/tsdbMain.c | 19 +++++++++++------- src/vnode/tsdb/src/tsdbMeta.c | 31 +++++++++++------------------ src/vnode/tsdb/tests/tsdbTests.cpp | 3 +-- 5 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index aff239712b..1f46c68abc 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -81,11 +81,13 @@ STSchema *tdDecodeSchema(void **psrc); */ typedef void *SDataRow; + #define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t)) #define dataRowLen(r) (*(int32_t *)(r)) #define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t))) #define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE) +#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) #define dataRowSetLen(r, l) (dataRowLen(r) = (l)) #define dataRowSetFLen(r, l) (dataRowFLen(r) = (l)) #define dataRowIdx(r, i) ((char *)(r) + i) diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h index 38f0818dfb..03031f16cd 100644 --- a/src/vnode/tsdb/inc/tsdbMeta.h +++ b/src/vnode/tsdb/inc/tsdbMeta.h @@ -33,20 +33,25 @@ extern "C" { #define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL) +typedef struct { + TSKEY keyFirst; + TSKEY keyLast; + int32_t numOfPoints; + void * pData; +} SMemTable; + // ---------- TSDB TABLE DEFINITION typedef struct STable { - int8_t type; - STableId tableId; - int32_t superUid; // Super table UID - int32_t sversion; - STSchema *schema; - STSchema *tagSchema; - SDataRow tagVal; - union { - void *pData; // For TSDB_NORMAL_TABLE and TSDB_CHILD_TABLE, it is the skiplist for cache data - void *pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - } content; - void * iData; // Skiplist to commit + int8_t type; + STableId tableId; + int32_t superUid; // Super table UID + int32_t sversion; + STSchema * schema; + STSchema * tagSchema; + SDataRow tagVal; + SMemTable * mem; + SMemTable * imem; + void * pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void * eventHandler; // TODO void * streamHandler; // TODO struct STable *next; // TODO: remove the next @@ -94,8 +99,9 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta); int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg); int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); -int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); +// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); +char *getTupleKey(const void * data); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index ed95eac5bc..208272fa8b 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -311,10 +311,9 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { // Loop to move pData to iData for (int i = 0; i < pRepo->config.maxTables; i++) { STable *pTable = pRepo->tsdbMeta->tables[i]; - if (pTable != NULL) { - void *pData = pTable->content.pData; - pTable->content.pData = NULL; - pTable->iData = pData; + if (pTable != NULL && pTable->mem != NULL) { + pTable->imem = pTable->mem; + pTable->mem = NULL; } } // Loop to move mem to imem @@ -669,7 +668,13 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable int32_t level = 0; int32_t headSize = 0; - tSkipListRandNodeInfo(pTable->content.pData, &level, &headSize); + if (pTable->mem == NULL) { + pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); + if (pTable->mem == NULL) return -1; + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); + } + + tSkipListRandNodeInfo(pTable->mem->pData, &level, &headSize); // Copy row into the memory SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row)); @@ -681,7 +686,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable dataRowCpy(SL_GET_NODE_DATA(pNode), row); // Insert the skiplist node into the data - tsdbInsertRowToTableImpl(pNode, pTable); + tSkipListPut(pTable->mem->pData, pNode); return 0; } @@ -712,7 +717,7 @@ static void *tsdbCommitToFile(void *arg) { for (int i = 0; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; - SSkipListIterator *pIter = tSkipListCreateIter(pTable->iData); + SSkipListIterator *pIter = tSkipListCreateIter(pTable->imem->pData); while (tSkipListIterNext(pIter)) { SSkipListNode *node = tSkipListIterGet(pIter); SDataRow row = SL_GET_NODE_DATA(node); diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 98dcd45bed..c7874fa5a2 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -18,7 +18,6 @@ static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable); static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbEstimateTableEncodeSize(STable *pTable); -static char * getTupleKey(const void *data); /** * Encode a TSDB table object as a binary content @@ -102,12 +101,9 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { if (pTable == NULL) return -1; if (pTable->type == TSDB_SUPER_TABLE) { - pTable->content.pIndex = + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, getTupleKey); - } else { - pTable->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); - } + } tsdbAddTableToMeta(pMeta, pTable, false); @@ -208,10 +204,10 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->schema = tdDupSchema(pCfg->schema); super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = tdDataRowDup(pCfg->tagValues); - super->content.pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, + super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, getTupleKey); // Allow duplicate key, no lock - if (super->content.pIndex == NULL) { + if (super->pIndex == NULL) { tdFreeSchema(super->schema); tdFreeSchema(super->tagSchema); tdFreeDataRow(super->tagVal); @@ -223,7 +219,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { } } - STable *table = (STable *)malloc(sizeof(STable)); + STable *table = (STable *)calloc(1, sizeof(STable)); if (table == NULL) { if (newSuper) tsdbFreeTable(super); return -1; @@ -239,7 +235,6 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->superUid = -1; table->schema = tdDupSchema(pCfg->schema); } - table->content.pData = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); // Register to meta if (newSuper) tsdbAddTableToMeta(pMeta, super, true); @@ -299,10 +294,10 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { return 0; } -int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) { - tSkipListPut(pTable->content.pData, pNode); - return 0; -} +// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) { +// tSkipListPut(pTable->mem->pData, pNode); +// return 0; +// } static int tsdbFreeTable(STable *pTable) { // TODO: finish this function @@ -314,10 +309,8 @@ static int tsdbFreeTable(STable *pTable) { // Free content if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) { - tSkipListDestroy(pTable->content.pIndex); - } else { - tSkipListDestroy(pTable->content.pData); - } + tSkipListDestroy(pTable->pIndex); + } free(pTable); return 0; @@ -404,7 +397,7 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) { return size; } -static char *getTupleKey(const void * data) { +char *getTupleKey(const void * data) { SDataRow row = (SDataRow)data; return dataRowAt(row, TD_DATA_ROW_HEAD_SIZE); diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 42a22553c7..de58d6337c 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -40,7 +40,6 @@ TEST(TsdbTest, tableEncodeDecode) { ASSERT_EQ(pTable->superUid, tTable->superUid); ASSERT_EQ(pTable->sversion, tTable->sversion); ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0); - ASSERT_EQ(tTable->content.pData, nullptr); } TEST(TsdbTest, createRepo) { @@ -72,7 +71,7 @@ TEST(TsdbTest, createRepo) { tsdbCreateTable(pRepo, &tCfg); // // 3. Loop to write some simple data - int nRows = 100; + int nRows = 1000; int rowsPerSubmit = 10; int64_t start_time = 1584081000000; From 7a56281311f098a188097a59e3a81dd112cfa5a0 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 21 Mar 2020 22:38:32 +0800 Subject: [PATCH 07/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 208272fa8b..a02343f817 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -672,6 +672,8 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); if (pTable->mem == NULL) return -1; pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); + pTable->mem->keyFirst = INT64_MAX; + pTable->mem->keyLast = 0; } tSkipListRandNodeInfo(pTable->mem->pData, &level, &headSize); @@ -687,6 +689,10 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable // Insert the skiplist node into the data tSkipListPut(pTable->mem->pData, pNode); + TSKEY key = dataRowKey(row); + if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; + if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; + pTable->mem->numOfPoints++; return 0; } From d93d1093b849a9363cb9c3e15b90d9cdf6a326f6 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 22 Mar 2020 10:12:41 +0800 Subject: [PATCH 08/85] [TD-15] refact the interface of sdb --- src/mnode/inc/mgmtSdb.h | 43 +-- src/mnode/src/mgmtSdb.c | 800 +++++++++++++++++++++------------------- 2 files changed, 438 insertions(+), 405 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index bf5ccb2634..e33bfce824 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -21,46 +21,47 @@ extern "C" { #endif typedef enum { - SDB_KEYTYPE_STRING, - SDB_KEYTYPE_AUTO, - SDB_KEYTYPE_MAX + SDB_KEY_TYPE_STRING, + SDB_KEY_TYPE_AUTO } ESdbKeyType; typedef enum { - SDB_OPER_GLOBAL, - SDB_OPER_LOCAL, - SDB_OPER_DISK + SDB_OPER_TYPE_GLOBAL, + SDB_OPER_TYPE_LOCAL } ESdbOperType; -enum _sdbaction { - SDB_TYPE_INSERT, - SDB_TYPE_DELETE, - SDB_TYPE_UPDATE, -} ESdbForwardType; +typedef struct { + ESdbOperType type; + int32_t maxRowSize; + int32_t rowSize; + void * rowData; + void * pObj; + void * table; + int64_t version; +} SSdbOperDesc; typedef struct { char *tableName; int32_t hashSessions; int32_t maxRowSize; ESdbKeyType keyType; - int32_t (*insertFp)(void *pObj); - int32_t (*deleteFp)(void *pObj); - int32_t (*updateFp)(void *pObj); - int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); - void * (*decodeFp)(void *pData); - int32_t (*destroyFp)(void *pObj); + int32_t (*insertFp)(SSdbOperDesc *pOper); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pDesc); + int32_t (*destroyFp)(SSdbOperDesc *pDesc); } SSdbTableDesc; void *sdbOpenTable(SSdbTableDesc *desc); void sdbCloseTable(void *handle); -int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper); -int32_t sdbDeleteRow(void *handle, void *key, ESdbOperType oper); -int32_t sdbUpdateRow(void *handle, void *row, int32_t rowSize, ESdbOperType oper); +int32_t sdbInsertRow(SSdbOperDesc *pOper); +int32_t sdbDeleteRow(SSdbOperDesc *pOper); +int32_t sdbUpdateRow(SSdbOperDesc *pOper); void *sdbGetRow(void *handle, void *key); void *sdbFetchRow(void *handle, void *pNode, void **ppRow); -int64_t sdbGetId(void *handle); int64_t sdbGetNumOfRows(void *handle); uint64_t sdbGetVersion(); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index ebdbba5605..f957dc4a83 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" +#include "taoserror.h" #include "tchecksum.h" #include "tglobalcfg.h" #include "tlog.h" @@ -39,29 +40,29 @@ typedef struct { typedef struct _SSdbTable { SSdbHeader header; - char name[TSDB_DB_NAME_LEN]; - char fn[TSDB_FILENAME_LEN]; + char tableName[TSDB_DB_NAME_LEN]; + char fileName[TSDB_FILENAME_LEN]; ESdbKeyType keyType; - int32_t dbId; + int32_t tableId; int32_t hashSessions; int32_t maxRowSize; - uint32_t autoIndex; - int64_t numOfRows; - int64_t id; - int64_t size; - void * iHandle; + int32_t autoIndex; int32_t fd; - int32_t (*insertFp)(void *pObj); - int32_t (*deleteFp)(void *pObj); - int32_t (*updateFp)(void *pObj); - void * (*decodeFp)(void *pData); // return pObj - int32_t (*encodeFp)(void *pObj, void *pData, int32_t maxRowSize); // return size of pData - int32_t (*destroyFp)(void *pObj); + int64_t numOfRows; + int64_t version; + int64_t fileSize; + void * iHandle; + int32_t (*insertFp)(SSdbOperDesc *pDesc); + int32_t (*deleteFp)(SSdbOperDesc *pOper); + int32_t (*updateFp)(SSdbOperDesc *pOper); + int32_t (*decodeFp)(SSdbOperDesc *pOper); + int32_t (*encodeFp)(SSdbOperDesc *pOper); + int32_t (*destroyFp)(SSdbOperDesc *pOper); pthread_mutex_t mutex; } SSdbTable; typedef struct { - int64_t id; + int64_t version; int64_t offset; int32_t rowSize; void * row; @@ -70,21 +71,27 @@ typedef struct { typedef struct { int32_t delimiter; int32_t rowSize; - int64_t id; + int64_t version; char data[]; } SRowHead; +typedef enum { + SDB_FORWARD_TYPE_INSERT, + SDB_FORWARD_TYPE_DELETE, + SDB_FORWARD_TYPE_UPDATE +} ESdbForwardType; + typedef struct { - uint8_t dbId; - int8_t type; - int16_t dataLen; - uint64_t version; - char data[]; + ESdbForwardType type; + int32_t tableId; + int64_t version; + int32_t rowSize; + void * rowData; } SForwardMsg; extern char version[]; const int16_t sdbFileVersion = 2; -int32_t (*mpeerForwardRequestFp)(SSdbTable *pTable, char type, void *cont, int32_t contLen) = NULL; +int32_t (*mpeerForwardRequestFp)(SForwardMsg *forwardMsg) = NULL; static SSdbTable *sdbTableList[10] = {0}; static int32_t sdbNumOfTables = 0; @@ -101,25 +108,36 @@ void sdbResetTable(SSdbTable *pTable); void sdbSaveSnapShot(void *handle); uint64_t sdbGetVersion() { return sdbVersion; } -int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->id; } +int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } -static int32_t sdbForwardDbReqToPeer(SSdbTable *pTable, char type, char *data, int32_t dataLen) { +static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { + static char str[16]; + switch (pTable->keyType) { + case SDB_KEY_TYPE_STRING: + return (char *)row; + case SDB_KEY_TYPE_AUTO: + sprintf(str, "%d", *(int32_t *)row); + return str; + default: + return "unknown"; + } +} + +static int32_t sdbForwardDbReqToPeer(SForwardMsg *forwardMsg) { if (mpeerForwardRequestFp) { - return mpeerForwardRequestFp(pTable, type, data, dataLen); + return mpeerForwardRequestFp(forwardMsg); } else { return 0; } } -static void sdbFinishCommit(void *handle) { - SSdbTable *pTable = (SSdbTable *)handle; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - +static void sdbFinishCommit(SSdbTable *pTable) { + uint32_t sdbEcommit = SDB_ENDCOMMIT; off_t offset = lseek(pTable->fd, 0, SEEK_END); - assert(offset == pTable->size); + assert(offset == pTable->fileSize); twrite(pTable->fd, &sdbEcommit, sizeof(sdbEcommit)); - pTable->size += sizeof(sdbEcommit); + pTable->fileSize += sizeof(sdbEcommit); } static int32_t sdbOpenSdbFile(SSdbTable *pTable) { @@ -136,40 +154,40 @@ static int32_t sdbOpenSdbFile(SSdbTable *pTable) { // check sdb.db and .sdb.db status char fn[TSDB_FILENAME_LEN] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); + dirc = strdup(pTable->fileName); + basec = strdup(pTable->fileName); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); tfree(dirc); tfree(basec); if (stat(fn, &ofstat) == 0) { // .sdb.db file exists - if (stat(pTable->fn, &fstat) == 0) { + if (stat(pTable->fileName, &fstat) == 0) { remove(fn); } else { - remove(pTable->fn); - rename(fn, pTable->fn); + remove(pTable->fileName); + rename(fn, pTable->fileName); } } - pTable->fd = open(pTable->fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pTable->fd = open(pTable->fileName, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); if (pTable->fd < 0) { - sdbError("failed to open file:%s", pTable->fn); + sdbError("table:%s, failed to open file:%s", pTable->tableName, pTable->fileName); return -1; } - pTable->size = 0; - stat(pTable->fn, &fstat); + pTable->fileSize = 0; + stat(pTable->fileName, &fstat); size = sizeof(pTable->header); if (fstat.st_size == 0) { pTable->header.swVersion = swVersion.iversion; pTable->header.sdbFileVersion = sdbFileVersion; if (taosCalcChecksumAppend(0, (uint8_t *)(&pTable->header), size) < 0) { - sdbError("failed to get file header checksum, file:%s", pTable->fn); + sdbError("table:%s, failed to get file header checksum, file:%s", pTable->tableName, pTable->fileName); tclose(pTable->fd); return -1; } twrite(pTable->fd, &(pTable->header), size); - pTable->size += size; + pTable->fileSize += size; sdbFinishCommit(pTable); } else { uint32_t sdbEcommit = 0; @@ -186,25 +204,25 @@ static int32_t sdbOpenSdbFile(SSdbTable *pTable) { ssize_t tsize = read(pTable->fd, &(pTable->header), size); if (tsize < size) { - sdbError("failed to read sdb file header, file:%s", pTable->fn); + sdbError("table:%s, failed to read sdb file header, file:%s", pTable->tableName, pTable->fileName); tclose(pTable->fd); return -1; } if (pTable->header.swVersion != swVersion.iversion) { - sdbWarn("sdb file:%s version not match software version", pTable->fn); + sdbWarn("table:%s, sdb file:%s version not match software version", pTable->tableName, pTable->fileName); } if (!taosCheckChecksumWhole((uint8_t *)(&pTable->header), size)) { - sdbError("sdb file header is broken since checksum mismatch, file:%s", pTable->fn); + sdbError("table:%s, sdb file header is broken since checksum mismatch, file:%s", pTable->tableName, pTable->fileName); tclose(pTable->fd); return -1; } - pTable->size += size; + pTable->fileSize += size; // skip end commit symbol lseek(pTable->fd, sizeof(sdbEcommit), SEEK_CUR); - pTable->size += sizeof(sdbEcommit); + pTable->fileSize += sizeof(sdbEcommit); } pTable->numOfRows = 0; @@ -213,106 +231,170 @@ static int32_t sdbOpenSdbFile(SSdbTable *pTable) { } static int32_t sdbInitTableByFile(SSdbTable *pTable) { - SRowMeta rowMeta; - int32_t numOfDels = 0; - int32_t bytes = 0; - int64_t oldId = 0; - void * pMetaRow = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - int32_t maxAutoIndex = 0; - - if (sdbOpenSdbFile(pTable) < 0) return -1; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + sdbTrace("table:%s, open sdb file:%s for read", pTable->tableName, pTable->fileName); + if (sdbOpenSdbFile(pTable) < 0) { + sdbError("table:%s, failed to open sdb file:%s for read", pTable->tableName, pTable->fileName); + return -1; + } + + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); + sdbError("table:%s, failed to allocate row head memory, sdb:%s", pTable->tableName, pTable->tableName); return -1; } - sdbTrace("open sdb file:%s for read", pTable->fn); - + int32_t numOfChanged = 0; + int32_t maxAutoIndex = 0; while (1) { memset(rowHead, 0, total_size); - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); + int32_t bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); - goto sdb_exit1; + sdbError("table:%s, failed to read sdb file:%s", pTable->tableName, pTable->fileName); + tfree(rowHead); + return -1; } if (bytes == 0) break; if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; + pTable->fileSize++; lseek(pTable->fd, -(bytes - 1), SEEK_CUR); continue; } if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); + sdbError("table:%s, error row size in sdb filesize:%d, version:%d rowSize:%d maxRowSize:%d", pTable->tableName, + pTable->fileSize, rowHead->version, rowHead->rowSize, pTable->maxRowSize); + pTable->fileSize += sizeof(SRowHead); continue; } bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - // TODO: Here may cause pTable->size not end of the file - sdbError("failed to read sdb file:%s id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); + // TODO: Here may cause pTable->fileSize not end of the file + sdbError("table:%s, failed to read sdb file, version:%d rowSize:%d", pTable->tableName, rowHead->version, + rowHead->rowSize); break; } - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; + sdbError("table:%s, error sdb checksum, version:%d, skip", pTable->tableName, rowHead->version); + pTable->fileSize += real_size; continue; } - - if (pTable->keyType == SDB_KEYTYPE_AUTO) { + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); } - pMetaRow = sdbGetRow(pTable, rowHead->data); + void *pMetaRow = sdbGetRow(pTable, rowHead->data); if (pMetaRow == NULL) { - if (rowHead->id < 0) { - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); + if (rowHead->version < 0) { + sdbError("table:%s, error sdb negative version:%d, record:%s, skip", pTable->tableName, rowHead->version, + sdbGetkeyStr(pTable, rowHead->data)); } else { - sdbInsertRow(pTable, rowHead->data, SDB_OPER_DISK); + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize + }; + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + pTable->numOfRows++; + sdbTrace("table:%s, read record:%s and insert, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } else { + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } } } else { - if (rowHead->id < 0) { - sdbDeleteRow(pTable, rowHead->data, SDB_OPER_DISK); + if (rowHead->version < 0) { + SSdbOperDesc oper = { + .table = pTable, + .pObj = pMetaRow + }; + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + pTable->numOfRows--; + sdbTrace("table:%s, read record:%s and delete, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); } else { - sdbUpdateRow(pTable, rowHead->data, rowHead->rowSize, SDB_OPER_DISK); + SRowMeta rowMeta; + rowMeta.version = rowHead->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = rowHead->rowSize; + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .rowSize = rowHead->rowSize, + .pObj = pMetaRow + }; + + (*pTable->destroyFp)(&oper); + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); + + int32_t code = (*pTable->decodeFp)(&oper); + if (code == TSDB_CODE_SUCCESS) { + rowMeta.row = oper.pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); + sdbTrace("table:%s, read record:%s and update, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } else { + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + } + } + numOfChanged++; + if (pTable->version < abs(rowHead->version)) { + pTable->version = abs(rowHead->version); } - numOfDels++; } - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); - - pTable->size += 4; + pTable->fileSize += real_size; + pTable->fileSize += 4; lseek(pTable->fd, 4, SEEK_CUR); } - if (pTable->keyType == SDB_KEYTYPE_AUTO) { + void *pNode = NULL; + while (1) { + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; + + SSdbOperDesc oper = { + .pObj = pMeta->row, + .table = pTable, + .version = pMeta->version, + }; + + int32_t code = (*pTable->insertFp)(&oper); + if (code != TSDB_CODE_SUCCESS) { + sdbError("table:%s, failed to insert record:%s", pTable->tableName, sdbGetkeyStr(pTable, rowHead->data)); + } + } + + sdbVersion += pTable->version; + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { pTable->autoIndex = maxAutoIndex; } - sdbVersion += pTable->id; - if (numOfDels > pTable->hashSessions / 4) { + if (numOfChanged > pTable->hashSessions / 4) { sdbSaveSnapShot(pTable); } tfree(rowHead); return 0; - -sdb_exit1: - tfree(rowHead); - return -1; } void *sdbOpenTable(SSdbTableDesc *pDesc) { @@ -328,8 +410,8 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->encodeFp = pDesc->encodeFp; pTable->decodeFp = pDesc->decodeFp; pTable->destroyFp = pDesc->destroyFp; - strcpy(pTable->name, pDesc->tableName); - sprintf(pTable->fn, "%s/%s.db", tsMnodeDir, pTable->name); + strcpy(pTable->tableName, pDesc->tableName); + sprintf(pTable->fileName, "%s/%s.db", tsMnodeDir, pTable->tableName); if (sdbInitIndexFp[pTable->keyType] != NULL) { pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SRowMeta)); @@ -339,10 +421,11 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { if (sdbInitTableByFile(pTable) < 0) return NULL; - pTable->dbId = sdbNumOfTables++; - sdbTableList[pTable->dbId] = pTable; + pTable->tableId = sdbNumOfTables++; + sdbTableList[pTable->tableId] = pTable; - sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d", pTable->name, pTable->numOfRows, sdbNumOfTables); + sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d, version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, pTable->numOfRows, sdbNumOfTables, pTable->version, sdbVersion); return pTable; } @@ -375,323 +458,267 @@ void *sdbGetRow(void *handle, void *key) { return pMeta->row; } -int32_t sdbInsertRow(void *handle, void *row, ESdbOperType oper) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta rowMeta; - void * pObj = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - +int32_t sdbInsertRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) { sdbError("sdb tables is null"); - return -1; - } - - if (sdbGetRow(handle, row)) { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to insert record:%s sdbVersion:%" PRId64 " id:%" PRId64 , pTable->name, (char *)row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to insert record:%d sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, *(int32_t *)row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to insert record sdbVersion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); - break; - } - return -1; + return TSDB_CODE_OTHERS; } - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("table:%s, failed to allocate row head memory", pTable->name); - return -1; + if (sdbGetRow(pTable, pOper->pObj)) { + sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); + return TSDB_CODE_ALREADY_THERE; } - memset(rowHead, 0, total_size); - if (oper == SDB_OPER_GLOBAL) { - pObj = row; - } else { - pObj = (*pTable->decodeFp)(row); - } - pthread_mutex_lock(&pTable->mutex); - if (oper == SDB_OPER_GLOBAL) { - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_INSERT, rowHead->data, rowHead->rowSize) != 0) { - sdbError("table:%s, failed to insert record", pTable->name); + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_INSERT, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + + if (sdbForwardDbReqToPeer(&forward) != 0) { + sdbError("table:%s, failed to forward record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; + return TSDB_CODE_OTHERS; } } - if (oper != SDB_OPER_DISK) { - rowHead->rowSize = (*pTable->encodeFp)(pObj, rowHead->data, pTable->maxRowSize); - assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + pthread_mutex_unlock(&pTable->mutex); + sdbError("table:%s, failed to allocate row head memory for record:%s version:%" PRId64 " sdbversion:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), pOper->version, sdbVersion); + return -1; + } + + if (pTable->keyType == SDB_KEY_TYPE_AUTO) { + *((uint32_t *)pOper->pObj) = ++pTable->autoIndex; + } + pTable->version++; + sdbVersion++; - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id + 1; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("table:%s, failed to get checksum while inserting", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } + pOper->rowData = rowHead->data; + (*pTable->encodeFp)(pOper); + rowHead->rowSize = pOper->rowSize; - twrite(pTable->fd, rowHead, real_size); - pTable->size += real_size; - sdbFinishCommit(pTable); + assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); + + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; } + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + tfree(rowHead); + // update in SDB layer - rowMeta.id = pTable->id; - rowMeta.offset = pTable->size; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = pObj; - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pObj, &rowMeta); - - if (pTable->keyType == SDB_KEYTYPE_AUTO) { - *((uint32_t *)pObj) = ++pTable->autoIndex; - } + SRowMeta rowMeta; + rowMeta.version = pTable->version; + rowMeta.offset = pTable->fileSize; + rowMeta.rowSize = pOper->rowSize; + rowMeta.row = pOper->pObj; + (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); pTable->numOfRows++; - - if (oper != SDB_OPER_DISK) { - pTable->id++; - sdbVersion++; - } pthread_mutex_unlock(&pTable->mutex); - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is inserted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, (char *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is inserted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - default: - sdbTrace("table:%s, a record is inserted, sdbVersion:%" PRId64 " id:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->name, sdbVersion, rowHead->id, rowHead->rowSize, pTable->numOfRows, pTable->size); - break; - } + sdbTrace("table:%s, a record is inserted:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pOper->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); - (*pTable->insertFp)(pObj); - - tfree(rowHead); + (*pTable->insertFp)(pOper); return 0; } // row here can be object or null-terminated string -int32_t sdbDeleteRow(void *handle, void *row, ESdbOperType oper) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - void * pMetaRow = NULL; - SRowHead * rowHead = NULL; - int32_t rowSize = 0; - int32_t total_size = 0; - +int32_t sdbDeleteRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - pMeta = sdbGetRowMeta(handle, row); + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { - sdbTrace("table:%s, record is not there, delete failed", pTable->name); + sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; } - pMetaRow = pMeta->row; + void * pMetaRow = pMeta->row; assert(pMetaRow != NULL); + pthread_mutex_lock(&pTable->mutex); + + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_DELETE, + .tableId = pTable->tableId, + .version = pTable->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + + if (sdbForwardDbReqToPeer(&forward) == 0) { + sdbError("table:%s, failed to delete record", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + } + + int32_t total_size = sizeof(SRowHead) + pOper->rowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); + pthread_mutex_unlock(&pTable->mutex); + return -1; + } + + pTable->version++; + sdbVersion++; + + int32_t rowSize = 0; switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - rowSize = strlen((char *)row) + 1; + case SDB_KEY_TYPE_STRING: + rowSize = strlen((char *)pOper->rowData) + 1; break; - case SDB_KEYTYPE_AUTO: + case SDB_KEY_TYPE_AUTO: rowSize = sizeof(uint64_t); break; default: return -1; } - total_size = sizeof(SRowHead) + rowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); + rowHead->delimiter = SDB_DELIMITER; + rowHead->rowSize = rowSize; + rowHead->version = -(pTable->version); + memcpy(rowHead->data, pOper->rowData, pOper->rowSize); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); return -1; } - memset(rowHead, 0, total_size); - pthread_mutex_lock(&pTable->mutex); - - if (oper == SDB_OPER_GLOBAL) { - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_DELETE, (char *)row, rowSize) == 0) { - sdbError("table:%s, failed to delete record", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - } - - if (oper != SDB_OPER_DISK) { - rowHead->delimiter = SDB_DELIMITER; - rowHead->rowSize = rowSize; - rowHead->id = -(pTable->id); - memcpy(rowHead->data, row, rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { - sdbError("failed to get checksum while inserting, sdb:%s", pTable->name); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, total_size); - pTable->size += total_size; - sdbFinishCommit(pTable); - } - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is deleted:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is deleted:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is deleted, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->name, sdbVersion, pTable->id, pTable->numOfRows); - break; - } - - // Delete from current layer - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, row); - - pTable->numOfRows--; - - if (oper != SDB_OPER_DISK) { - pTable->id++; - sdbVersion++; - } - - pthread_mutex_unlock(&pTable->mutex); + twrite(pTable->fd, rowHead, total_size); + pTable->fileSize += total_size; + sdbFinishCommit(pTable); tfree(rowHead); - (*pTable->deleteFp)(pMetaRow); - (*pTable->destroyFp)(pMetaRow); + sdbTrace("table:%s, a record is deleted:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + + // Delete from current layer + (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + + pTable->numOfRows--; + + pthread_mutex_unlock(&pTable->mutex); + + (*pTable->deleteFp)(pOper); + (*pTable->destroyFp)(pOper); return 0; } // row here can be the object or the string info (encoded string) -int32_t sdbUpdateRow(void *handle, void *row, int32_t updateSize, ESdbOperType oper) { - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta = NULL; - int32_t total_size = 0; - int32_t real_size = 0; +int32_t sdbUpdateRow(SSdbOperDesc *pOper) { + SSdbTable *pTable = (SSdbTable *)pOper->table; + if (pTable == NULL) return -1; - if (pTable == NULL || row == NULL) return -1; - pMeta = sdbGetRowMeta(handle, row); + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->rowData); if (pMeta == NULL) { - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbError("table:%s, failed to update record:%s, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, (char *) row, sdbVersion, pTable->id); - break; - case SDB_KEYTYPE_AUTO: - sdbError("table:%s, failed to update record:%d, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, *(int32_t *) row, sdbVersion, pTable->id); - break; - default: - sdbError("table:%s, failed to update record, record is not there, sdbVersion:%" PRId64 " id:%" PRId64, - pTable->name, sdbVersion, pTable->id); - break; - } + sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " id:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version); return -1; } void *pMetaRow = pMeta->row; assert(pMetaRow != NULL); - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory, sdb:%s", pTable->name); - return -1; - } - memset(rowHead, 0, total_size); - pthread_mutex_lock(&pTable->mutex); - if (oper == SDB_OPER_GLOBAL) { - if (sdbForwardDbReqToPeer(pTable, SDB_TYPE_UPDATE, rowHead->data, rowHead->rowSize) == 0) { - sdbError("table:%s, failed to update record", pTable->name); + if (pOper->type == SDB_OPER_TYPE_GLOBAL) { + SForwardMsg forward = { + .type = SDB_FORWARD_TYPE_UPDATE, + .tableId = pTable->tableId, + .version = pOper->version + 1, + .rowSize = pOper->rowSize, + .rowData = pOper->rowData, + }; + if (sdbForwardDbReqToPeer(&forward) == 0) { + sdbError("table:%s, failed to update record", pTable->tableName); pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); return -1; } } - if (pMetaRow != row) { - memcpy(rowHead->data, row, updateSize); - rowHead->rowSize = updateSize; + int32_t total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); + SRowHead *rowHead = (SRowHead *)calloc(1, total_size); + if (rowHead == NULL) { + sdbError("table:%s, failed to allocate row head memory", pTable->tableName); + return -1; + } + + if (pMetaRow != pOper->pObj) { + memcpy(rowHead->data, pOper->rowData, pOper->rowSize); + rowHead->rowSize = pOper->rowSize; } else { - rowHead->rowSize = (*pTable->encodeFp)(pMetaRow, rowHead->data, pTable->maxRowSize); + SSdbOperDesc oper = { + .table = pTable, + .rowData = rowHead->data, + .maxRowSize = pTable->maxRowSize, + .pObj = pOper->pObj + }; + (*pTable->encodeFp)(&oper); + rowHead->rowSize = oper.rowSize; } - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + pTable->version++; + sdbVersion++; - // write to the new position - if (oper != SDB_OPER_DISK) { - rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pTable->id; - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s id:%d", pTable->name, rowHead->id); - pthread_mutex_unlock(&pTable->mutex); - tfree(rowHead); - return -1; - } - - twrite(pTable->fd, rowHead, real_size); - - pMeta->id = pTable->id; - pMeta->offset = pTable->size; - pMeta->rowSize = rowHead->rowSize; - pTable->size += real_size; - - sdbFinishCommit(pTable); - } - - switch (pTable->keyType) { - case SDB_KEYTYPE_STRING: - sdbTrace("table:%s, a record is updated:%s, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, (char *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - case SDB_KEYTYPE_AUTO: - sdbTrace("table:%s, a record is updated:%d, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->name, *(int32_t *)row, sdbVersion, pTable->id, pTable->numOfRows); - break; - default: - sdbTrace("table:%s, a record is updated, sdbVersion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, pTable->name, sdbVersion, - pTable->id, pTable->numOfRows); - break; - } - - if (oper != SDB_OPER_DISK) { - pTable->id++; - sdbVersion++; + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { + sdbError("failed to get checksum, sdb:%s version:%d", pTable->tableName, rowHead->version); + pTable->version--; + sdbVersion--; + pthread_mutex_unlock(&pTable->mutex); + tfree(rowHead); + return -1; } + + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; + sdbFinishCommit(pTable); + + sdbTrace("table:%s, a record is updated:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + pMeta->version = pTable->version; + pMeta->offset = pTable->fileSize; + pMeta->rowSize = rowHead->rowSize; + pthread_mutex_unlock(&pTable->mutex); - (*pTable->updateFp)(pMetaRow); // update in upper layer + (*pTable->updateFp)(pOper); // update in upper layer tfree(rowHead); @@ -708,7 +735,12 @@ void sdbCloseTable(void *handle) { while (1) { pNode = sdbFetchRow(handle, pNode, &row); if (row == NULL) break; - (*pTable->destroyFp)(row); + + SSdbOperDesc oper = { + .table = pTable, + .rowData = row, + }; + (*pTable->destroyFp)(&oper); } if (sdbCleanUpIndexFp[pTable->keyType]) (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); @@ -718,20 +750,20 @@ void sdbCloseTable(void *handle) { pthread_mutex_destroy(&pTable->mutex); sdbNumOfTables--; - sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->name, pTable->id, sdbNumOfTables); + sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); tfree(pTable); } void sdbResetTable(SSdbTable *pTable) { - /* SRowHead rowHead; */ + /* SRowMeta rowMeta; int32_t bytes; int32_t total_size = 0; int32_t real_size = 0; SRowHead *rowHead = NULL; void * pMetaRow = NULL; - int64_t oldId = pTable->id; + int64_t oldId = pTable->version; int32_t oldNumOfRows = pTable->numOfRows; if (sdbOpenSdbFile(pTable) < 0) return; @@ -740,18 +772,18 @@ void sdbResetTable(SSdbTable *pTable) { total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->name); + sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->tableName); return; } - sdbPrint("open sdb file:%s for reset table", pTable->fn); + sdbPrint("open sdb file:%s for reset table", pTable->fileName); while (1) { memset(rowHead, 0, total_size); bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fn); + sdbError("failed to read sdb file:%s", pTable->fileName); tfree(rowHead); return; } @@ -759,40 +791,40 @@ void sdbResetTable(SSdbTable *pTable) { if (bytes == 0) break; if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->size++; + pTable->fileSize++; lseek(pTable->fd, -(bytes - 1), SEEK_CUR); continue; } if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s for reset, id:%d rowSize:%d maxRowSize:%d", - pTable->fn, rowHead->id, rowHead->rowSize, pTable->maxRowSize); - pTable->size += sizeof(SRowHead); + sdbError("error row size in sdb file:%s for reset, version:%d rowSize:%d maxRowSize:%d", + pTable->fileName, rowHead->version, rowHead->rowSize, pTable->maxRowSize); + pTable->fileSize += sizeof(SRowHead); continue; } bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - sdbError("failed to read sdb file:%s for reset, id:%d rowSize:%d", pTable->fn, rowHead->id, rowHead->rowSize); + sdbError("failed to read sdb file:%s for reset, version:%d rowSize:%d", pTable->fileName, rowHead->version, rowHead->rowSize); break; } real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s id:%d, skip", pTable->name, rowHead->id); - pTable->size += real_size; + sdbError("error sdb checksum, sdb:%s version:%d, skip", pTable->tableName, rowHead->version); + pTable->fileSize += real_size; continue; } - if (abs(rowHead->id) > oldId) { // not operated + if (abs(rowHead->version) > oldId) { // not operated pMetaRow = sdbGetRow(pTable, rowHead->data); if (pMetaRow == NULL) { // New object - if (rowHead->id < 0) { - sdbError("error sdb negative id:%d, sdb:%s, skip", rowHead->id, pTable->name); + if (rowHead->version < 0) { + sdbError("error sdb negative version:%d, sdb:%s, skip", rowHead->version, pTable->tableName); } else { - rowMeta.id = rowHead->id; + rowMeta.version = rowHead->version; // TODO:Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->size; + rowMeta.offset = pTable->fileSize; rowMeta.rowSize = rowHead->rowSize; rowMeta.row = (*pTable->decodeFp)(rowHead->data); (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); @@ -801,7 +833,7 @@ void sdbResetTable(SSdbTable *pTable) { (*pTable->insertFp)(rowMeta.row); } } else { // already exists - if (rowHead->id < 0) { // Delete the object + if (rowHead->version < 0) { // Delete the object (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); (*pTable->destroyFp)(pMetaRow); pTable->numOfRows--; @@ -811,38 +843,39 @@ void sdbResetTable(SSdbTable *pTable) { } } - pTable->size += real_size; - if (pTable->id < abs(rowHead->id)) pTable->id = abs(rowHead->id); + pTable->fileSize += real_size; + if (pTable->version < abs(rowHead->version)) pTable->version = abs(rowHead->version); } - sdbVersion += (pTable->id - oldId); + sdbVersion += (pTable->version - oldId); tfree(rowHead); - sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->name, sdbVersion, pTable->id); + sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->tableName, sdbVersion, pTable->version); + */ } // TODO:A problem here :use snapshot file to sync another node will cause problem void sdbSaveSnapShot(void *handle) { + /* SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; void * pNode = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - int32_t size = 0; - int32_t numOfRows = 0; + int32_t total_size = 0; + int32_t real_size = 0; + int32_t size = 0; + int32_t numOfRows = 0; uint32_t sdbEcommit = SDB_ENDCOMMIT; char * dirc = NULL; char * basec = NULL; - /* char action = SDB_TYPE_INSERT; */ if (pTable == NULL) return; - sdbTrace("Table:%s, save the snapshop", pTable->name); + sdbTrace("Table:%s, save the snapshop", pTable->tableName); char fn[128] = "\0"; - dirc = strdup(pTable->fn); - basec = strdup(pTable->fn); + dirc = strdup(pTable->fileName); + basec = strdup(pTable->fileName); sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); int32_t fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); tfree(dirc); @@ -851,7 +884,7 @@ void sdbSaveSnapShot(void *handle) { total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)malloc(total_size); if (rowHead == NULL) { - sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->name); + sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->tableName); return; } memset(rowHead, 0, size); @@ -867,36 +900,35 @@ void sdbSaveSnapShot(void *handle) { if (pMeta == NULL) break; rowHead->delimiter = SDB_DELIMITER; - rowHead->id = pMeta->id; + rowHead->version = pMeta->id; rowHead->rowSize = (*pTable->encodeFp)(pMeta->row, rowHead->data, pTable->maxRowSize); real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while save sdb %s snapshot", pTable->name); + sdbError("failed to get checksum while save sdb %s snapshot", pTable->tableName); tfree(rowHead); return; } - /* write(fd, &action, sizeof(action)); */ - /* size += sizeof(action); */ - twrite(fd, rowHead, real_size); - size += real_size; - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - numOfRows++; + twrite(fd, rowHead, real_size); + size += real_size; + twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); + size += sizeof(sdbEcommit); + numOfRows++; } tfree(rowHead); // Remove the old file tclose(pTable->fd); - remove(pTable->fn); + remove(pTable->fileName); // Rename the .sdb.db file to sdb.db file - rename(fn, pTable->fn); + rename(fn, pTable->fileName); pTable->fd = fd; - pTable->size = size; + pTable->fileSize = size; pTable->numOfRows = numOfRows; fdatasync(pTable->fd); + */ } void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { From 7546da5e544c2e1bd09669d119c2ecebe0b7f377 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 12:23:45 +0800 Subject: [PATCH 09/85] TD-34 --- src/vnode/tsdb/inc/tsdbCache.h | 14 +++++++++++--- src/vnode/tsdb/inc/tsdbFile.h | 5 +++++ src/vnode/tsdb/src/tsdbCache.c | 32 +++++++++++++++++++++----------- src/vnode/tsdb/src/tsdbFile.c | 6 +----- src/vnode/tsdb/src/tsdbMain.c | 21 +++++++++++++-------- 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbCache.h b/src/vnode/tsdb/inc/tsdbCache.h index 3bffa1c6a9..2676829c75 100644 --- a/src/vnode/tsdb/inc/tsdbCache.h +++ b/src/vnode/tsdb/inc/tsdbCache.h @@ -17,6 +17,7 @@ #include +#include "taosdef.h" #include "tlist.h" #ifdef __cplusplus @@ -38,18 +39,25 @@ typedef struct { SList * memPool; } STsdbCachePool; +typedef struct { + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfPoints; + SList * list; +} SCacheMem; + typedef struct { int maxBytes; int cacheBlockSize; STsdbCachePool pool; STsdbCacheBlock *curBlock; - SList * mem; - SList * imem; + SCacheMem * mem; + SCacheMem * imem; } STsdbCache; STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize); void tsdbFreeCache(STsdbCache *pCache); -void * tsdbAllocFromCache(STsdbCache *pCache, int bytes); +void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 89159a06e7..2324ead451 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -18,11 +18,15 @@ #include #include "taosdef.h" +#include "tglobalcfg.h" #ifdef __cplusplus extern "C" { #endif +#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) +#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) + typedef enum { TSDB_FILE_TYPE_HEAD = 0, // .head file type TSDB_FILE_TYPE_DATA, // .data file type @@ -66,6 +70,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32 void tsdbCloseFile(STsdbFileH *pFileH); int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables); +void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } #endif diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 6a0741dced..01f56bb5a0 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -17,7 +17,7 @@ #include "tsdbCache.h" static int tsdbAllocBlockFromPool(STsdbCache *pCache); -static void tsdbFreeBlockList(SList *list); +static void tsdbFreeBlockList(SCacheMem *mem); STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); @@ -46,11 +46,8 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { tdListAppend(pPool->memPool, (void *)(&pBlock)); } - pCache->mem = tdListNew(sizeof(STsdbCacheBlock *)); - if (pCache->mem == NULL) goto _err; - - pCache->imem = tdListNew(sizeof(STsdbCacheBlock *)); - if (pCache->imem == NULL) goto _err; + pCache->mem = NULL; + pCache->imem = NULL; return pCache; @@ -66,11 +63,20 @@ void tsdbFreeCache(STsdbCache *pCache) { free(pCache); } -void *tsdbAllocFromCache(STsdbCache *pCache, int bytes) { +void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { if (pCache == NULL) return NULL; if (bytes > pCache->cacheBlockSize) return NULL; - if (isListEmpty(pCache->mem)) { + if (pCache->mem == NULL) { // Create a new one + pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem)); + if (pCache->mem == NULL) return NULL; + pCache->mem->keyFirst = INT64_MAX; + pCache->mem->keyLast = 0; + pCache->mem->numOfPoints = 0; + pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *)); + } + + if (isListEmpty(pCache->mem->list)) { if (tsdbAllocBlockFromPool(pCache) < 0) { // TODO: deal with the error } @@ -86,12 +92,15 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes) { pCache->curBlock->offset += bytes; pCache->curBlock->remain -= bytes; memset(ptr, 0, bytes); + if (key < pCache->mem->keyFirst) pCache->mem->keyFirst = key; + if (key > pCache->mem->keyLast) pCache->mem->keyLast = key; return ptr; } -static void tsdbFreeBlockList(SList *list) { - if (list == NULL) return; +static void tsdbFreeBlockList(SCacheMem *mem) { + if (mem == NULL) return; + SList * list = mem->list; SListNode * node = NULL; STsdbCacheBlock *pBlock = NULL; while ((node = tdListPopHead(list)) != NULL) { @@ -100,6 +109,7 @@ static void tsdbFreeBlockList(SList *list) { listNodeFree(node); } tdListFree(list); + free(mem); } static int tsdbAllocBlockFromPool(STsdbCache *pCache) { @@ -114,7 +124,7 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache) { pBlock->offset = 0; pBlock->remain = pCache->cacheBlockSize; - tdListAppendNode(pCache->mem, node); + tdListAppendNode(pCache->mem->list, node); pCache->curBlock = pBlock; return 0; diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 8a7e40cabd..1c91c03b44 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -22,15 +22,11 @@ #include #include -#include "tglobalcfg.h" #include "tsdbFile.h" #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F -#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) -#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) - typedef struct { int32_t len; int32_t padding; // For padding purpose @@ -228,7 +224,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32 return pTsdbFileH; } -static void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, +void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index a02343f817..7c1f7efb8d 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -316,8 +316,9 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { pTable->mem = NULL; } } - // Loop to move mem to imem - tdListMove(pRepo->tsdbCache->mem, pRepo->tsdbCache->imem); + // TODO: Loop to move mem to imem + pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; + pRepo->tsdbCache->mem = NULL; pthread_create(&(pRepo->commitThread), NULL, tsdbCommitToFile, (void *)repo); pthread_mutex_unlock(&(pRepo->mutex)); @@ -678,8 +679,9 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListRandNodeInfo(pTable->mem->pData, &level, &headSize); + TSKEY key = dataRowKey(row); // Copy row into the memory - SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row)); + SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); if (pNode == NULL) { // TODO: deal with allocate failure } @@ -689,7 +691,6 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable // Insert the skiplist node into the data tSkipListPut(pTable->mem->pData, pNode); - TSKEY key = dataRowKey(row); if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; pTable->mem->numOfPoints++; @@ -716,20 +717,24 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } +// Commit to file static void *tsdbCommitToFile(void *arg) { // TODO STsdbRepo *pRepo = (STsdbRepo *)arg; STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { + for (int i = 0; i < pRepo->config.maxTables; i++) { // Loop over table STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - SSkipListIterator *pIter = tSkipListCreateIter(pTable->imem->pData); + if (pTable == NULL || pTable->imem == NULL) continue; + + SMemTable *pMem = pTable->imem; + SSkipListIterator *pIter = tSkipListCreateIter(pMem->pData); + // Loop to commit to file while (tSkipListIterNext(pIter)) { SSkipListNode *node = tSkipListIterGet(pIter); SDataRow row = SL_GET_NODE_DATA(node); int k = 0; - } + tSkipListDestroyIter(pIter); } return NULL; From ca7340121daf767ffb5fa47ea510b7c44d0161a3 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 12:26:19 +0800 Subject: [PATCH 10/85] TD-34 --- src/vnode/tsdb/src/tsdbCache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 01f56bb5a0..3b1d44e6c7 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -94,6 +94,7 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { memset(ptr, 0, bytes); if (key < pCache->mem->keyFirst) pCache->mem->keyFirst = key; if (key > pCache->mem->keyLast) pCache->mem->keyLast = key; + pCache->mem->numOfPoints++; return ptr; } From 41f8bf16357acbb5434a9c29948909d981d999a0 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 16:12:53 +0800 Subject: [PATCH 11/85] TD-34 --- src/vnode/tsdb/inc/tsdbMeta.h | 2 ++ src/vnode/tsdb/src/tsdbMain.c | 64 +++++++++++++++++++++++++++-------- src/vnode/tsdb/src/tsdbMeta.c | 2 ++ 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h index 03031f16cd..aba506f30d 100644 --- a/src/vnode/tsdb/inc/tsdbMeta.h +++ b/src/vnode/tsdb/inc/tsdbMeta.h @@ -74,6 +74,8 @@ typedef struct { void *map; // table map of (uid ===> table) SMetaFile *mfh; // meta file handle + int maxRowBytes; + int maxCols; } STsdbMeta; STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables); diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 7c1f7efb8d..3a4e427798 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -44,6 +44,7 @@ #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" +#define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7 enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING }; @@ -717,25 +718,60 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey, int maxRowsToRead, void *dst) { + int numOfRows = 0; + do { + SSkipListNode *node = tSkiplistIterGet(pIter); + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) > maxKey) break; + } while (tSkipListIterNext(pIter)); + return numOfRows; +} + // Commit to file static void *tsdbCommitToFile(void *arg) { // TODO - STsdbRepo *pRepo = (STsdbRepo *)arg; - STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { // Loop over table - STable *pTable = pMeta->tables[i]; - if (pTable == NULL || pTable->imem == NULL) continue; + STsdbRepo * pRepo = (STsdbRepo *)arg; + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbCache *pCache = pRepo->tsdbCache; + STsdbRepo * pCfg = &(pRepo->config); + if (pCache->imem == NULL) return; - SMemTable *pMem = pTable->imem; - SSkipListIterator *pIter = tSkipListCreateIter(pMem->pData); - // Loop to commit to file - while (tSkipListIterNext(pIter)) { - SSkipListNode *node = tSkipListIterGet(pIter); - SDataRow row = SL_GET_NODE_DATA(node); - int k = 0; - } - tSkipListDestroyIter(pIter); + int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst); + int efid = tsdbGetKeyFileId(pCache->imem->keyLast); + SSkipListIterator **iters = (SSkipListIterator **)calloc(pCfg->maxTables, sizeof(SSkipListIterator *)); + if (iters == NULL) { + // TODO: deal with the error + return NULL; } + for (int fid = sfid; fid <= efid; fid++) { + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + + for (int tid = 0; tid < pCfg->maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL || pTable->imem == NULL) continue; + if (iters[tid] == NULL) { // create table iterator + iters[tid] = tSkipListCreateIter(pTable->imem); + // TODO: deal with the error + if (iters[tid] == NULL) break; + if (!tSkipListIterNext(iters[tid])) { + // assert(0); + } + } + + // Loop the iterator + // tsdbReadRowsFromCache(); + } + } + + // Free the iterator + for (int tid = 0; tid < pCfg->maxTables; tid++) { + if (iters[tid] != NULL) tSkipListDestroyIter(iters[tid]); + } + + free(iters); + return NULL; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index c7874fa5a2..b8b5450d23 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -133,6 +133,8 @@ STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) { pMeta->nTables = 0; pMeta->superList = NULL; pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *)); + pMeta->maxRowBytes = 0; + pMeta->maxCols = 0; if (pMeta->tables == NULL) { free(pMeta); return NULL; From 3f5bf97c88ceca6c5e19be2ad69a3694326753f6 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 16:41:29 +0800 Subject: [PATCH 12/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 3a4e427798..08d5022122 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -721,7 +721,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey, int maxRowsToRead, void *dst) { int numOfRows = 0; do { - SSkipListNode *node = tSkiplistIterGet(pIter); + SSkipListNode *node = tSkipListIterGet(pIter); SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; } while (tSkipListIterNext(pIter)); @@ -734,11 +734,12 @@ static void *tsdbCommitToFile(void *arg) { STsdbRepo * pRepo = (STsdbRepo *)arg; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCache *pCache = pRepo->tsdbCache; - STsdbRepo * pCfg = &(pRepo->config); + STsdbCfg * pCfg = &(pRepo->config); if (pCache->imem == NULL) return; - int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst); - int efid = tsdbGetKeyFileId(pCache->imem->keyLast); + int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + SSkipListIterator **iters = (SSkipListIterator **)calloc(pCfg->maxTables, sizeof(SSkipListIterator *)); if (iters == NULL) { // TODO: deal with the error From a9cf4bd32e5fdffdda3ee840477d5618f72e34d9 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 17:05:55 +0800 Subject: [PATCH 13/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 08d5022122..cd673d2d89 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -718,12 +718,18 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } -static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey, int maxRowsToRead, void *dst) { +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, void *dst) { int numOfRows = 0; do { SSkipListNode *node = tSkipListIterGet(pIter); - SDataRow row = SL_GET_NODE_DATA(node); + if (node == NULL) break; + + SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; + // Convert row data to column data + + numOfRows++; + if (numOfRows > maxRowsToRead) break; } while (tSkipListIterNext(pIter)); return numOfRows; } @@ -746,6 +752,9 @@ static void *tsdbCommitToFile(void *arg) { return NULL; } + int maxCols = pMeta->maxCols; + int maxBytes = pMeta->maxRowBytes; + for (int fid = sfid; fid <= efid; fid++) { TSKEY minKey = 0, maxKey = 0; tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); @@ -754,7 +763,7 @@ static void *tsdbCommitToFile(void *arg) { STable *pTable = pMeta->tables[tid]; if (pTable == NULL || pTable->imem == NULL) continue; if (iters[tid] == NULL) { // create table iterator - iters[tid] = tSkipListCreateIter(pTable->imem); + iters[tid] = tSkipListCreateIter(pTable->imem->pData); // TODO: deal with the error if (iters[tid] == NULL) break; if (!tSkipListIterNext(iters[tid])) { @@ -763,7 +772,10 @@ static void *tsdbCommitToFile(void *arg) { } // Loop the iterator - // tsdbReadRowsFromCache(); + int rowsRead = 0; + while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, NULL)) > 0) { + int k = 0; + } } } From 987708acceb8bce1588195cf855b19e22a7bd0fd Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 21:06:02 +0800 Subject: [PATCH 14/85] TD-34 --- src/common/src/dataformat.c | 2 +- src/vnode/tsdb/src/tsdbMain.c | 20 ++++++++++++++++++-- src/vnode/tsdb/src/tsdbMeta.c | 4 ++++ src/vnode/tsdb/tests/tsdbTests.cpp | 6 +++--- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 9c356b0cbc..04826e43ac 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -150,7 +150,7 @@ void tdFreeSchema(STSchema *pSchema) { */ void tdUpdateSchema(STSchema *pSchema) { STColumn *pCol = NULL; - int32_t offset = 0; + int32_t offset = TD_DATA_ROW_HEAD_SIZE; for (int i = 0; i < schemaNCols(pSchema); i++) { pCol = schemaColAt(pSchema, i); colSetOffset(pCol, offset); diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index cd673d2d89..7da80cb50f 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -718,7 +718,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } -static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, void *dst) { +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCol **cols, STSchema *pSchema) { int numOfRows = 0; do { SSkipListNode *node = tSkipListIterGet(pIter); @@ -727,6 +727,11 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; // Convert row data to column data + // for (int i = 0; i < schemaNCols(pSchema); i++) { + // STColumn *pCol = schemaColAt(pSchema, i); + // memcpy(cols[i]->data + TYPE_BYTES[colType(pCol)] * numOfRows, dataRowAt(row, pCol->offset), + // TYPE_BYTES[colType(pCol)]); + // } numOfRows++; if (numOfRows > maxRowsToRead) break; @@ -754,6 +759,8 @@ static void *tsdbCommitToFile(void *arg) { int maxCols = pMeta->maxCols; int maxBytes = pMeta->maxRowBytes; + SDataCol **cols = (SDataCol **)malloc(sizeof(SDataCol *) * maxCols); + void *buf = malloc((maxBytes + sizeof(SDataCol)) * pCfg->maxRowsPerFileBlock); for (int fid = sfid; fid <= efid; fid++) { TSKEY minKey = 0, maxKey = 0; @@ -771,9 +778,16 @@ static void *tsdbCommitToFile(void *arg) { } } + // Init row data part + cols[0] = (SDataCol *)buf; + for (int col = 1; col < schemaNCols(pTable->schema); col++) { + cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); + } + // Loop the iterator int rowsRead = 0; - while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, NULL)) > 0) { + while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > + 0) { int k = 0; } } @@ -784,6 +798,8 @@ static void *tsdbCommitToFile(void *arg) { if (iters[tid] != NULL) tSkipListDestroyIter(iters[tid]); } + free(buf); + free(cols); free(iters); return NULL; diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index b8b5450d23..72c1667fe1 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -236,6 +236,10 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->type = TSDB_NORMAL_TABLE; table->superUid = -1; table->schema = tdDupSchema(pCfg->schema); + if (schemaNCols(table->schema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(table->schema); + tdUpdateSchema(table->schema); + int bytes = tdMaxRowBytesFromSchema(table->schema); + if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes; } // Register to meta diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index de58d6337c..8895258b1a 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -6,7 +6,7 @@ #include "tsdbFile.h" #include "tsdbMeta.h" -TEST(TsdbTest, tableEncodeDecode) { +TEST(TsdbTest, DISABLED_tableEncodeDecode) { STable *pTable = (STable *)malloc(sizeof(STable)); pTable->type = TSDB_NORMAL_TABLE; @@ -106,12 +106,12 @@ TEST(TsdbTest, createRepo) { } -TEST(TsdbTest, openRepo) { +TEST(TsdbTest, DISABLED_openRepo) { tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0"); ASSERT_NE(pRepo, nullptr); } -TEST(TsdbTest, createFileGroup) { +TEST(TsdbTest, DISABLED_createFileGroup) { SFileGroup fGroup; ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0); From ed7d5fe85019cd8c0d201af31356ac8fc8a637f5 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 22 Mar 2020 22:35:10 +0800 Subject: [PATCH 15/85] TD-34 --- src/vnode/tsdb/inc/tsdb.h | 2 ++ src/vnode/tsdb/inc/tsdbCache.h | 5 ++- src/vnode/tsdb/src/tsdbCache.c | 41 +++++++++++++---------- src/vnode/tsdb/src/tsdbMain.c | 52 +++++++++++++++++++++++++----- src/vnode/tsdb/tests/tsdbTests.cpp | 24 +++++++++++--- 5 files changed, 94 insertions(+), 30 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 4964ac673f..1368515cfd 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -59,6 +59,8 @@ tsdb_repo_t * tsdbOpenRepo(char *tsdbDir); int32_t tsdbCloseRepo(tsdb_repo_t *repo); int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg); int32_t tsdbTriggerCommit(tsdb_repo_t *repo); +int32_t tsdbLockRepo(tsdb_repo_t *repo); +int32_t tsdbUnLockRepo(tsdb_repo_t *repo); // --------- TSDB TABLE DEFINITION typedef struct { diff --git a/src/vnode/tsdb/inc/tsdbCache.h b/src/vnode/tsdb/inc/tsdbCache.h index 2676829c75..3e9eabc90d 100644 --- a/src/vnode/tsdb/inc/tsdbCache.h +++ b/src/vnode/tsdb/inc/tsdbCache.h @@ -19,6 +19,7 @@ #include "taosdef.h" #include "tlist.h" +#include "tsdb.h" #ifdef __cplusplus extern "C" { @@ -49,13 +50,15 @@ typedef struct { typedef struct { int maxBytes; int cacheBlockSize; + int totalCacheBlocks; STsdbCachePool pool; STsdbCacheBlock *curBlock; SCacheMem * mem; SCacheMem * imem; + tsdb_repo_t * pRepo; } STsdbCache; -STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize); +STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo); void tsdbFreeCache(STsdbCache *pCache); void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key); diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 3b1d44e6c7..f51c7c12d4 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -14,12 +14,13 @@ */ #include +#include "tsdb.h" #include "tsdbCache.h" static int tsdbAllocBlockFromPool(STsdbCache *pCache); static void tsdbFreeBlockList(SCacheMem *mem); -STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { +STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo) { STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); if (pCache == NULL) return NULL; @@ -27,9 +28,11 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize) { pCache->maxBytes = maxBytes; pCache->cacheBlockSize = cacheBlockSize; + pCache->pRepo = pRepo; int nBlocks = maxBytes / cacheBlockSize + 1; if (nBlocks <= 1) nBlocks = 2; + pCache->totalCacheBlocks = nBlocks; STsdbCachePool *pPool = &(pCache->pool); pPool->index = 0; @@ -67,22 +70,10 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { if (pCache == NULL) return NULL; if (bytes > pCache->cacheBlockSize) return NULL; - if (pCache->mem == NULL) { // Create a new one - pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem)); - if (pCache->mem == NULL) return NULL; - pCache->mem->keyFirst = INT64_MAX; - pCache->mem->keyLast = 0; - pCache->mem->numOfPoints = 0; - pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *)); - } - - if (isListEmpty(pCache->mem->list)) { - if (tsdbAllocBlockFromPool(pCache) < 0) { - // TODO: deal with the error + if (pCache->curBlock == NULL || pCache->curBlock->remain < bytes) { + if (pCache->curBlock !=NULL && (pCache->mem->list) >= pCache->totalCacheBlocks/2) { + tsdbTriggerCommit(pCache->pRepo); } - } - - if (pCache->curBlock->remain < bytes) { if (tsdbAllocBlockFromPool(pCache) < 0) { // TODO: deal with the error } @@ -115,7 +106,12 @@ static void tsdbFreeBlockList(SCacheMem *mem) { static int tsdbAllocBlockFromPool(STsdbCache *pCache) { STsdbCachePool *pPool = &(pCache->pool); - if (listNEles(pPool->memPool) == 0) return -1; + + tsdbLockRepo(pCache->pRepo); + if (listNEles(pPool->memPool) == 0) { + tsdbUnLockRepo(pCache->pRepo); + return -1; + } SListNode *node = tdListPopHead(pPool->memPool); @@ -125,8 +121,19 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache) { pBlock->offset = 0; pBlock->remain = pCache->cacheBlockSize; + if (pCache->mem == NULL) { // Create a new one + pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem)); + if (pCache->mem == NULL) return NULL; + pCache->mem->keyFirst = INT64_MAX; + pCache->mem->keyLast = 0; + pCache->mem->numOfPoints = 0; + pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *)); + } + tdListAppendNode(pCache->mem->list, node); pCache->curBlock = pBlock; + tsdbUnLockRepo(pCache->pRepo); + return 0; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 7da80cb50f..a8a80dd164 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -150,6 +150,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO pRepo->rootDir = strdup(rootDir); pRepo->config = *pCfg; pRepo->limiter = limiter; + pthread_mutex_init(&pRepo->mutex, NULL); // Create the environment files and directories if (tsdbSetRepoEnv(pRepo) < 0) { @@ -168,7 +169,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO pRepo->tsdbMeta = pMeta; // Initialize cache - STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize, -1); + STsdbCache *pCache = tsdbInitCache(pCfg->maxCacheSize, -1, (tsdb_repo_t *)pRepo); if (pCache == NULL) { free(pRepo->rootDir); tsdbFreeMeta(pRepo->tsdbMeta); @@ -249,7 +250,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1); + pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (tsdb_repo_t *)pRepo); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir); @@ -305,9 +306,12 @@ int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) { int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; - - if (pthread_mutex_lock(&(pRepo->mutex)) < 0) return -1; - if (pRepo->commit) return 0; + + tsdbLockRepo(repo); + if (pRepo->commit) { + tsdbUnLockRepo(repo); + return -1; + } pRepo->commit = 1; // Loop to move pData to iData for (int i = 0; i < pRepo->config.maxTables; i++) { @@ -320,15 +324,25 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { // TODO: Loop to move mem to imem pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; pRepo->tsdbCache->mem = NULL; + pRepo->tsdbCache->curBlock = NULL; + // TODO: here should set as detached or use join for memory leak pthread_create(&(pRepo->commitThread), NULL, tsdbCommitToFile, (void *)repo); - pthread_mutex_unlock(&(pRepo->mutex)); - - pthread_join(pRepo->commitThread, NULL); + tsdbUnLockRepo(repo); return 0; } +int32_t tsdbLockRepo(tsdb_repo_t *repo) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + return pthread_mutex_lock(repo); +} + +int32_t tsdbUnLockRepo(tsdb_repo_t *repo) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + return pthread_mutex_unlock(repo); +} + /** * Get the TSDB repository information, including some statistics * @param pRepo the TSDB repository handle @@ -691,6 +705,13 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable dataRowCpy(SL_GET_NODE_DATA(pNode), row); // Insert the skiplist node into the data + if (pTable->mem == NULL) { + pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); + if (pTable->mem == NULL) return -1; + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); + pTable->mem->keyFirst = INT64_MAX; + pTable->mem->keyLast = 0; + } tSkipListPut(pTable->mem->pData, pNode); if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; @@ -788,6 +809,7 @@ static void *tsdbCommitToFile(void *arg) { int rowsRead = 0; while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > 0) { + // printf("rowsRead:%d-----------\n", rowsRead); int k = 0; } } @@ -802,5 +824,19 @@ static void *tsdbCommitToFile(void *arg) { free(cols); free(iters); + tsdbLockRepo(arg); + tdListMove(pCache->imem->list, pCache->pool.memPool); + free(pCache->imem); + pCache->imem = NULL; + pRepo->commit = 0; + // TODO: free the skiplist + for (int i = 0; i < pCfg->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->imem) { // Here has memory leak + pTable->imem = NULL; + } + } + tsdbUnLockRepo(arg); + return NULL; } \ No newline at end of file diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 8895258b1a..459d531c10 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -1,11 +1,18 @@ #include #include +#include #include "tsdb.h" #include "dataformat.h" #include "tsdbFile.h" #include "tsdbMeta.h" +double getCurTime() { + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec + tv.tv_usec * 1E-6; +} + TEST(TsdbTest, DISABLED_tableEncodeDecode) { STable *pTable = (STable *)malloc(sizeof(STable)); @@ -71,19 +78,22 @@ TEST(TsdbTest, createRepo) { tsdbCreateTable(pRepo, &tCfg); // // 3. Loop to write some simple data - int nRows = 1000; - int rowsPerSubmit = 10; + int nRows = 10000000; + int rowsPerSubmit = 100; int64_t start_time = 1584081000000; SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); + double stime = getCurTime(); + for (int k = 0; k < nRows/rowsPerSubmit; k++) { SSubmitBlk *pBlock = pMsg->blocks; pBlock->tableId = {.uid = 987607499877672L, .tid = 0}; pBlock->sversion = 0; pBlock->len = 0; for (int i = 0; i < rowsPerSubmit; i++) { - start_time += 1000; + // start_time += 1000; + start_time -= 1000; SDataRow row = (SDataRow)(pBlock->data + pBlock->len); tdInitDataRow(row, schema); @@ -102,7 +112,13 @@ TEST(TsdbTest, createRepo) { tsdbInsertData(pRepo, pMsg); } - tsdbTriggerCommit(pRepo); + double etime = getCurTime(); + + printf("Spent %f seconds to write %d records\n", etime - stime, nRows); + + + + // tsdbTriggerCommit(pRepo); } From 2e3e7f324035683a388680f8d1a6d3cdd3c83622 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 00:05:02 +0800 Subject: [PATCH 16/85] [TD-15] fix sdb error while delete user --- src/mnode/inc/mgmtSdb.h | 6 +-- src/mnode/src/mgmtSdb.c | 64 +++++++++++------------ src/mnode/src/mgmtUser.c | 110 +++++++++++++++++++++------------------ 3 files changed, 93 insertions(+), 87 deletions(-) diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index e33bfce824..ccddc11e04 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -32,12 +32,12 @@ typedef enum { typedef struct { ESdbOperType type; + void * table; + void * pObj; + int64_t version; int32_t maxRowSize; int32_t rowSize; void * rowData; - void * pObj; - void * table; - int64_t version; } SSdbOperDesc; typedef struct { diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index f957dc4a83..ea05cb59f4 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -290,6 +290,8 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { maxAutoIndex = MAX(maxAutoIndex, *(int32_t *) rowHead->data); } + pTable->version = MAX(pTable->version, abs(rowHead->version)); + void *pMetaRow = sdbGetRow(pTable, rowHead->data); if (pMetaRow == NULL) { if (rowHead->version < 0) { @@ -310,11 +312,11 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); pTable->numOfRows++; - sdbTrace("table:%s, read record:%s and insert, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbTrace("table:%s, read new record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } } } else { @@ -326,8 +328,8 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { (*pTable->destroyFp)(&oper); (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); pTable->numOfRows--; - sdbTrace("table:%s, read record:%s and delete, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbTrace("table:%s, read deleted record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } else { SRowMeta rowMeta; rowMeta.version = rowHead->version; @@ -347,17 +349,14 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { if (code == TSDB_CODE_SUCCESS) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - sdbTrace("table:%s, read record:%s and update, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbTrace("table:%s, read updated record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version, sdbVersion); + sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , + pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); } } numOfChanged++; - if (pTable->version < abs(rowHead->version)) { - pTable->version = abs(rowHead->version); - } } pTable->fileSize += real_size; @@ -424,7 +423,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->tableId = sdbNumOfTables++; sdbTableList[pTable->tableId] = pTable; - sdbTrace("table:%s is initialized, numOfRows:%d, numOfTables:%d, version:%" PRId64 " sdbversion:%" PRId64, + sdbTrace("table:%s, is initialized, numOfRows:%d numOfTables:%d version:%" PRId64 " sdbversion:%" PRId64, pTable->tableName, pTable->numOfRows, sdbNumOfTables, pTable->version, sdbVersion); return pTable; @@ -470,6 +469,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { return TSDB_CODE_ALREADY_THERE; } + pOper->maxRowSize = pTable->maxRowSize; pthread_mutex_lock(&pTable->mutex); if (pOper->type == SDB_OPER_TYPE_GLOBAL) { @@ -507,12 +507,11 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pOper->rowData = rowHead->data; (*pTable->encodeFp)(pOper); rowHead->rowSize = pOper->rowSize; - + rowHead->delimiter = SDB_DELIMITER; + rowHead->version = pTable->version; assert(rowHead->rowSize > 0 && rowHead->rowSize <= pTable->maxRowSize); int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pTable->version; if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { sdbError("table:%s, failed to get checksum while inserting", pTable->tableName); pTable->version--; @@ -540,7 +539,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); sdbTrace("table:%s, a record is inserted:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pOper->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); (*pTable->insertFp)(pOper); @@ -568,11 +567,11 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { .type = SDB_FORWARD_TYPE_DELETE, .tableId = pTable->tableId, .version = pTable->version + 1, - .rowSize = pOper->rowSize, - .rowData = pOper->rowData, + .rowSize = pMeta->rowSize, + .rowData = pMeta->row, }; - if (sdbForwardDbReqToPeer(&forward) == 0) { + if (sdbForwardDbReqToPeer(&forward) != 0) { sdbError("table:%s, failed to delete record", pTable->tableName); pthread_mutex_unlock(&pTable->mutex); return -1; @@ -593,7 +592,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { int32_t rowSize = 0; switch (pTable->keyType) { case SDB_KEY_TYPE_STRING: - rowSize = strlen((char *)pOper->rowData) + 1; + rowSize = strlen((char *)pOper->pObj) + 1; break; case SDB_KEY_TYPE_AUTO: rowSize = sizeof(uint64_t); @@ -605,8 +604,9 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { rowHead->delimiter = SDB_DELIMITER; rowHead->rowSize = rowSize; rowHead->version = -(pTable->version); - memcpy(rowHead->data, pOper->rowData, pOper->rowSize); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, total_size) < 0) { + memcpy(rowHead->data, pOper->pObj, rowSize); + int32_t real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); + if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { sdbError("failed to get checksum while inserting, sdb:%s", pTable->tableName); pTable->version--; sdbVersion--; @@ -615,14 +615,14 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { return -1; } - twrite(pTable->fd, rowHead, total_size); - pTable->fileSize += total_size; + twrite(pTable->fd, rowHead, real_size); + pTable->fileSize += real_size; sdbFinishCommit(pTable); tfree(rowHead); sdbTrace("table:%s, a record is deleted:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%d", - pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); // Delete from current layer (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); @@ -642,10 +642,10 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->rowData); + SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " id:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); return -1; } @@ -658,11 +658,11 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SForwardMsg forward = { .type = SDB_FORWARD_TYPE_UPDATE, .tableId = pTable->tableId, - .version = pOper->version + 1, + .version = pTable->version + 1, .rowSize = pOper->rowSize, .rowData = pOper->rowData, }; - if (sdbForwardDbReqToPeer(&forward) == 0) { + if (sdbForwardDbReqToPeer(&forward) != 0) { sdbError("table:%s, failed to update record", pTable->tableName); pthread_mutex_unlock(&pTable->mutex); return -1; @@ -710,7 +710,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { sdbFinishCommit(pTable); sdbTrace("table:%s, a record is updated:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->rowData), sdbVersion, pTable->version, pTable->numOfRows); + pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); pMeta->version = pTable->version; pMeta->offset = pTable->fileSize; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 6e72a6c8c2..11285764b8 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -38,23 +38,20 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); -static int32_t mgmtUserActionDestroy(void *pObj) { - tfree(pObj); +static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionInsert(void *pObj) { - SUserObj *pUser = (SUserObj *) pObj; +static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - - pUser->pAcct = pAcct; mgmtAddUserIntoAcct(pAcct, pUser); - return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionDelete(void *pObj) { - SUserObj *pUser = (SUserObj *) pObj; +static int32_t mgmtUserActionDelete(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pUser->acct); mgmtRemoveUserFromAcct(pAcct, pUser); @@ -62,36 +59,32 @@ static int32_t mgmtUserActionDelete(void *pObj) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionUpdate(void *pObj) { +static int32_t mgmtUserActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtUserActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SUserObj *pUser = (SUserObj *) pObj; +static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { + SUserObj *pUser = pOper->pObj; - if (maxRowSize < tsUserUpdateSize) { + if (pOper->maxRowSize < tsUserUpdateSize) { return -1; } else { - memcpy(pData, pUser, tsUserUpdateSize); - return tsUserUpdateSize; + memcpy(pOper->rowData, pUser, tsUserUpdateSize); + pOper->rowSize = tsUserUpdateSize; + return TSDB_CODE_SUCCESS; } } -static void *mgmtUserActionDecode(void *pData) { - SUserObj *pUser = (SUserObj *) malloc(sizeof(SUserObj)); - if (pUser == NULL) return NULL; - memset(pUser, 0, sizeof(SUserObj)); - memcpy(pUser, pData, tsUserUpdateSize); +static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { + SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj)); + if (pUser == NULL) return -1; - return pUser; + memcpy(pUser, pOper->rowData, tsUserUpdateSize); + pOper->pObj = pUser; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitUsers() { - void *pNode = NULL; - SUserObj *pUser = NULL; - SAcctObj *pAcct = NULL; - int32_t numOfUsers = 0; - SUserObj tObj; tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; @@ -99,7 +92,7 @@ int32_t mgmtInitUsers() { .tableName = "users", .hashSessions = TSDB_MAX_USERS, .maxRowSize = tsUserUpdateSize, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtUserActionInsert, .deleteFp = mgmtUserActionDelete, .updateFp = mgmtUserActionUpdate, @@ -114,20 +107,7 @@ int32_t mgmtInitUsers() { return -1; } - while (1) { - pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser); - if (pUser == NULL) break; - - pUser->prev = NULL; - pUser->next = NULL; - - pAcct = mgmtGetAcct(pUser->acct); - mgmtAddUserIntoAcct(pAcct, pUser); - - numOfUsers++; - } - - pAcct = mgmtGetAcct("root"); + SAcctObj *pAcct = mgmtGetAcct("root"); mgmtCreateUser(pAcct, "root", "taosdata"); mgmtCreateUser(pAcct, "monitor", tsInternalPass); mgmtCreateUser(pAcct, "_root", tsInternalPass); @@ -151,7 +131,20 @@ SUserObj *mgmtGetUser(char *name) { } static int32_t mgmtUpdateUser(SUserObj *pUser) { - return sdbUpdateRow(tsUserSdb, pUser, tsUserUpdateSize, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = tsUserUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pUser); + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { @@ -186,8 +179,15 @@ static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass) { pUser->superAuth = 1; } - code = TSDB_CODE_SUCCESS; - if (sdbInsertRow(tsUserSdb, pUser, SDB_OPER_GLOBAL) < 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pUser); code = TSDB_CODE_SDB_ERROR; } @@ -208,9 +208,19 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { return TSDB_CODE_NO_RIGHTS; } - sdbDeleteRow(tsUserSdb, pUser, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsUserSdb, + .pObj = pUser + }; - return 0; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pUser); + code = TSDB_CODE_SDB_ERROR; + } + + return code; } static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { @@ -359,7 +369,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t*)pAlter->pass, strlen(pAlter->pass), pUser->pass); code = mgmtUpdateUser(pUser); - mLPrint("user:%s password is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s password is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } @@ -394,10 +404,6 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } if (hasRight) { - //if (pAlter->privilege == 1) { // super - // pUser->superAuth = 1; - // pUser->writeAuth = 1; - //} if (pAlter->privilege == 2) { // read pUser->superAuth = 0; pUser->writeAuth = 0; @@ -408,7 +414,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { } code = mgmtUpdateUser(pUser); - mLPrint("user:%s privilege is altered by %s, code:%d", pAlter->user, pUser->user, code); + mLPrint("user:%s privilege is altered by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } else { code = TSDB_CODE_NO_RIGHTS; } From 74b546108c424bc662868dd87eac54b05b5cfbad Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 23 Mar 2020 02:29:14 +0800 Subject: [PATCH 17/85] [td-32] fix bugs in insertion and retrieve data --- src/client/src/tscParseInsert.c | 34 +-- src/client/src/tscPrepare.c | 16 +- src/client/src/tscSchemaUtil.c | 2 +- src/client/src/tscServer.c | 28 +-- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 69 +++++- src/dnode/src/dnodeRead.c | 24 +- src/dnode/src/dnodeWrite.c | 6 +- src/inc/taosmsg.h | 56 +++-- src/query/inc/queryExecutor.h | 3 +- src/query/src/queryExecutor.c | 401 ++++++++++++++++---------------- src/util/src/tskiplist.c | 24 +- src/vnode/tsdb/inc/tsdb.h | 20 +- src/vnode/tsdb/inc/tsdbFile.h | 20 ++ src/vnode/tsdb/inc/tsdbMeta.h | 4 +- src/vnode/tsdb/src/tsdbFile.c | 20 -- src/vnode/tsdb/src/tsdbMain.c | 28 ++- src/vnode/tsdb/src/tsdbRead.c | 377 +++++++++++++++++++++++++++--- 18 files changed, 779 insertions(+), 355 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 8fb6b925ef..ccd734ec7b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -18,9 +18,10 @@ #define _XOPEN_SOURCE -#include "hash.h" #include "os.h" -#include "tscSecondaryMerge.h" + +#include "hash.h" +//#include "tscSecondaryMerge.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" @@ -32,6 +33,8 @@ #include "tstoken.h" #include "ttime.h" +#include "dataformat.h" + enum { TSDB_USE_SERVER_TS = 0, TSDB_USE_CLI_TS = 1, @@ -393,7 +396,6 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - // bool isPrevOptr; //fang, never used SSQLToken sToken = {0}; char * payload = pDataBlocks->pData + pDataBlocks->size; @@ -604,8 +606,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 return TSDB_CODE_SUCCESS; } -static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { - pBlocks->sid = pTableMeta->sid; +static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { + pBlocks->tid = pTableMeta->sid; pBlocks->uid = pTableMeta->uid; pBlocks->sversion = pTableMeta->sversion; pBlocks->numOfRows += numOfRows; @@ -613,10 +615,10 @@ static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableM // data block is disordered, sort it in ascending order void sortRemoveDuplicates(STableDataBlocks *dataBuf) { - SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)dataBuf->pData; + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. - assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SShellSubmitBlock) == dataBuf->size); + assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); // if use server time, this block must be ordered if (dataBuf->tsSource == TSDB_USE_SERVER_TS) { @@ -624,7 +626,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) { } if (!dataBuf->ordered) { - char *pBlockData = pBlocks->payLoad; + char *pBlockData = pBlocks->data; qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); int32_t i = 0; @@ -650,7 +652,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) { dataBuf->ordered = true; pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SShellSubmitBlock) + dataBuf->rowSize * pBlocks->numOfRows; + dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; } } @@ -663,7 +665,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char STableDataBlocks *dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pTableHashList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SShellSubmitBlock), tinfo.rowSize, pTableMetaInfo->name, + sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &dataBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -691,11 +693,11 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char SParamInfo *param = dataBuf->params + i; if (param->idx == -1) { param->idx = pCmd->numOfParams++; - param->offset -= sizeof(SShellSubmitBlock); + param->offset -= sizeof(SSubmitBlk); } } - SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(dataBuf->pData); + SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); dataBuf->vgId = pTableMeta->vgId; @@ -1141,7 +1143,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { STableDataBlocks *pDataBlock = NULL; STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SShellSubmitBlock), pTableMetaInfo->name, + int32_t ret = tscCreateDataBlock(PATH_MAX, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pDataBlock); if (ret != TSDB_CODE_SUCCESS) { goto _error_clean; @@ -1353,7 +1355,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock assert(pCmd->numOfClause == 1); STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; - SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(pTableDataBlocks->pData); + SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { @@ -1394,7 +1396,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { pCmd->pDataBlocks = tscCreateBlockArrayList(); STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SShellSubmitBlock), + int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock); if (ret != TSDB_CODE_SUCCESS) { return -1; @@ -1435,7 +1437,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { } pTableDataBlock = pCmd->pDataBlocks->pData[0]; - pTableDataBlock->size = sizeof(SShellSubmitBlock); + pTableDataBlock->size = sizeof(SSubmitBlk); pTableDataBlock->rowSize = tinfo.rowSize; numOfRows += pSql->res.numOfRows; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 96215ce73c..241f24a747 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -325,12 +325,12 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; - uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); + uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); uint32_t dataSize = totalDataSize / alloced; assert(dataSize * alloced == totalDataSize); if (alloced == binded) { - totalDataSize += dataSize + sizeof(SShellSubmitBlock); + totalDataSize += dataSize + sizeof(SSubmitBlk); if (totalDataSize > pBlock->nAllocSize) { const double factor = 1.5; void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); @@ -342,7 +342,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { } } - char* data = pBlock->pData + sizeof(SShellSubmitBlock) + dataSize * binded; + char* data = pBlock->pData + sizeof(SSubmitBlk) + dataSize * binded; for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { SParamInfo* param = pBlock->params + j; int code = doBindParam(data, param, bind + param->idx); @@ -365,10 +365,10 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; - uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); + uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); pBlock->size += totalDataSize / alloced; - SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData; + SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; pSubmit->numOfRows += pSubmit->numOfRows / alloced; } @@ -398,10 +398,10 @@ static int insertStmtReset(STscStmt* pStmt) { for (int32_t i = 0; i < pCmd->pDataBlocks->nSize; ++i) { STableDataBlocks* pBlock = pCmd->pDataBlocks->pData[i]; - uint32_t totalDataSize = pBlock->size - sizeof(SShellSubmitBlock); - pBlock->size = sizeof(SShellSubmitBlock) + totalDataSize / alloced; + uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); + pBlock->size = sizeof(SSubmitBlk) + totalDataSize / alloced; - SShellSubmitBlock* pSubmit = (SShellSubmitBlock*)pBlock->pData; + SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; pSubmit->numOfRows = pSubmit->numOfRows / alloced; } } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index be0065ff4a..78d29be1af 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -58,7 +58,7 @@ SSchema *tscGetTableSchema(const STableMeta *pTableMeta) { return pSTableMeta->schema; } - return pTableMeta->schema; + return (SSchema*) pTableMeta->schema; } SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8fb3df94d5..58fc32c824 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -508,14 +508,17 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pRetrieveMsg->free = htons(pQueryInfo->type); pMsg += sizeof(pQueryInfo->type); - pSql->cmd.payloadLen = pMsg - pStart; + pRetrieveMsg->header.vgId = htonl(1); + pMsg += sizeof(SRetrieveTableMsg); + + pRetrieveMsg->header.contLen = htonl(pSql->cmd.payloadLen); + pSql->cmd.msgType = TSDB_MSG_TYPE_RETRIEVE; - return TSDB_CODE_SUCCESS; } void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) { - //SShellSubmitMsg *pShellMsg; + //SSubmitMsg *pShellMsg; //char * pMsg; //STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); @@ -524,14 +527,14 @@ void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) { //pMsg = buf + tsRpcHeadSize; //TODO set iplist - //pShellMsg = (SShellSubmitMsg *)pMsg; + //pShellMsg = (SSubmitMsg *)pMsg; //pShellMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); //tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pSql->index].ip), // htons(pShellMsg->vnode)); } int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SShellSubmitMsg *pShellMsg; + SSubmitMsg *pShellMsg; char * pMsg, *pStart; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); @@ -539,24 +542,23 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pStart = pSql->cmd.payload + tsRpcHeadSize; pMsg = pStart; - - pShellMsg = (SShellSubmitMsg *)pMsg; - pShellMsg->desc.numOfVnodes = htonl(1); + SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; + pMsgDesc->numOfVnodes = htonl(1); //set the number of vnodes + pMsg += sizeof(SMsgDesc); - pShellMsg->import = htons(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT) ? 0 : 1); + pShellMsg = (SSubmitMsg *)pMsg; pShellMsg->header.vgId = htonl(pTableMeta->vgId); pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen); + pShellMsg->length = pShellMsg->header.contLen; - pShellMsg->numOfTables = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted + pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + // tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip), // htons(pShellMsg->vnode)); - -// pSql->cmd.payloadLen = sizeof(SShellSubmitMsg); - return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 5d93e44c77..63612d0f5f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -147,7 +147,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } // tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid - tsInsertHeadSize = tsRpcHeadSize + sizeof(SShellSubmitMsg); + tsInsertHeadSize = tsRpcHeadSize + sizeof(SMsgDesc) + sizeof(SSubmitMsg); return pObj; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index aa4e5c93aa..2a9673b192 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -695,6 +695,49 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, return TSDB_CODE_SUCCESS; } +static void trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { + int32_t firstPartLen = 0; + + STableMeta* pTableMeta = pTableDataBlock->pTableMeta; + STableComInfo tinfo = tscGetTableInfo(pTableMeta); + SSchema* pSchema = tscGetTableSchema(pTableMeta); + + memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); + pDataBlock += sizeof(SSubmitBlk); + + int32_t total = sizeof(int32_t)*2; + for(int32_t i = 0; i < tinfo.numOfColumns; ++i) { + switch (pSchema[i].type) { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_BINARY: { + assert(0); // not support binary yet + firstPartLen += sizeof(int32_t);break; + } + default: + firstPartLen += tDataTypeDesc[pSchema[i].type].nSize; + total += tDataTypeDesc[pSchema[i].type].nSize; + } + } + + char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); + + SSubmitBlk* pBlock = pTableDataBlock->pData; + int32_t rows = htons(pBlock->numOfRows); + + for(int32_t i = 0; i < rows; ++i) { + *(int32_t*) pDataBlock = total; + pDataBlock += sizeof(int32_t); + + *(int32_t*) pDataBlock = firstPartLen; + pDataBlock += sizeof(int32_t); + + memcpy(pDataBlock, p, pTableDataBlock->rowSize); + + p += pTableDataBlock->rowSize; + pDataBlock += pTableDataBlock->rowSize; + } +} + int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) { SSqlCmd* pCmd = &pSql->cmd; @@ -716,7 +759,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi return ret; } - int64_t destSize = dataBuf->size + pOneTableBlock->size; + int64_t destSize = dataBuf->size + pOneTableBlock->size + pOneTableBlock->size*sizeof(int32_t)*2; if (dataBuf->nAllocSize < destSize) { while (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = dataBuf->nAllocSize * 1.5; @@ -730,29 +773,33 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi tscError("%p failed to allocate memory for merging submit block, size:%d", pSql, dataBuf->nAllocSize); taosHashCleanup(pVnodeDataBlockHashList); - tfree(dataBuf->pData); tscDestroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); return TSDB_CODE_CLI_OUT_OF_MEMORY; } } - SShellSubmitBlock* pBlocks = (SShellSubmitBlock*)pOneTableBlock->pData; + SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; sortRemoveDuplicates(pOneTableBlock); - char* e = (char*)pBlocks->payLoad + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); + char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, pBlocks->sid, - pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->payLoad), GET_INT64_VAL(e)); + tscTrace("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, + pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(e)); - pBlocks->sid = htonl(pBlocks->sid); + int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + sizeof(int32_t) * 2); + + pBlocks->tid = htonl(pBlocks->tid); pBlocks->uid = htobe64(pBlocks->uid); pBlocks->sversion = htonl(pBlocks->sversion); pBlocks->numOfRows = htons(pBlocks->numOfRows); - - memcpy(dataBuf->pData + dataBuf->size, pOneTableBlock->pData, pOneTableBlock->size); - - dataBuf->size += pOneTableBlock->size; + + pBlocks->len = htonl(len); + + // erase the empty space reserved for binary data + trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock); + dataBuf->size += (len + sizeof(SSubmitBlk)); dataBuf->numOfTables += 1; } diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index b511a6bf08..887845b00e 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -77,7 +77,9 @@ void dnodeRead(SRpcMsg *pMsg) { int32_t leftLen = pMsg->contLen; char *pCont = (char *) pMsg->pCont; SRpcContext *pRpcContext = NULL; - + + dTrace("dnode read msg disposal"); + // SMsgDesc *pDesc = pCont; // pDesc->numOfVnodes = htonl(pDesc->numOfVnodes); // pCont += sizeof(SMsgDesc); @@ -229,7 +231,8 @@ static void dnodeProcessQueryMsg(SReadMsg *pMsg) { SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont; SQInfo* pQInfo = NULL; - int32_t code = qCreateQueryInfo(pQueryTableMsg, &pQInfo); + void* tsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; @@ -243,17 +246,17 @@ static void dnodeProcessQueryMsg(SReadMsg *pMsg) { .msgType = 0 }; + rpcSendResponse(&rpcRsp); + // do execute query qTableQuery(pQInfo); - - rpcSendResponse(&rpcRsp); } static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { SRetrieveTableMsg *pRetrieve = pMsg->pCont; void *pQInfo = htobe64(pRetrieve->qhandle); - dTrace("retrieve msg is disposed, qInfo:%p", pQInfo); + dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId); int32_t rowSize = 0; int32_t numOfRows = 0; @@ -284,11 +287,12 @@ static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { contLen = 100; SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - pRsp->numOfRows = 0; - pRsp->precision = 0; - pRsp->offset = 0; - pRsp->useconds = 0; - + pRsp->numOfRows = htonl(1); + pRsp->precision = htons(0); + pRsp->offset = htobe64(0); + pRsp->useconds = htobe64(0); + + // todo set the data *(int64_t*) pRsp->data = 1000; rpcRsp = (SRpcMsg) { diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index 7ed731c953..eb9c42a93f 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -276,7 +276,10 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { pRsp->affectedRows = htonl(1); pRsp->numOfFailedBlocks = 0; - // todo write to tsdb + void* tsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + assert(tsdb != NULL); + + tsdbInsertData(tsdb, pMsg->pCont); SRpcMsg rpcRsp = { .handle = pMsg->rpcMsg.handle, @@ -285,6 +288,7 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { .code = 0, .msgType = 0 }; + rpcSendResponse(&rpcRsp); } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 467c2a2995..cc17df9bec 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -188,14 +188,14 @@ extern char *taosMsg[]; #pragma pack(push, 1) -typedef struct { - int32_t vnode; - int32_t sid; - int32_t sversion; - uint64_t uid; - int16_t numOfRows; - char payLoad[]; -} SShellSubmitBlock; +//typedef struct { +// int32_t vnode; +// int32_t sid; +// int32_t sversion; +// uint64_t uid; +// int16_t numOfRows; +// char payLoad[]; +//} SShellSubmitBlock; typedef struct { int32_t numOfVnodes; @@ -206,13 +206,33 @@ typedef struct SMsgHead { int32_t vgId; } SMsgHead; -typedef struct { - SMsgDesc desc; - SMsgHead header; - int16_t import; - int32_t numOfTables; // total number of sid - char blks[]; // number of data blocks, each table has at least one data block -} SShellSubmitMsg; +//typedef struct { +// SMsgDesc desc; +// SMsgHead header; +// int16_t import; +// int32_t numOfTables; // total number of sid +// char blks[]; // number of data blocks, each table has at least one data block +//} SShellSubmitMsg; + +// Submit message for one table +typedef struct SSubmitBlk { + int64_t uid; // table unique id + int32_t tid; // table id + int32_t padding; // TODO just for padding here + int32_t sversion; // data schema version + int32_t len; // data part length, not including the SSubmitBlk head + int16_t numOfRows; // total number of rows in current submit block + char data[]; +} SSubmitBlk; + +// Submit message for this TSDB +typedef struct SSubmitMsg { + SMsgHead header; + int32_t length; + int32_t compressed:2; + int32_t numOfBlocks:30; + SSubmitBlk blocks[]; +} SSubmitMsg; typedef struct { int32_t index; // index of failed block in submit blocks @@ -506,14 +526,16 @@ typedef struct { } SQueryTableRsp; typedef struct { + SMsgHead header; uint64_t qhandle; uint16_t free; } SRetrieveTableMsg; -typedef struct { +typedef struct SRetrieveTableRsp { int32_t numOfRows; + int8_t completed; // all results are returned to client int16_t precision; - int64_t offset; // updated offset value for multi-vnode projection query + int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; char data[]; } SRetrieveTableRsp; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 8956fb52b1..346170cde8 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -171,6 +171,7 @@ typedef struct SQueryRuntimeEnv { typedef struct SQInfo { uint64_t signature; + void* pVnode; TSKEY startTime; int64_t elapsedTime; SResultRec rec; @@ -205,7 +206,7 @@ typedef struct SQInfo { * @param pQInfo * @return */ -int32_t qCreateQueryInfo(SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); +int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); /** * query on single table diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 0a5abccdd5..69ae0c06bd 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -14,9 +14,9 @@ */ #include "os.h" -#include "taosmsg.h" #include "hash.h" #include "hashfunc.h" +#include "taosmsg.h" #include "tlog.h" #include "tlosertree.h" #include "tscompression.h" @@ -1482,10 +1482,11 @@ static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool i } } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order, bool isSTableQuery) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order, + bool isSTableQuery) { dTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); - SQuery* pQuery = pRuntimeEnv->pQuery; - + SQuery *pQuery = pRuntimeEnv->pQuery; + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); @@ -1890,7 +1891,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; } - + // pQuery->pointsOffset = pQuery->pointsToRead; } @@ -2242,7 +2243,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { size_t s = taosArrayGetSize(pQInfo->pTableIdList); num = MAX(s, INITIAL_RESULT_ROWS_VALUE); } else { // for super table query, one page for each subset -// num = pQInfo->pSidSet->numOfSubSet; + // num = pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -2289,7 +2290,7 @@ void vnodeQueryFreeQInfoEx(SQInfo *pQInfo) { // tSidSetDestroy(&pQInfo->pSidSet); if (pQInfo->pTableDataInfo != NULL) { -// size_t num = taosHashGetSize(pQInfo->pTableIdList); + // size_t num = taosHashGetSize(pQInfo->pTableIdList); for (int32_t j = 0; j < 0; ++j) { destroyMeterQueryInfo(pQInfo->pTableDataInfo[j].pTableQInfo, pQuery->numOfOutputCols); } @@ -2328,7 +2329,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { pQuery->lastKey = pQuery->window.skey; // create runtime environment -// SColumnModel *pTagSchemaInfo = pQInfo->pSidSet->pColumnModel; + // SColumnModel *pTagSchemaInfo = pQInfo->pSidSet->pColumnModel; // get one queried meter assert(0); @@ -2388,7 +2389,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { taosArrayPush(cols, &pQuery->colList[i]); } - pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(&cond, sa, cols); + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(NULL, &cond, sa, cols); // metric query do not invoke interpolation, it will be done at the second-stage merge if (!isPointInterpoQuery(pQuery)) { @@ -2409,7 +2410,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { */ void vnodeDecMeterRefcnt(SQInfo *pQInfo) { if (pQInfo != NULL) { -// assert(taosHashGetSize(pQInfo->pTableIdList) >= 1); + // assert(taosHashGetSize(pQInfo->pTableIdList) >= 1); } #if 0 @@ -2587,23 +2588,23 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; TSKEY *keyList; - + if (num <= 0) return -1; - + keyList = (TSKEY *)pValue; firstPos = 0; lastPos = num - 1; - + if (order == 0) { // find the first position which is smaller than the key while (1) { if (key >= keyList[lastPos]) return lastPos; if (key == keyList[firstPos]) return firstPos; if (key < keyList[firstPos]) return firstPos - 1; - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -2612,13 +2613,13 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { break; } } - + } else { // find the first position which is bigger than the key while (1) { if (key <= keyList[firstPos]) return firstPos; if (key == keyList[lastPos]) return lastPos; - + if (key > keyList[lastPos]) { lastPos = lastPos + 1; if (lastPos >= num) @@ -2626,10 +2627,10 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { else return lastPos; } - + numOfPoints = lastPos - firstPos + 1; midPos = (numOfPoints >> 1) + firstPos; - + if (key < keyList[midPos]) { lastPos = midPos - 1; } else if (key > keyList[midPos]) { @@ -2639,7 +2640,7 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } } } - + return midPos; } @@ -2648,65 +2649,68 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int64_t cnt = 0; dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", - GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - + GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); + tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; - + while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check if (isQueryKilled(pQuery)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return cnt; } - + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - + if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { - TSKEY skey1, ekey1; - STimeWindow w = {0}; - SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (QUERY_IS_ASC_QUERY(pQuery)) { - doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1, &w); + doGetAlignedIntervalQueryRangeImpl(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, + &skey1, &ekey1, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp TSKEY winStart = blockInfo.window.ekey - pQuery->intervalTime; - doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); - + doGetAlignedIntervalQueryRangeImpl(pQuery, winStart, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, + &w); + pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } - + int32_t numOfRes = 0; - + SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); - -// dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, rows:%d, checked:%d", -// GET_QINFO_ADDR(pQuery), blockInfo.window.skey, blockInfo.window.ekey, pQueryHandle->cur.fileId, pQueryHandle->cur.slot, -// pQuery->pos, blockInfo.size, forwardStep); - + + // dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, rows:%d, + // checked:%d", + // GET_QINFO_ADDR(pQuery), blockInfo.window.skey, blockInfo.window.ekey, pQueryHandle->cur.fileId, + // pQueryHandle->cur.slot, pQuery->pos, blockInfo.size, forwardStep); + // save last access position cnt += forwardStep; if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { break; } } - + // if the result buffer is not full, set the query completed flag if (!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { setQueryStatus(pQuery, QUERY_COMPLETED); } - + if (isIntervalQuery(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { - int32_t step = QUERY_IS_ASC_QUERY(pQuery)? QUERY_ASC_FORWARD_STEP:QUERY_DESC_FORWARD_STEP; - + int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP; + closeAllTimeWindow(&pRuntimeEnv->windowResInfo); removeRedundantWindow(&pRuntimeEnv->windowResInfo, pQuery->lastKey - step, step); pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; @@ -2714,7 +2718,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); } } - + return cnt; } @@ -2976,29 +2980,30 @@ int32_t mergeMetersResultToOneGroups(SQInfo *pQInfo) { int64_t st = taosGetTimestampMs(); int32_t ret = TSDB_CODE_SUCCESS; -// while (pQInfo->subgroupIdx < pQInfo->pSidSet->numOfSubSet) { -// int32_t start = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx]; -// int32_t end = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx + 1]; -// -// assert(0); -// // ret = doMergeMetersResultsToGroupRes(pQInfo, pQuery, pRuntimeEnv, pQInfo->pTableDataInfo, start, end); -// if (ret < 0) { // not enough disk space to save the data into disk -// return -1; -// } -// -// pQInfo->subgroupIdx += 1; -// -// // this group generates at least one result, return results -// if (ret > 0) { -// break; -// } -// -// assert(pQInfo->numOfGroupResultPages == 0); -// dTrace("QInfo:%p no result in group %d, continue", GET_QINFO_ADDR(pQuery), pQInfo->subgroupIdx - 1); -// } -// -// dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", GET_QINFO_ADDR(pQuery), -// pQInfo->subgroupIdx - 1, pQInfo->pSidSet->numOfSubSet, taosGetTimestampMs() - st); + // while (pQInfo->subgroupIdx < pQInfo->pSidSet->numOfSubSet) { + // int32_t start = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx]; + // int32_t end = pQInfo->pSidSet->starterPos[pQInfo->subgroupIdx + 1]; + // + // assert(0); + // // ret = doMergeMetersResultsToGroupRes(pQInfo, pQuery, pRuntimeEnv, pQInfo->pTableDataInfo, start, end); + // if (ret < 0) { // not enough disk space to save the data into disk + // return -1; + // } + // + // pQInfo->subgroupIdx += 1; + // + // // this group generates at least one result, return results + // if (ret > 0) { + // break; + // } + // + // assert(pQInfo->numOfGroupResultPages == 0); + // dTrace("QInfo:%p no result in group %d, continue", GET_QINFO_ADDR(pQuery), pQInfo->subgroupIdx - 1); + // } + // + // dTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", + // GET_QINFO_ADDR(pQuery), + // pQInfo->subgroupIdx - 1, pQInfo->pSidSet->numOfSubSet, taosGetTimestampMs() - st); return TSDB_CODE_SUCCESS; } @@ -3013,10 +3018,10 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { } // set current query completed -// if (pQInfo->numOfGroupResultPages == 0 && pQInfo->subgroupIdx == pQInfo->pSidSet->numOfSubSet) { - // pQInfo->tableIndex = pQInfo->pSidSet->numOfTables; -// return; -// } + // if (pQInfo->numOfGroupResultPages == 0 && pQInfo->subgroupIdx == pQInfo->pSidSet->numOfSubSet) { + // pQInfo->tableIndex = pQInfo->pSidSet->numOfTables; + // return; + // } } SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; @@ -3923,7 +3928,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); } else { -// totalSubset = pQInfo->pSidSet->numOfSubSet; + // totalSubset = pQInfo->pSidSet->numOfSubSet; } return totalSubset; @@ -4303,7 +4308,7 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param) { +int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param, void* tsdb) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; @@ -4327,8 +4332,8 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param) { pQuery->lastKey = pQuery->window.skey; STsdbQueryCond cond = {0}; - - cond.twindow = (STimeWindow) {.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; + + cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; cond.order = pQuery->order.order; cond.colList = *pQuery->colList; @@ -4337,7 +4342,8 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param) { taosArrayPush(cols, &pQuery->colList[i]); } - pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(&cond, pQInfo->pTableIdList, cols); + + pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; pRuntimeEnv->pQuery = pQuery; @@ -5248,41 +5254,42 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { // pQInfo->pointsRead - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } -void qTableQuery(SQInfo* pQInfo) { +void qTableQuery(SQInfo *pQInfo) { assert(pQInfo != NULL); - + if (pQInfo->killed) { dTrace("QInfo:%p it is already killed, abort", pQInfo); return; } - + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - -// dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pQInfo); - + SQuery * pQuery = pRuntimeEnv->pQuery; + + // dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pQInfo); + if (vnodeHasRemainResults(pQInfo)) { /* * There are remain results that are not returned due to result interpolation * So, we do keep in this procedure instead of launching retrieve procedure for next results. */ int32_t numOfInterpo = 0; - + int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); pQuery->rec.pointsRead = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, - (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); - + (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); + doRevisedResultsByLimit(pQInfo); - + pQInfo->pointsInterpo += numOfInterpo; pQInfo->rec.pointsRead += pQuery->rec.pointsRead; - -// dTrace("QInfo:%p %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", -// pQInfo, pQuery->pointsRead, numOfInterpo, pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); + + // dTrace("QInfo:%p %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", + // pQInfo, pQuery->pointsRead, numOfInterpo, pQInfo->pointsRead, pQInfo->pointsInterpo, + // pQInfo->pointsReturned); sem_post(&pQInfo->dataReady); return; } - + // here we have scan all qualified data in both data file and cache if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { // continue to get push data from the group result @@ -5291,66 +5298,68 @@ void qTableQuery(SQInfo* pQInfo) { // todo limit the output for interval query? pQuery->rec.pointsRead = 0; pQInfo->subgroupIdx = 0; // always start from 0 - + if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQInfo->rec.pointsRead += pQuery->rec.pointsRead; - + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); - + if (pQuery->rec.pointsRead > 0) { -// dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d totalReturn:%d", -// pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, -// pQInfo->pointsInterpo, pQInfo->pointsReturned); - + // dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d + // totalReturn:%d", + // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, + // pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); + sem_post(&pQInfo->dataReady); return; } } } - -// dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, -// pMeterObj->meterId, pQInfo->pointsRead); - -// vnodePrintQueryStatistics(pSupporter); + + // dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, + // pMeterObj->sid, + // pMeterObj->meterId, pQInfo->pointsRead); + + // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); return; } - + // number of points returned during this query pQuery->rec.pointsRead = 0; - + int64_t st = taosGetTimestampUs(); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) -// assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); + // assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); tableIntervalProcessor(pQInfo); } else { if (isFixedOutputQuery(pQuery)) { assert(pQuery->checkBufferInLoop == 0); - + tableFixedOutputProcessor(pQInfo); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBufferInLoop == 1); tableMultiOutputProcessor(pQInfo); } } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); - + /* check if query is killed or not */ if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query is killed", pQInfo); -// pQInfo->over = 1; + // pQInfo->over = 1; } else { -// dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, -// pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); + // dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, + // pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); } - + sem_post(&pQInfo->dataReady); -// vnodeDecRefCount(pQInfo); + // vnodeDecRefCount(pQInfo); } void qSuperTableQuery(void *pReadMsg) { @@ -5460,10 +5469,10 @@ static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { return 0; } -static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray** pTableIdList) { +static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList) { pQueryTableMsg->vgId = htons(pQueryTableMsg->vgId); pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); - + pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); @@ -5483,12 +5492,12 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray** pTa pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); - + pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); pQueryTableMsg->tsNumOfBlocks = htonl(pQueryTableMsg->tsNumOfBlocks); pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); - + // query msg safety check if (validateQueryMeterMsg(pQueryTableMsg) != 0) { return TSDB_CODE_INVALID_QUERY_MSG; @@ -5590,24 +5599,24 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray** pTa pQueryTableMsg->colNameList = (int64_t)pMsg; pMsg += pQueryTableMsg->colNameLen; } - + *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); - - STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg; + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + taosArrayPush(*pTableIdList, pTableIdInfo); pMsg += sizeof(STableIdInfo); - + for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { pTableIdInfo = (STableIdInfo *)pMsg; - + pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + taosArrayPush(*pTableIdList, pTableIdInfo); pMsg += sizeof(STableIdInfo); } @@ -5710,7 +5719,7 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; - + SSqlFunctionExpr *pExprs = (SSqlFunctionExpr *)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols); if (pExprs == NULL) { tfree(pQueryMsg->pSqlFuncExprs); @@ -5752,11 +5761,11 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct type = TSDB_DATA_TYPE_DOUBLE; bytes = tDataTypeDesc[type].nSize; - } else { // parse the normal column + } else { // parse the normal column int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); assert(j < pQueryMsg->numOfCols); - SColumnInfo* pCol = &pQueryMsg->colList[j]; + SColumnInfo *pCol = &pQueryMsg->colList[j]; type = pCol->type; bytes = pCol->bytes; } @@ -5909,7 +5918,8 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_SUCCESS; } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, SArray* pTableIdList) { +static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, + SArray *pTableIdList) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { goto _clean_memory; @@ -5944,20 +5954,20 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou if (pQuery->colList == NULL) { goto _clean_memory; } - + for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i].info = pQueryMsg->colList[i]; -// SColumnInfo *pColInfo = &pQuery->colList[i].data; -// pColInfo->filters = NULL; -// if (colList[i].numOfFilters > 0) { -// pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); -// -// for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { -// tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); -// } -// } else { -// pQuery->colList[i].data.filters = NULL; -// } + // SColumnInfo *pColInfo = &pQuery->colList[i].data; + // pColInfo->filters = NULL; + // if (colList[i].numOfFilters > 0) { + // pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); + // + // for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { + // tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); + // } + // } else { + // pQuery->colList[i].data.filters = NULL; + // } } // calculate the result row size @@ -6003,9 +6013,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou // to make sure third party won't overwrite this structure pQInfo->signature = (uint64_t)pQInfo; pQInfo->pTableIdList = pTableIdList; - + pQuery->pos = -1; - // dTrace("vid:%d sid:%d meterId:%s, QInfo is allocated:%p", pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQInfo); + // dTrace("vid:%d sid:%d meterId:%s, QInfo is allocated:%p", pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, + // pQInfo); return pQInfo; @@ -6035,7 +6046,7 @@ bool isQInfoValid(void *param) { if (pQInfo == NULL) { return false; } - + /* * pQInfo->signature may be changed by another thread, so we assign value of signature * into local variable, then compare by using local variable @@ -6044,75 +6055,75 @@ bool isQInfoValid(void *param) { return (sig == (uint64_t)pQInfo); } -void vnodeFreeQInfo(SQInfo* pQInfo, bool decQueryRef) { +void vnodeFreeQInfo(SQInfo *pQInfo, bool decQueryRef) { if (!isQInfoValid(pQInfo)) { return; } - + pQInfo->killed = 1; dTrace("QInfo:%p start to free SQInfo", pQInfo); - + if (decQueryRef) { vnodeDecMeterRefcnt(pQInfo); } - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - + for (int col = 0; col < pQuery->numOfOutputCols; ++col) { tfree(pQuery->sdata[col]); } - -// for (int col = 0; col < pQuery->numOfCols; ++col) { -// vnodeFreeColumnInfo(&pQuery->colList[col].data); -// } -// -// if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { -// tfree(pQuery->tsData); -// } - + + // for (int col = 0; col < pQuery->numOfCols; ++col) { + // vnodeFreeColumnInfo(&pQuery->colList[col].data); + // } + // + // if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + // tfree(pQuery->tsData); + // } + sem_destroy(&(pQInfo->dataReady)); vnodeQueryFreeQInfoEx(pQInfo); - + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { tfree(pColFilter->pFilters); } } - + tfree(pQuery->pFilterInfo); tfree(pQuery->colList); tfree(pQuery->sdata); - + if (pQuery->pSelectExpr != NULL) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; - + if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr, NULL); } } - + tfree(pQuery->pSelectExpr); } - + if (pQuery->defaultVal != NULL) { tfree(pQuery->defaultVal); } - + tfree(pQuery->pGroupbyExpr); tfree(pQuery); - -// dTrace("QInfo:%p vid:%d sid:%d meterId:%s, QInfo is freed", pQInfo, pObj->vnode, pObj->sid, pObj->meterId); - - //destroy signature, in order to avoid the query process pass the object safety check + + // dTrace("QInfo:%p vid:%d sid:%d meterId:%s, QInfo is freed", pQInfo, pObj->vnode, pObj->sid, pObj->meterId); + + // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, - SArray* pTableIdList, SQInfo **pQInfo) { + SArray *pTableIdList, void* tsdb, SQInfo **pQInfo) { int32_t code = TSDB_CODE_SUCCESS; (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pSqlExprs, pTableIdList); @@ -6121,16 +6132,12 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE goto _error; } - SQuery* pQuery = (*pQInfo)->runtimeEnv.pQuery; + SQuery *pQuery = (*pQInfo)->runtimeEnv.pQuery; dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); -// STableIdInfo **pTableIdList = (STableIdInfo **)pQueryMsg->pSidExtInfo; -// if (pTableIdList != NULL && pTableIdList[0]->key > 0) { -// pQuery->window.skey = pTableIdList[0]->key; -// } else { pQuery->window.skey = pQueryMsg->window.skey; pQuery->window.ekey = pQueryMsg->window.ekey; - + pQuery->lastKey = pQuery->window.skey; if (sem_init(&(*pQInfo)->dataReady, 0, 0) != 0) { @@ -6151,7 +6158,7 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE tsBufNextPos(pTSBuf); } - if ((code = vnodeQueryTablePrepare(*pQInfo, pTSBuf)) != TSDB_CODE_SUCCESS) { + if ((code = vnodeQueryTablePrepare(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -6163,18 +6170,18 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, // pQInfo->refCount); return code; - + _error: // table query ref will be decrease during error handling vnodeFreeQInfo(*pQInfo, false); return code; } -int32_t qCreateQueryInfo(SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { assert(pQueryTableMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; - SArray* pTableIdList = NULL; + SArray *pTableIdList = NULL; if ((code = convertQueryTableMsg(pQueryTableMsg, &pTableIdList)) != TSDB_CODE_SUCCESS) { return code; } @@ -6205,14 +6212,14 @@ int32_t qCreateQueryInfo(SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { if (QUERY_IS_STABLE_QUERY(pQueryTableMsg->queryType)) { // pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryTableMsg, &code); } else { - code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, pQInfo); + code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); } _query_over: if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pTableIdList); } - + // if failed to add ref for all meters in this query, abort current query // if (code != TSDB_CODE_SUCCESS) { // vnodeDecQueryRefCount(pQueryTableMsg, pMeterObjList, incNumber); @@ -6231,31 +6238,31 @@ _query_over: return TSDB_CODE_SUCCESS; } -int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize) { +int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *rowsize) { if (pQInfo == NULL || !isQInfoValid(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - + if (pQInfo->killed) { dTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); if (pQInfo->code == TSDB_CODE_SUCCESS) { return TSDB_CODE_QUERY_CANCELLED; - } else { // in case of not TSDB_CODE_SUCCESS, return the code to client + } else { // in case of not TSDB_CODE_SUCCESS, return the code to client return abs(pQInfo->code); } } - + sem_wait(&pQInfo->dataReady); - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - -// *numOfRows = pQInfo->rec.pointsRead; -// *rowsize = pQuery->rowSize; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // *numOfRows = pQInfo->rec.pointsRead; + // *rowsize = pQuery->rowSize; *numOfRows = 1; - -// dTrace("QInfo:%p, retrieve data info completed, precision:%d, rowsize:%d, rows:%d, code:%d", pQInfo, *timePrec, -// *rowsize, *numOfRows, pQInfo->code); - + + // dTrace("QInfo:%p, retrieve data info completed, precision:%d, rowsize:%d, rows:%d, code:%d", pQInfo, *timePrec, + // *rowsize, *numOfRows, pQInfo->code); + if (pQInfo->code < 0) { // less than 0 means there are error existed. return -pQInfo->code; } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 85b1d3d206..c46f45fd37 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -15,7 +15,6 @@ #include "os.h" #include "tlog.h" -// #include "tsdb.h" #include "tskiplist.h" #include "tutil.h" @@ -395,6 +394,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { SSkipListNode *px = pSkipList->pHead; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; + bool identical = false; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); while (p != NULL) { @@ -402,11 +402,16 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); // if the forward element is less than the specified key, forward one step - if (pSkipList->comparFn(key, newDatakey) < 0) { + int32_t ret = pSkipList->comparFn(key, newDatakey); + if (ret < 0) { px = p; p = SL_GET_FORWARD_POINTER(px, i); } else { + if (identical == false) { + identical = (ret == 0); + } + break; } } @@ -418,17 +423,12 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { } // if the skip list does not allowed identical key inserted, the new data will be discarded. - if (pSkipList->keyInfo.dupKey == 0 && forward[0] != pSkipList->pHead) { - char *key = SL_GET_NODE_KEY(pSkipList, forward[0]); - char *pNewDataKey = SL_GET_NODE_KEY(pSkipList, pNode); - - if (pSkipList->comparFn(key, pNewDataKey) == 0) { - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return forward[0]; + if (pSkipList->keyInfo.dupKey == 0 && identical) { + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); } + + return forward[0]; } #if SKIP_LIST_RECORD_PERFORMANCE diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 4964ac673f..b88b13e288 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -88,15 +88,6 @@ int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg); int tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId); int tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg); -// Submit message for one table -typedef struct { - STableId tableId; - int32_t padding; // TODO just for padding here - int32_t sversion; // data schema version - int32_t len; // data part length, not including the SSubmitBlk head - char data[]; -} SSubmitBlk; - typedef struct { int32_t totalLen; int32_t len; @@ -106,15 +97,10 @@ typedef struct { int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); -// Submit message for this TSDB -typedef struct { - int32_t length; - int32_t compressed; - SSubmitBlk blocks[]; -} SSubmitMsg; - #define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) +struct STsdbRepo; + // SSubmitMsg Iterator typedef struct { int32_t totalLen; @@ -243,7 +229,7 @@ typedef void *tsdbpos_t; * @param pTableList table sid list * @return */ -tsdb_query_handle_t *tsdbQueryByTableId(STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo); +tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo); /** * move to next block diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 2324ead451..7f3acb6624 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -63,6 +63,26 @@ typedef struct { SFileGroup fGroup[]; } STsdbFileH; +/** + * if numOfSubBlocks == -1, then the SCompBlock is a sub-block + * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to + * the data block offset and length + * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the + * binary + */ +typedef struct { + int64_t last : 1; // If the block in data file or last file + int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks + int32_t algorithm : 8; // Compression algorithm + int32_t numOfPoints : 24; // Number of total points + int32_t sversion; // Schema version + int32_t len; // Data block length or nothing + int16_t numOfSubBlocks; // Number of sub-blocks; + int16_t numOfCols; + TSKEY keyFirst; + TSKEY keyLast; +} SCompBlock; + #define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX) STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, diff --git a/src/vnode/tsdb/inc/tsdbMeta.h b/src/vnode/tsdb/inc/tsdbMeta.h index aba506f30d..059ec87e91 100644 --- a/src/vnode/tsdb/inc/tsdbMeta.h +++ b/src/vnode/tsdb/inc/tsdbMeta.h @@ -44,7 +44,7 @@ typedef struct { typedef struct STable { int8_t type; STableId tableId; - int32_t superUid; // Super table UID + int64_t superUid; // Super table UID int32_t sversion; STSchema * schema; STSchema * tagSchema; @@ -98,6 +98,8 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta); #define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id] #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */ +STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo); + int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg); int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 1c91c03b44..98562be0cc 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -33,26 +33,6 @@ typedef struct { int64_t offset; } SCompIdx; -/** - * if numOfSubBlocks == -1, then the SCompBlock is a sub-block - * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to - * the data block offset and length - * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the - * binary - */ -typedef struct { - int64_t last : 1; // If the block in data file or last file - int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks - int32_t algorithm : 8; // Compression algorithm - int32_t numOfPoints : 24; // Number of total points - int32_t sversion; // Schema version - int32_t len; // Data block length or nothing - int16_t numOfSubBlocks; // Number of sub-blocks; - int16_t numOfCols; - TSKEY keyFirst; - TSKEY keyLast; -} SCompBlock; - typedef struct { int32_t delimiter; // For recovery usage int32_t checksum; // TODO: decide if checksum logic in this file or make it one API diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index cd673d2d89..6c8e998630 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -485,6 +485,10 @@ SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL || pIter == NULL) return -1; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + pMsg->compressed = htonl(pMsg->compressed); + pIter->totalLen = pMsg->length; pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE; if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) { @@ -499,7 +503,15 @@ int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { SSubmitBlk *pBlock = pIter->pBlock; if (pBlock == NULL) return NULL; - + + pBlock->len = htonl(pBlock->len); + pBlock->numOfRows = htons(pBlock->numOfRows); + pBlock->uid = htobe64(pBlock->uid); + pBlock->tid = htonl(pBlock->tid); + + pBlock->sversion = htonl(pBlock->sversion); + pBlock->padding = htonl(pBlock->padding); + pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len; if (pIter->len >= pIter->totalLen) { pIter->pBlock = NULL; @@ -510,6 +522,11 @@ SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { return pBlock; } +STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo) { + STsdbRepo *tsdb = (STsdbRepo *)pRepo; + return tsdb->tsdbMeta; +} + // Check the configuration and set default options static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // Check precision @@ -681,6 +698,8 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListRandNodeInfo(pTable->mem->pData, &level, &headSize); TSKEY key = dataRowKey(row); + printf("insert:%lld, size:%d\n", key, pTable->mem->numOfPoints); + // Copy row into the memory SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); if (pNode == NULL) { @@ -694,7 +713,9 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListPut(pTable->mem->pData, pNode); if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; - pTable->mem->numOfPoints++; + + pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData); +// pTable->mem->numOfPoints++; return 0; } @@ -702,7 +723,8 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { STsdbRepo *pRepo = (STsdbRepo *)repo; - STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, pBlock->tableId); + STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; + STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId); if (pTable == NULL) return -1; SSubmitBlkIter blkIter; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index a62299c45f..d654308f99 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -14,53 +14,378 @@ */ #include "os.h" +#include "tutil.h" + #include "tsdb.h" +#include "tsdbFile.h" +#include "tsdbMeta.h" -tsdb_query_handle_t *tsdbQueryByTableId(STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo) { +#define EXTRA_BYTES 2 +#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx *)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) +#define QUERY_IS_ASC_QUERY(o) (o == TSQL_SO_ASC) +#define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) +typedef struct SField { + // todo need the definition +} SField; + +typedef struct SHeaderFileInfo { + int32_t fileId; +} SHeaderFileInfo; + +typedef struct SQueryHandlePos { + int32_t fileId; + int32_t slot; + int32_t pos; + int32_t fileIndex; +} SQueryHandlePos; + +typedef struct SDataBlockLoadInfo { + int32_t fileListIndex; + int32_t fileId; + int32_t slotIdx; + int32_t sid; + SArray *pLoadedCols; +} SDataBlockLoadInfo; + +typedef struct SLoadCompBlockInfo { + int32_t sid; /* meter sid */ + int32_t fileId; + int32_t fileListIndex; +} SLoadCompBlockInfo; + +typedef struct SQueryFilesInfo { + SArray *pFileInfo; + int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap. + int32_t vnodeId; + + int32_t headerFd; // header file fd + int64_t headerFileSize; + int32_t dataFd; + int32_t lastFd; + + char headerFilePath[PATH_MAX]; // current opened header file name + char dataFilePath[PATH_MAX]; // current opened data file name + char lastFilePath[PATH_MAX]; // current opened last file path + char dbFilePathPrefix[PATH_MAX]; +} SQueryFilesInfo; + +typedef struct STableQueryRec { + TSKEY lastKey; + STable * pTableObj; + int64_t offsetInHeaderFile; + int32_t numOfBlocks; + int32_t start; + SCompBlock *pBlock; +} STableQueryRec; + +typedef struct { + SCompBlock *compBlock; + SField * fields; +} SCompBlockFields; + +typedef struct STableDataBlockInfoEx { + SCompBlockFields pBlock; + STableQueryRec * pMeterDataInfo; + int32_t blockIndex; + int32_t groupIdx; /* number of group is less than the total number of meters */ +} STableDataBlockInfoEx; + +typedef struct STsdbQueryHandle { + struct STsdbRepo* pTsdb; + + SQueryHandlePos cur; // current position + SQueryHandlePos start; // the start position, used for secondary/third iteration + int32_t unzipBufSize; + char *unzipBuffer; + char *secondaryUnzipBuffer; + + SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ + SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ + + SQueryFilesInfo vnodeFileInfo; + + int16_t numOfRowsPerPage; + uint16_t flag; // denotes reversed scan of data or not + int16_t order; + STimeWindow window; // the primary query time window that applies to all queries + TSKEY lastKey; + int32_t blockBufferSize; + SCompBlock *pBlock; + int32_t numOfBlocks; + SField ** pFields; + SArray * pColumns; // column list, SColumnInfoEx array list + SArray * pTableIdList; // table id object list + bool locateStart; + int32_t realNumOfRows; + bool loadDataAfterSeek; // load data after seek. + + STableDataBlockInfoEx *pDataBlockInfoEx; + STableQueryRec * pTableQueryInfo; + int32_t tableIndex; + bool isFirstSlot; + void * qinfo; // query info handle, for debug purpose +} STsdbQueryHandle; + +int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { + // record the maximum column width among columns of this meter/metric + SColumnInfoEx *pColumn = taosArrayGet(pQueryHandle->pColumns, 0); + + int32_t maxColWidth = pColumn->info.bytes; + for (int32_t i = 1; i < QH_GET_NUM_OF_COLS(pQueryHandle); ++i) { + int32_t bytes = pColumn[i].info.bytes; + if (bytes > maxColWidth) { + maxColWidth = bytes; + } + } + + // only one unzip buffer required, since we can unzip each column one by one + pQueryHandle->unzipBufSize = (size_t)(maxColWidth * rowsPerFileBlock + EXTRA_BYTES); // plus extra_bytes + pQueryHandle->unzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); + + pQueryHandle->secondaryUnzipBuffer = (char *)calloc(1, pQueryHandle->unzipBufSize); + + if (pQueryHandle->unzipBuffer == NULL || pQueryHandle->secondaryUnzipBuffer == NULL) { + goto _error_clean; + } + + return TSDB_CODE_SUCCESS; + +_error_clean: + tfree(pQueryHandle->unzipBuffer); + tfree(pQueryHandle->secondaryUnzipBuffer); + + return TSDB_CODE_SERV_OUT_OF_MEMORY; } +static void initQueryFileInfoFD(SQueryFilesInfo *pVnodeFilesInfo) { + pVnodeFilesInfo->current = -1; + pVnodeFilesInfo->headerFileSize = -1; + + pVnodeFilesInfo->headerFd = FD_INITIALIZER; // set the initial value + pVnodeFilesInfo->dataFd = FD_INITIALIZER; + pVnodeFilesInfo->lastFd = FD_INITIALIZER; +} + +static void vnodeInitDataBlockLoadInfo(SDataBlockLoadInfo *pBlockLoadInfo) { + pBlockLoadInfo->slotIdx = -1; + pBlockLoadInfo->fileId = -1; + pBlockLoadInfo->sid = -1; + pBlockLoadInfo->fileListIndex = -1; +} + +static void vnodeInitCompBlockLoadInfo(SLoadCompBlockInfo *pCompBlockLoadInfo) { + pCompBlockLoadInfo->sid = -1; + pCompBlockLoadInfo->fileId = -1; + pCompBlockLoadInfo->fileListIndex = -1; +} + +static int fileOrderComparFn(const void *p1, const void *p2) { + SHeaderFileInfo *pInfo1 = (SHeaderFileInfo *)p1; + SHeaderFileInfo *pInfo2 = (SHeaderFileInfo *)p2; + + if (pInfo1->fileId == pInfo2->fileId) { + return 0; + } + + return (pInfo1->fileId > pInfo2->fileId) ? 1 : -1; +} + +void vnodeRecordAllFiles(int32_t vnodeId, SQueryFilesInfo *pVnodeFilesInfo) { + char suffix[] = ".head"; + pVnodeFilesInfo->pFileInfo = taosArrayInit(4, sizeof(int32_t)); + + struct dirent *pEntry = NULL; + pVnodeFilesInfo->vnodeId = vnodeId; + char* tsDirectory = ""; + + sprintf(pVnodeFilesInfo->dbFilePathPrefix, "%s/vnode%d/db/", tsDirectory, vnodeId); + DIR *pDir = opendir(pVnodeFilesInfo->dbFilePathPrefix); + if (pDir == NULL) { + // dError("QInfo:%p failed to open directory:%s, %s", pQInfo, pVnodeFilesInfo->dbFilePathPrefix, + // strerror(errno)); + return; + } + + while ((pEntry = readdir(pDir)) != NULL) { + if ((pEntry->d_name[0] == '.' && pEntry->d_name[1] == '\0') || (strcmp(pEntry->d_name, "..") == 0)) { + continue; + } + + if (pEntry->d_type & DT_DIR) { + continue; + } + + size_t len = strlen(pEntry->d_name); + if (strcasecmp(&pEntry->d_name[len - 5], suffix) != 0) { + continue; + } + + int32_t vid = 0; + int32_t fid = 0; + sscanf(pEntry->d_name, "v%df%d", &vid, &fid); + if (vid != vnodeId) { /* ignore error files */ + // dError("QInfo:%p error data file:%s in vid:%d, ignore", pQInfo, pEntry->d_name, vnodeId); + continue; + } + +// int32_t firstFid = pVnode->fileId - pVnode->numOfFiles + 1; +// if (fid > pVnode->fileId || fid < firstFid) { +// dError("QInfo:%p error data file:%s in vid:%d, fid:%d, fid range:%d-%d", pQInfo, pEntry->d_name, vnodeId, +// fid, firstFid, pVnode->fileId); +// continue; +// } + + assert(fid >= 0 && vid >= 0); + taosArrayPush(pVnodeFilesInfo->pFileInfo, &fid); + } + + closedir(pDir); + + // dTrace("QInfo:%p find %d data files in %s to be checked", pQInfo, pVnodeFilesInfo->numOfFiles, + // pVnodeFilesInfo->dbFilePathPrefix); + + // order the files information according their names */ + size_t numOfFiles = taosArrayGetSize(pVnodeFilesInfo->pFileInfo); + qsort(pVnodeFilesInfo->pFileInfo->pData, numOfFiles, sizeof(SHeaderFileInfo), fileOrderComparFn); +} + +tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond, SArray *idList, SArray *pColumnInfo) { + // todo 1. filter not exist table + + // todo 2. add the reference count for each table that is involved in query + + STsdbQueryHandle *pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + pQueryHandle->order = pCond->order; + pQueryHandle->window = pCond->twindow; + pQueryHandle->pTsdb = tsdb; + + pQueryHandle->pTableIdList = idList; + pQueryHandle->pColumns = pColumnInfo; + pQueryHandle->loadDataAfterSeek = false; + pQueryHandle->isFirstSlot = true; + + pQueryHandle->lastKey = pQueryHandle->window.skey; // ascending query + + // malloc buffer in order to load data from file + int32_t numOfCols = taosArrayGetSize(pColumnInfo); + size_t bufferCapacity = 4096; + + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoEx)); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoEx *pCol = taosArrayGet(pColumnInfo, i); + SColumnInfoEx pDest = {{0}, 0}; + + pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCol->info.bytes); + pDest.info = pCol->info; + taosArrayPush(pQueryHandle->pColumns, &pDest); + } + + if (doAllocateBuf(pQueryHandle, bufferCapacity) != TSDB_CODE_SUCCESS) { + return NULL; + } + + initQueryFileInfoFD(&pQueryHandle->vnodeFileInfo); + vnodeInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + vnodeInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + + int32_t vnodeId = 1; + vnodeRecordAllFiles(vnodeId, &pQueryHandle->vnodeFileInfo); + + return (tsdb_query_handle_t)pQueryHandle; +} + +static int32_t next = 1; bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - return false; + if (next == 0) { + return false; + } else { + next = 0; + return true; + } } +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, + TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) { + int numOfRows = 0; + int32_t numOfCols = taosArrayGetSize(pHandle->pColumns); + *skey = INT64_MIN; + + while(tSkipListIterNext(pIter)) { + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) break; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) > maxKey) break; + // Convert row data to column data + + if (*skey == INT64_MIN) { + *skey = dataRowKey(row); + } + + *ekey = dataRowKey(row); + + int32_t offset = 0; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, 0); + memcpy(pColInfo->pData + numOfRows*pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); + offset += pColInfo->info.bytes; + } + + numOfRows++; + if (numOfRows > maxRowsToRead) break; + }; + + return numOfRows; +} + +// copy data from cache into data block SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { - + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + STableIdInfo* idInfo = taosArrayGet(pHandle->pTableIdList, 0); + + STableId tableId = {.uid = idInfo->uid, .tid = idInfo->sid}; + STable *pTable = tsdbIsValidTableToInsert(tsdbGetMeta(pHandle->pTsdb), tableId); + assert(pTable != NULL); + + TSKEY skey = 0, ekey = 0; + int32_t rows = 0; + + if (pTable->mem != NULL) { + SSkipListIterator* iter = tSkipListCreateIter(pTable->mem->pData); + rows = tsdbReadRowsFromCache(iter, INT64_MAX, 4000, &skey, &ekey, pHandle); + } + + SDataBlockInfo blockInfo = { + .uid = tableId.uid, + .sid = tableId.tid, + .size = rows, + .window = {.skey = skey, .ekey = ekey} + }; + + return blockInfo; } +// return null for data block in cache int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SDataStatis **pBlockStatis) { - + *pBlockStatis = NULL; + return TSDB_CODE_SUCCESS; } SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList) { } -int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow* window, tsdbpos_t position, int16_t order) { +int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order) {} -} +int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos) {} -int32_t tsdbDataBlockSeek(tsdb_query_handle_t *pQueryHandle, tsdbpos_t pos) { +tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle) { return NULL; } -} +SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond) {} -tsdbpos_t tsdbDataBlockTell(tsdb_query_handle_t *pQueryHandle) { - return NULL; -} +tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) {} -SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond) { - -} - -tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) { - -} - -STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) { - -} - -STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond) { - -} +STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) {} +STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond) {} From 1575d9f70b447eb730bafd187b47fbcf2255d360 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Mon, 23 Mar 2020 13:47:52 +0800 Subject: [PATCH 18/85] TD-34 --- src/vnode/tsdb/tests/tsdbTests.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 459d531c10..e623425e85 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -88,7 +88,8 @@ TEST(TsdbTest, createRepo) { for (int k = 0; k < nRows/rowsPerSubmit; k++) { SSubmitBlk *pBlock = pMsg->blocks; - pBlock->tableId = {.uid = 987607499877672L, .tid = 0}; + pBlock->uid = 987607499877672L; + pBlock->tid = 0; pBlock->sversion = 0; pBlock->len = 0; for (int i = 0; i < rowsPerSubmit; i++) { From 09962c0039d36a64a9995c2dd34bd68b12ce8dab Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 23 Mar 2020 13:58:44 +0800 Subject: [PATCH 19/85] [td-32] refactor codes --- src/query/src/queryExecutor.c | 114 ++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 69ae0c06bd..04626227dd 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5469,34 +5469,60 @@ static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { return 0; } -static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList) { - pQueryTableMsg->vgId = htons(pQueryTableMsg->vgId); - pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); +static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArray** pTableIdList) { + assert(pQueryTableMsg->numOfTables > 0); + + *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + taosArrayPush(*pTableIdList, pTableIdInfo); + pMsg += sizeof(STableIdInfo); + + for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { + pTableIdInfo = (STableIdInfo *)pMsg; + + pTableIdInfo->sid = htonl(pTableIdInfo->sid); + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + taosArrayPush(*pTableIdList, pTableIdInfo); + pMsg += sizeof(STableIdInfo); + } + + return pMsg; +} - pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); - pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); +static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList) { + pQueryTableMsg->vgId = htons(pQueryTableMsg->vgId); + pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); - pQueryTableMsg->order = htons(pQueryTableMsg->order); - pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId); + pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); + pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); + pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); + pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); + + pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); + pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); + + pQueryTableMsg->order = htons(pQueryTableMsg->order); + pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId); - pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); - - pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); - pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); + pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); pQueryTableMsg->numOfTagsCols = htons(pQueryTableMsg->numOfTagsCols); - pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); + pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); - pQueryTableMsg->tagLength = htons(pQueryTableMsg->tagLength); + pQueryTableMsg->tagLength = htons(pQueryTableMsg->tagLength); - pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); - pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); - - pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); - pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); + pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); + pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); pQueryTableMsg->tsNumOfBlocks = htonl(pQueryTableMsg->tsNumOfBlocks); - pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); + pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); // query msg safety check if (validateQueryMeterMsg(pQueryTableMsg) != 0) { @@ -5506,23 +5532,23 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTa char *pMsg = (char *)(pQueryTableMsg->colList) + sizeof(SColumnInfo) * pQueryTableMsg->numOfCols; for (int32_t col = 0; col < pQueryTableMsg->numOfCols; ++col) { - pQueryTableMsg->colList[col].colId = htons(pQueryTableMsg->colList[col].colId); - pQueryTableMsg->colList[col].type = htons(pQueryTableMsg->colList[col].type); - pQueryTableMsg->colList[col].bytes = htons(pQueryTableMsg->colList[col].bytes); - pQueryTableMsg->colList[col].numOfFilters = htons(pQueryTableMsg->colList[col].numOfFilters); + SColumnInfo* pColInfo = &pQueryTableMsg->colList[col]; + + pColInfo->colId = htons(pColInfo->colId); + pColInfo->type = htons(pColInfo->type); + pColInfo->bytes = htons(pColInfo->bytes); + pColInfo->numOfFilters = htons(pColInfo->numOfFilters); - assert(pQueryTableMsg->colList[col].type >= TSDB_DATA_TYPE_BOOL && - pQueryTableMsg->colList[col].type <= TSDB_DATA_TYPE_NCHAR); - - int32_t numOfFilters = pQueryTableMsg->colList[col].numOfFilters; + assert(pColInfo->type >= TSDB_DATA_TYPE_BOOL && pColInfo->type <= TSDB_DATA_TYPE_NCHAR); + int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { - pQueryTableMsg->colList[col].filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); } for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; - SColumnFilterInfo *pDestFilterInfo = &pQueryTableMsg->colList[col].filters[f]; + SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; pDestFilterInfo->filterOnBinary = htons(pFilterInfo->filterOnBinary); @@ -5599,27 +5625,8 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTa pQueryTableMsg->colNameList = (int64_t)pMsg; pMsg += pQueryTableMsg->colNameLen; } - - *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableIdInfo->sid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - - for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { - pTableIdInfo = (STableIdInfo *)pMsg; - - pTableIdInfo->sid = htonl(pTableIdInfo->sid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - taosArrayPush(*pTableIdList, pTableIdInfo); - pMsg += sizeof(STableIdInfo); - } + + pMsg = createTableIdList(pQueryTableMsg, pMsg, pTableIdList); if (pQueryTableMsg->numOfGroupCols > 0 || pQueryTableMsg->numOfTagsCols > 0) { // group by tag columns pQueryTableMsg->pTagSchema = (uint64_t)pMsg; @@ -5653,9 +5660,7 @@ static int32_t convertQueryTableMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTa dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, timestamp order:%d, " "tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 - ", fillType:%d, comptslen:%d, limit:%" PRId64 - ", " - "offset:%" PRId64, + ", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64, pQueryTableMsg, pQueryTableMsg->numOfTables, pQueryTableMsg->window.skey, pQueryTableMsg->window.ekey, pQueryTableMsg->numOfGroupCols, pQueryTableMsg->numOfTagsCols, pQueryTableMsg->order, pQueryTableMsg->orderType, pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, @@ -6181,8 +6186,9 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ assert(pQueryTableMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; + SArray *pTableIdList = NULL; - if ((code = convertQueryTableMsg(pQueryTableMsg, &pTableIdList)) != TSDB_CODE_SUCCESS) { + if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList)) != TSDB_CODE_SUCCESS) { return code; } From 1b601eee7577c988a708866b4addefa1fac9427f Mon Sep 17 00:00:00 2001 From: hzcheng Date: Mon, 23 Mar 2020 15:07:59 +0800 Subject: [PATCH 20/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 2 +- src/vnode/tsdb/tests/tsdbTests.cpp | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index b07cce1b3b..2df7974844 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -712,7 +712,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListRandNodeInfo(pTable->mem->pData, &level, &headSize); TSKEY key = dataRowKey(row); - printf("insert:%lld, size:%d\n", key, pTable->mem->numOfPoints); + // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfPoints); // Copy row into the memory SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index e623425e85..a76aef2d41 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -79,7 +79,7 @@ TEST(TsdbTest, createRepo) { // // 3. Loop to write some simple data int nRows = 10000000; - int rowsPerSubmit = 100; + int rowsPerSubmit = 10; int64_t start_time = 1584081000000; SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit); @@ -108,7 +108,18 @@ TEST(TsdbTest, createRepo) { } pBlock->len += dataRowLen(row); } + pBlock->len = htonl(pBlock->len); + pBlock->numOfRows = htonl(pBlock->numOfRows); + pBlock->uid = htobe64(pBlock->uid); + pBlock->tid = htonl(pBlock->tid); + + pBlock->sversion = htonl(pBlock->sversion); + pBlock->padding = htonl(pBlock->padding); + pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + pMsg->compressed = htonl(pMsg->numOfBlocks); tsdbInsertData(pRepo, pMsg); } From 4271952deaf1d1c0ea3d4ec4ccb2e54352def555 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 15:52:56 +0800 Subject: [PATCH 21/85] [TD-15] refact the code to delete the database --- src/mnode/src/mgmtDb.c | 342 ++++++++++++++++++++++----------------- src/mnode/src/mgmtSdb.c | 197 +--------------------- src/mnode/src/mgmtUser.c | 13 +- 3 files changed, 210 insertions(+), 342 deletions(-) diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index e502d82a19..a4eeb88d91 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,11 +15,15 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "mgmtDb.h" +#include "taoserror.h" +#include "tstatus.h" +#include "tutil.h" +#include "name.h" +#include "mnode.h" #include "mgmtAcct.h" #include "mgmtBalance.h" #include "mgmtChildTable.h" +#include "mgmtDb.h" #include "mgmtDnode.h" #include "mgmtGrant.h" #include "mgmtShell.h" @@ -31,19 +35,13 @@ #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" -#include "mnode.h" - -#include "taoserror.h" -#include "tstatus.h" -#include "tutil.h" -#include "name.h" static void *tsDbSdb = NULL; static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(void *handle, void *tmrId); -static void mgmtSetDbDirty(SDbObj *pDb); +static int32_t mgmtSetDbDirty(SDbObj *pDb); static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -51,64 +49,73 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg); -static int32_t mgmtDbActionDestroy(void *pObj) { - tfree(pObj); +static int32_t mgmtDbActionDestroy(SSdbOperDesc *pOper) { + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionInsert(void *pObj) { - SDbObj *pDb = (SDbObj *) pObj; +static int32_t mgmtDbActionInsert(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); pDb->pHead = NULL; pDb->pTail = NULL; + pDb->prev = NULL; + pDb->next = NULL; pDb->numOfVgroups = 0; pDb->numOfTables = 0; - mgmtAddDbIntoAcct(pAcct, pDb); + pDb->numOfSuperTables = 0; + + if (pAcct != NULL) { + mgmtAddDbIntoAcct(pAcct, pDb); + } + else { + mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + return TSDB_CODE_INVALID_ACCT; + } return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionDelete(void *pObj) { - SDbObj *pDb = (SDbObj *) pObj; +static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtRemoveDbFromAcct(pAcct, pDb); + mgmtRemoveDbFromAcct(pAcct, pDb); mgmtDropAllNormalTables(pDb); mgmtDropAllChildTables(pDb); mgmtDropAllSuperTables(pDb); - + mgmtDropAllVgroups(pDb); + return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionUpdate(void *pObj) { +static int32_t mgmtDbActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtDbActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SDbObj *pDb = (SDbObj *)pObj; - if (maxRowSize < tsDbUpdateSize) { +static int32_t mgmtDbActionEncode(SSdbOperDesc *pOper) { + SDbObj *pDb = pOper->pObj; + + if (pOper->maxRowSize < tsDbUpdateSize) { return -1; } else { - memcpy(pData, pDb, tsDbUpdateSize); - return tsDbUpdateSize; + memcpy(pOper->rowData, pDb, tsDbUpdateSize); + pOper->rowSize = tsDbUpdateSize; + return TSDB_CODE_SUCCESS; } } -static void *mgmtDbActionDecode(void *pData) { - SDbObj *pDb = (SDbObj *) malloc(sizeof(SDbObj)); - if (pDb == NULL) return NULL; - memset(pDb, 0, sizeof(SDbObj)); - memcpy(pDb, pData, tsDbUpdateSize); - - return (void *)pDb; +static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { + SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj)); + if (pDb == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; + + memcpy(pDb, pOper->rowData, tsDbUpdateSize); + pOper->pObj = pDb; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitDbs() { - void * pNode = NULL; - SDbObj * pDb = NULL; - SAcctObj *pAcct = NULL; - SDbObj tObj; tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; @@ -116,7 +123,7 @@ int32_t mgmtInitDbs() { .tableName = "dbs", .hashSessions = TSDB_MAX_DBS, .maxRowSize = tsDbUpdateSize, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtDbActionInsert, .deleteFp = mgmtDbActionDelete, .updateFp = mgmtDbActionUpdate, @@ -131,25 +138,6 @@ int32_t mgmtInitDbs() { return -1; } - while (1) { - pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb); - if (pDb == NULL) break; - - pDb->pHead = NULL; - pDb->pTail = NULL; - pDb->prev = NULL; - pDb->next = NULL; - pDb->numOfTables = 0; - pDb->numOfVgroups = 0; - pDb->numOfSuperTables = 0; - pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct != NULL) - mgmtAddDbIntoAcct(pAcct, pDb); - else { - mError("db:%s acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); - } - } - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DB, mgmtProcessCreateDbMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_DB, mgmtProcessAlterDbMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mgmtProcessDropDbMsg); @@ -250,14 +238,14 @@ static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) { static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { // assign default parameters - if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; // - if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; // - if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; // - if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; // - if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; // - if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; // - if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; // - if (pCreate->compression < 0) pCreate->compression = tsCompression; // + if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; + if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; + if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; + if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; + if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; + if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; + if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; + if (pCreate->compression < 0) pCreate->compression = tsCompression; if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; if (pCreate->replications < 0) pCreate->replications = tsReplications; // if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // @@ -321,9 +309,17 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { pDb->createdTime = taosGetTimestampMs(); pDb->cfg = *pCreate; - if (sdbInsertRow(tsDbSdb, pDb, SDB_OPER_GLOBAL) < 0) { - code = TSDB_CODE_SDB_ERROR; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj) + }; + + code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { tfree(pDb); + code = TSDB_CODE_SDB_ERROR; } return code; @@ -337,72 +333,6 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } -static int32_t mgmtAlterDb(SAcctObj *pAcct, SCMAlterDbMsg *pAlter) { - return 0; -// int32_t code = TSDB_CODE_SUCCESS; -// -// SDbObj *pDb = (SDbObj *) sdbGetRow(tsDbSdb, pAlter->db); -// if (pDb == NULL) { -// mTrace("db:%s is not exist", pAlter->db); -// return TSDB_CODE_INVALID_DB; -// } -// -// int32_t oldReplicaNum = pDb->cfg.replications; -// if (pAlter->daysToKeep > 0) { -// mTrace("db:%s daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, pAlter->daysToKeep); -// pDb->cfg.daysToKeep = pAlter->daysToKeep; -// } else if (pAlter->replications > 0) { -// mTrace("db:%s replica:%d change to %d", pDb->name, pDb->cfg.replications, pAlter->replications); -// if (pAlter->replications < TSDB_REPLICA_MIN_NUM || pAlter->replications > TSDB_REPLICA_MAX_NUM) { -// mError("invalid db option replica: %d valid range: %d--%d", pAlter->replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); -// return TSDB_CODE_INVALID_OPTION; -// } -// pDb->cfg.replications = pAlter->replications; -// } else if (pAlter->maxSessions > 0) { -// mTrace("db:%s tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, pAlter->maxSessions); -// if (pAlter->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pAlter->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { -// mError("invalid db option tables: %d valid range: %d--%d", pAlter->maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); -// return TSDB_CODE_INVALID_OPTION; -// } -// if (pAlter->maxSessions < pDb->cfg.maxSessions) { -// mError("invalid db option tables: %d should larger than original:%d", pAlter->maxSessions, pDb->cfg.maxSessions); -// return TSDB_CODE_INVALID_OPTION; -// } -// return TSDB_CODE_INVALID_OPTION; -// //The modification of tables needs to rewrite the head file, so disable this option -// //pDb->cfg.maxSessions = pAlter->maxSessions; -// } else { -// mError("db:%s alter msg, replica:%d, keep:%d, tables:%d, origin replica:%d keep:%d", pDb->name, -// pAlter->replications, pAlter->maxSessions, pAlter->daysToKeep, -// pDb->cfg.replications, pDb->cfg.daysToKeep); -// return TSDB_CODE_INVALID_OPTION; -// } -// -// if (sdbUpdateRow(tsDbSdb, pDb, tsDbUpdateSize, 1) < 0) { -// return TSDB_CODE_SDB_ERROR; -// } -// -// SVgObj *pVgroup = pDb->pHead; -// while (pVgroup != NULL) { -// balanceUpdateVgroupState(pVgroup, TSDB_VG_LB_STATUS_UPDATE, 0); -// if (oldReplicaNum < pDb->cfg.replications) { -// if (!balanceAddVnode(pVgroup, NULL, NULL)) { -// mWarn("db:%s vgroup:%d not enough dnode to add vnode", pAlter->db, pVgroup->vgId); -// code = TSDB_CODE_NO_ENOUGH_DNODES; -// } -// } -// if (pAlter->maxSessions > 0) { -// //rebuild meterList in mgmtVgroup.c -// mgmtUpdateVgroup(pVgroup); -// } -//// mgmtSendCreateVnodeMsg(pVgroup); -// pVgroup = pVgroup->next; -// } -// mgmtStartBalanceTimer(10); -// -// return code; -} - int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { pVgroup->next = pDb->pHead; pVgroup->prev = NULL; @@ -472,7 +402,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -600,7 +530,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -char *mgmtGetDbStr(char *src) { +static char *mgmtGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); return ++pos; @@ -740,8 +670,23 @@ void mgmtRemoveTableFromDb(SDbObj *pDb) { atomic_add_fetch_32(&pDb->numOfTables, -1); } -static void mgmtSetDbDirty(SDbObj *pDb) { +static int32_t mgmtSetDbDirty(SDbObj *pDb) { + if (pDb->dirty) return TSDB_CODE_SUCCESS; + pDb->dirty = true; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + + return code; } static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { @@ -766,32 +711,121 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } else { code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate); if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is created by %s", pCreate->db, pMsg->pUser->user); + mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user); } } mgmtSendSimpleResp(pMsg->thandle, code); } +static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = pDb->cfg; + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t maxSessions = htonl(pAlter->maxSessions) + 1; + int8_t replications = pAlter->replications; + + terrno = TSDB_CODE_SUCCESS; + + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { + mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); + newCfg.daysToKeep = daysToKeep; + } else if (replications > 0 && replications != pDb->cfg.replications) { + mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); + if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.replications = replications; + } else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions); + if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + terrno = TSDB_CODE_INVALID_OPTION; + } + if (maxSessions < pDb->cfg.maxSessions) { + mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions); + terrno = TSDB_CODE_INVALID_OPTION; + } + newCfg.maxSessions = maxSessions; + } else { + } + + return newCfg; +} + +static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { + SDbCfg newCfg = mgmtGetAlterDbOption(pDb, pAlter); + if (terrno != TSDB_CODE_SUCCESS) { + return terrno; + } + + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { + pDb->cfg = newCfg; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = tsDbUpdateSize + }; + + int32_t code = sdbUpdateRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_SDB_ERROR; + } + } + + return TSDB_CODE_SUCCESS; +} + static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { if (mgmtCheckRedirect(pMsg->thandle)) return; SCMAlterDbMsg *pAlter = pMsg->pCont; - pAlter->daysPerFile = htonl(pAlter->daysPerFile); - pAlter->daysToKeep = htonl(pAlter->daysToKeep); - pAlter->maxSessions = htonl(pAlter->maxSessions) + 1; + mTrace("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->thandle); - int32_t code; - if (!pMsg->pUser->writeAuth) { - code = TSDB_CODE_NO_RIGHTS; - } else { - code = mgmtAlterDb(pMsg->pUser->pAcct, pAlter); - if (code == TSDB_CODE_SUCCESS) { - mLPrint("DB:%s is altered by %s", pAlter->db, pMsg->pUser->user); - } + if (mgmtCheckExpired()) { + mError("db:%s, failed to alter, grant expired", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); + return; } - mgmtSendSimpleResp(pMsg->thandle, code); + if (!pMsg->pUser->writeAuth) { + mError("db:%s, failed to alter, no rights", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); + return; + } + + SDbObj *pDb = mgmtGetDb(pAlter->db); + if (pDb == NULL) { + mError("db:%s, failed to alter, invalid db", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + return; + } + + int32_t code = mgmtAlterDb(pDb, pAlter); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to alter, invalid db option", pAlter->db); + mgmtSendSimpleResp(pMsg->thandle, code); + } + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + SVgObj *pVgroup = pDb->pHead; + if (pVgroup != NULL) { + mPrint("vgroup:%d, will be altered", pVgroup->vgId); + newMsg->ahandle = pVgroup; + newMsg->expected = pVgroup->numOfVnodes; + mgmtAlterVgroup(pVgroup, newMsg); + return; + } + + mTrace("db:%s, all vgroups is altered", pDb->name); + + mgmtSendSimpleResp(newMsg->thandle, TSDB_CODE_SUCCESS); + rpcFreeCont(newMsg->pCont); + free(newMsg); } static void mgmtDropDb(void *handle, void *tmrId) { @@ -799,7 +833,12 @@ static void mgmtDropDb(void *handle, void *tmrId) { SDbObj *pDb = newMsg->ahandle; mPrint("db:%s, drop db from sdb", pDb->name); - int32_t code = sdbDeleteRow(tsDbSdb, pDb, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsDbSdb, + .pObj = pDb + }; + int32_t code = sdbDeleteRow(&oper); if (code != 0) { code = TSDB_CODE_SDB_ERROR; } @@ -846,7 +885,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { return; } - mgmtSetDbDirty(pDb); + int32_t code = mgmtSetDbDirty(pDb); + if (code != TSDB_CODE_SUCCESS) { + mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code)); + mgmtSendSimpleResp(pMsg->thandle, code); + return; + } SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index ea05cb59f4..eee8c0f1a0 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -104,9 +104,6 @@ static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, s static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash}; static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData}; -void sdbResetTable(SSdbTable *pTable); -void sdbSaveSnapShot(void *handle); - uint64_t sdbGetVersion() { return sdbVersion; } int64_t sdbGetId(void *handle) { return ((SSdbTable *)handle)->version; } int64_t sdbGetNumOfRows(void *handle) { return ((SSdbTable *)handle)->numOfRows; } @@ -388,10 +385,6 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { pTable->autoIndex = maxAutoIndex; } - if (numOfChanged > pTable->hashSessions / 4) { - sdbSaveSnapShot(pTable); - } - tfree(rowHead); return 0; } @@ -538,7 +531,7 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, a record is inserted:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, + sdbTrace("table:%s, inserte record:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); (*pTable->insertFp)(pOper); @@ -578,7 +571,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { } } - int32_t total_size = sizeof(SRowHead) + pOper->rowSize + sizeof(TSCKSUM); + int32_t total_size = sizeof(SRowHead) + pMeta->rowSize + sizeof(TSCKSUM); SRowHead *rowHead = (SRowHead *)calloc(1, total_size); if (rowHead == NULL) { sdbError("failed to allocate row head memory, sdb:%s", pTable->tableName); @@ -621,7 +614,7 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { tfree(rowHead); - sdbTrace("table:%s, a record is deleted:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%d", + sdbTrace("table:%s, delete record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); // Delete from current layer @@ -644,7 +637,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { SRowMeta *pMeta = sdbGetRowMeta(pTable, pOper->pObj); if (pMeta == NULL) { - sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " id:%" PRId64, + sdbError("table:%s, failed to update record:%s, record is not there, sdbversion:%" PRId64 " version:%" PRId64, pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version); return -1; } @@ -697,7 +690,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { rowHead->delimiter = SDB_DELIMITER; rowHead->version = pTable->version; if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum, sdb:%s version:%d", pTable->tableName, rowHead->version); + sdbError("table:%s, failed to get checksum, version:%d", pTable->tableName, rowHead->version); pTable->version--; sdbVersion--; pthread_mutex_unlock(&pTable->mutex); @@ -709,7 +702,7 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { pTable->fileSize += real_size; sdbFinishCommit(pTable); - sdbTrace("table:%s, a record is updated:%s, sdbversion:%" PRId64 " id:%" PRId64 " numOfRows:%" PRId64, + sdbTrace("table:%s, update record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%" PRId64, pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); pMeta->version = pTable->version; @@ -750,187 +743,11 @@ void sdbCloseTable(void *handle) { pthread_mutex_destroy(&pTable->mutex); sdbNumOfTables--; - sdbTrace("table:%s is closed, id:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); + sdbTrace("table:%s, is closed, version:%" PRId64 " numOfTables:%d", pTable->tableName, pTable->version, sdbNumOfTables); tfree(pTable); } -void sdbResetTable(SSdbTable *pTable) { - /* - SRowMeta rowMeta; - int32_t bytes; - int32_t total_size = 0; - int32_t real_size = 0; - SRowHead *rowHead = NULL; - void * pMetaRow = NULL; - int64_t oldId = pTable->version; - int32_t oldNumOfRows = pTable->numOfRows; - - if (sdbOpenSdbFile(pTable) < 0) return; - pTable->numOfRows = oldNumOfRows; - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate row head memory for reset, sdb:%s", pTable->tableName); - return; - } - - sdbPrint("open sdb file:%s for reset table", pTable->fileName); - - while (1) { - memset(rowHead, 0, total_size); - - bytes = read(pTable->fd, rowHead, sizeof(SRowHead)); - if (bytes < 0) { - sdbError("failed to read sdb file:%s", pTable->fileName); - tfree(rowHead); - return; - } - - if (bytes == 0) break; - - if (bytes < sizeof(SRowHead) || rowHead->delimiter != SDB_DELIMITER) { - pTable->fileSize++; - lseek(pTable->fd, -(bytes - 1), SEEK_CUR); - continue; - } - - if (rowHead->rowSize < 0 || rowHead->rowSize > pTable->maxRowSize) { - sdbError("error row size in sdb file:%s for reset, version:%d rowSize:%d maxRowSize:%d", - pTable->fileName, rowHead->version, rowHead->rowSize, pTable->maxRowSize); - pTable->fileSize += sizeof(SRowHead); - continue; - } - - bytes = read(pTable->fd, rowHead->data, rowHead->rowSize + sizeof(TSCKSUM)); - if (bytes < rowHead->rowSize + sizeof(TSCKSUM)) { - sdbError("failed to read sdb file:%s for reset, version:%d rowSize:%d", pTable->fileName, rowHead->version, rowHead->rowSize); - break; - } - - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)rowHead, real_size)) { - sdbError("error sdb checksum, sdb:%s version:%d, skip", pTable->tableName, rowHead->version); - pTable->fileSize += real_size; - continue; - } - - if (abs(rowHead->version) > oldId) { // not operated - pMetaRow = sdbGetRow(pTable, rowHead->data); - if (pMetaRow == NULL) { // New object - if (rowHead->version < 0) { - sdbError("error sdb negative version:%d, sdb:%s, skip", rowHead->version, pTable->tableName); - } else { - rowMeta.version = rowHead->version; - // TODO:Get rid of the rowMeta.offset and rowSize - rowMeta.offset = pTable->fileSize; - rowMeta.rowSize = rowHead->rowSize; - rowMeta.row = (*pTable->decodeFp)(rowHead->data); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - pTable->numOfRows++; - - (*pTable->insertFp)(rowMeta.row); - } - } else { // already exists - if (rowHead->version < 0) { // Delete the object - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); - (*pTable->destroyFp)(pMetaRow); - pTable->numOfRows--; - } else { // update the object - (*pTable->updateFp)(pMetaRow); - } - } - } - - pTable->fileSize += real_size; - if (pTable->version < abs(rowHead->version)) pTable->version = abs(rowHead->version); - } - - sdbVersion += (pTable->version - oldId); - - tfree(rowHead); - - sdbPrint("table:%s is updated, sdbVerion:%" PRId64 " id:%" PRId64, pTable->tableName, sdbVersion, pTable->version); - */ -} - -// TODO:A problem here :use snapshot file to sync another node will cause problem -void sdbSaveSnapShot(void *handle) { - /* - SSdbTable *pTable = (SSdbTable *)handle; - SRowMeta * pMeta; - void * pNode = NULL; - int32_t total_size = 0; - int32_t real_size = 0; - int32_t size = 0; - int32_t numOfRows = 0; - uint32_t sdbEcommit = SDB_ENDCOMMIT; - char * dirc = NULL; - char * basec = NULL; - - if (pTable == NULL) return; - - sdbTrace("Table:%s, save the snapshop", pTable->tableName); - - char fn[128] = "\0"; - dirc = strdup(pTable->fileName); - basec = strdup(pTable->fileName); - sprintf(fn, "%s/.%s", dirname(dirc), basename(basec)); - int32_t fd = open(fn, O_RDWR | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - tfree(dirc); - tfree(basec); - - total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); - SRowHead *rowHead = (SRowHead *)malloc(total_size); - if (rowHead == NULL) { - sdbError("failed to allocate memory while saving SDB snapshot, sdb:%s", pTable->tableName); - return; - } - memset(rowHead, 0, size); - - // Write the header - twrite(fd, &(pTable->header), sizeof(SSdbHeader)); - size += sizeof(SSdbHeader); - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - - while (1) { - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; - - rowHead->delimiter = SDB_DELIMITER; - rowHead->version = pMeta->id; - rowHead->rowSize = (*pTable->encodeFp)(pMeta->row, rowHead->data, pTable->maxRowSize); - real_size = sizeof(SRowHead) + rowHead->rowSize + sizeof(TSCKSUM); - if (taosCalcChecksumAppend(0, (uint8_t *)rowHead, real_size) < 0) { - sdbError("failed to get checksum while save sdb %s snapshot", pTable->tableName); - tfree(rowHead); - return; - } - - twrite(fd, rowHead, real_size); - size += real_size; - twrite(fd, &sdbEcommit, sizeof(sdbEcommit)); - size += sizeof(sdbEcommit); - numOfRows++; - } - - tfree(rowHead); - - // Remove the old file - tclose(pTable->fd); - remove(pTable->fileName); - // Rename the .sdb.db file to sdb.db file - rename(fn, pTable->fileName); - pTable->fd = fd; - pTable->fileSize = size; - pTable->numOfRows = numOfRows; - - fdatasync(pTable->fd); - */ -} - void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { SSdbTable *pTable = (SSdbTable *)handle; SRowMeta * pMeta; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 11285764b8..e55ba9ea60 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -46,7 +46,15 @@ static int32_t mgmtUserActionDestroy(SSdbOperDesc *pOper) { static int32_t mgmtUserActionInsert(SSdbOperDesc *pOper) { SUserObj *pUser = pOper->pObj; SAcctObj *pAcct = mgmtGetAcct(pUser->acct); - mgmtAddUserIntoAcct(pAcct, pUser); + + if (pAcct != NULL) { + mgmtAddUserIntoAcct(pAcct, pUser); + } + else { + mError("user:%s, acct:%s info not exist in sdb", pUser->user, pUser->acct); + return TSDB_CODE_INVALID_ACCT; + } + return TSDB_CODE_SUCCESS; } @@ -77,7 +85,7 @@ static int32_t mgmtUserActionEncode(SSdbOperDesc *pOper) { static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { SUserObj *pUser = (SUserObj *) calloc(1, sizeof(SUserObj)); - if (pUser == NULL) return -1; + if (pUser == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; memcpy(pUser, pOper->rowData, tsUserUpdateSize); pOper->pObj = pUser; @@ -216,7 +224,6 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { int32_t code = sdbDeleteRow(&oper); if (code != TSDB_CODE_SUCCESS) { - tfree(pUser); code = TSDB_CODE_SDB_ERROR; } From 00e18b70d7e18c2aab8394c08ba9e869a66eacf9 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 19:07:57 +0800 Subject: [PATCH 22/85] [TD-15] refact code of tidpool --- src/inc/mnode.h | 3 + src/mnode/inc/mgmtDb.h | 10 +- src/mnode/inc/mgmtProfile.h | 2 + src/mnode/inc/mgmtVgroup.h | 5 +- src/mnode/src/mgmtAcct.c | 6 +- src/mnode/src/mgmtDb.c | 39 ++--- src/mnode/src/mgmtDnode.c | 5 - src/mnode/src/mgmtProfile.c | 10 ++ src/mnode/src/mgmtVgroup.c | 339 +++++++++++++++++++----------------- src/util/inc/tidpool.h | 6 +- src/util/src/tidpool.c | 139 ++++++--------- 11 files changed, 283 insertions(+), 281 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 1452e1f53a..cde26dad4a 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -159,6 +159,8 @@ typedef struct { SSchema* schema; } SNormalTableObj; +struct _db_obj; + typedef struct _vg_obj { uint32_t vgId; char dbName[TSDB_DB_NAME_LEN + 1]; @@ -172,6 +174,7 @@ typedef struct _vg_obj { int8_t reserved[14]; int8_t updateEnd[1]; struct _vg_obj *prev, *next; + struct _db_obj *pDb; void * idPool; STableInfo ** tableList; } SVgObj; diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index 32bb9d9ec6..f2049f836c 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,11 +22,11 @@ extern "C" { #include "mnode.h" -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup); -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup); +void mgmtAddVgroupIntoDb(SVgObj *pVgroup); +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup); +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup); +void mgmtMoveVgroupToTail(SVgObj *pVgroup); +void mgmtMoveVgroupToHead(SVgObj *pVgroup); int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); diff --git a/src/mnode/inc/mgmtProfile.h b/src/mnode/inc/mgmtProfile.h index 7f9fd9622c..8c2e073c2d 100644 --- a/src/mnode/inc/mgmtProfile.h +++ b/src/mnode/inc/mgmtProfile.h @@ -28,6 +28,8 @@ bool mgmtCheckQhandle(uint64_t qhandle); void mgmtSaveQhandle(void *qhandle); void mgmtFreeQhandle(void *qhandle); +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index b7c68b5f80..3379a93f28 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -28,13 +28,12 @@ int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode); +void mgmtDropAllVgroups(SDbObj *pDropDb); void mgmtCreateVgroup(SQueuedMsg *pMsg); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); void mgmtUpdateVgroup(SVgObj *pVgroup); -void mgmtUpdateVgroupIp(SDnodeObj *pDnode); - -void mgmtSetVgroupIdPool(); +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index b1ee72386e..0075f53319 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -85,8 +85,10 @@ int32_t mgmtRemoveUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { pUser->prev->next = pUser->next; } - if (pUser->next) pUser->next->prev = pUser->prev; - + if (pUser->next) { + pUser->next->prev = pUser->prev; + } + if (pUser->prev == NULL) { pAcct->pUser = pUser->next; } diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index a4eeb88d91..e8ab5aaa8c 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -333,7 +333,8 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) { return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb)); } -int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = pDb->pHead; pVgroup->prev = NULL; @@ -342,11 +343,10 @@ int32_t mgmtAddVgroupIntoDb(SDbObj *pDb, SVgObj *pVgroup) { pDb->pHead = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; pVgroup->next = NULL; pVgroup->prev = pDb->pTail; @@ -355,32 +355,25 @@ int32_t mgmtAddVgroupIntoDbTail(SDbObj *pDb, SVgObj *pVgroup) { pDb->pTail = pVgroup; pDb->numOfVgroups++; - - return 0; } -int32_t mgmtRemoveVgroupFromDb(SDbObj *pDb, SVgObj *pVgroup) { +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup) { + SDbObj *pDb = pVgroup->pDb; if (pVgroup->prev) pVgroup->prev->next = pVgroup->next; if (pVgroup->next) pVgroup->next->prev = pVgroup->prev; if (pVgroup->prev == NULL) pDb->pHead = pVgroup->next; if (pVgroup->next == NULL) pDb->pTail = pVgroup->prev; pDb->numOfVgroups--; - - return 0; } -int32_t mgmtMoveVgroupToTail(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToTail(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDbTail(pVgroup); } -int32_t mgmtMoveVgroupToHead(SDbObj *pDb, SVgObj *pVgroup) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); - mgmtAddVgroupIntoDb(pDb, pVgroup); - - return 0; +void mgmtMoveVgroupToHead(SVgObj *pVgroup) { + mgmtRemoveVgroupFromDb(pVgroup); + mgmtAddVgroupIntoDb(pVgroup); } void mgmtCleanUpDbs() { @@ -600,7 +593,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (strcmp(pUser->user, "root") == 0) { #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxSessions - 1; // table num can be created should minus 1 + *(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -613,9 +606,9 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; #ifdef _TD_ARM_32_ - *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #else - *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions - 1)); + *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); #endif cols++; @@ -721,7 +714,7 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { SDbCfg newCfg = pDb->cfg; int32_t daysToKeep = htonl(pAlter->daysToKeep); - int32_t maxSessions = htonl(pAlter->maxSessions) + 1; + int32_t maxSessions = htonl(pAlter->maxSessions); int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index f088ac75a4..0d83e262ae 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -566,11 +566,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { mgmtSetDnodeMaxVnodes(pDnode); } - if (lastPrivateIp != pDnode->privateIp || lastPublicIp != pDnode->publicIp) { - mgmtUpdateVgroupIp(pDnode); - //mgmtUpdateMnodeIp(); - } - int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { pDnode->vload[j].vgId = htonl(pStatus->load[j].vgId); diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index b20e66dd87..1b98ac9596 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -762,3 +762,13 @@ int32_t mgmtInitProfile() { void mgmtCleanUpProfile() { } + +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) { + if (pMsg != NULL) { + if (pMsg->pCont != NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + free(pMsg); + } +} \ No newline at end of file diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 5b1968259f..b37b363077 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -23,62 +23,90 @@ #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtTable.h" #include "mgmtVgroup.h" -static void *tsVgroupSdb = NULL; +static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg); +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) ; static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); static void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); -static int32_t mgmtVgroupActionDestroy(void *pObj) { - SVgObj *pVgroup = (SVgObj *) pObj; +static int32_t mgmtVgroupActionDestroy(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; if (pVgroup->idPool) { taosIdPoolCleanUp(pVgroup->idPool); pVgroup->idPool = NULL; } - if (pVgroup->tableList) tfree(pVgroup->tableList); - tfree(pObj); + if (pVgroup->tableList) { + tfree(pVgroup->tableList); + } + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionInsert(void *pObj) { - SVgObj *pVgroup = pObj; +static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + if (pDb == NULL) { + return TSDB_CODE_INVALID_DB; + } + + pVgroup->pDb = pDb; + pVgroup->prev = NULL; + pVgroup->next = NULL; + + int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; + pVgroup->tableList = (STableInfo **)calloc(pDb->cfg.maxSessions, sizeof(STableInfo *)); + if (pVgroup->tableList == NULL) { + mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); + return -1; + } + + pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + if (pVgroup->idPool == NULL) { + mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); + tfree(pVgroup->tableList); + return -1; + } + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); + pVgroup->vnodeGid[i].privateIp = pDnode->privateIp; + pVgroup->vnodeGid[i].publicIp = pDnode->publicIp; pVgroup->vnodeGid[i].vnode = pVgroup->vgId; } + mgmtAddVgroupIntoDb(pVgroup); + return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionDelete(void *pObj) { - SVgObj *pVgroup = pObj; - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - - if (pDb != NULL) { - mgmtRemoveVgroupFromDb(pDb, pVgroup); +static int32_t mgmtVgroupActionDelete(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + + if (pVgroup->pDb != NULL) { + mgmtRemoveVgroupFromDb(pVgroup); } - // mgmtUnSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes); - tfree(pVgroup->tableList); - return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionUpdate(void *pObj) { - SVgObj *pVgroup = (SVgObj *) pObj; +static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + SDbObj *pDb = pVgroup->pDb; if (pDb != NULL) { if (pDb->cfg.maxSessions != oldTables) { mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); @@ -88,33 +116,31 @@ static int32_t mgmtVgroupActionUpdate(void *pObj) { } } - mTrace("vgroup:%d update, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); + mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes); return TSDB_CODE_SUCCESS; } -static int32_t mgmtVgroupActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SVgObj *pVgroup = (SVgObj *) pObj; - if (maxRowSize < tsVgUpdateSize) { +static int32_t mgmtVgroupActionEncode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = pOper->pObj; + if (pOper->maxRowSize < tsVgUpdateSize) { return -1; } else { - memcpy(pData, pVgroup, tsVgUpdateSize); - return tsVgUpdateSize; + memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); + pOper->rowSize = tsVgUpdateSize; + return TSDB_CODE_SUCCESS; } } -static void *mgmtVgroupActionDecode(void *pObj) { - SVgObj *pVgroup = (SVgObj *) malloc(sizeof(SVgObj)); - if (pVgroup == NULL) return NULL; - memset(pVgroup, 0, sizeof(SVgObj)); - memcpy(pVgroup, pObj, tsVgUpdateSize); +static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { + SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj)); + if (pVgroup == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - return pVgroup; + memcpy(pVgroup, pOper->rowData, tsVgUpdateSize); + pOper->pObj = pVgroup; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitVgroups() { - void *pNode = NULL; - SVgObj *pVgroup = NULL; - SVgObj tObj; tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; @@ -122,7 +148,7 @@ int32_t mgmtInitVgroups() { .tableName = "vgroups", .hashSessions = TSDB_MAX_VGROUPS, .maxRowSize = tsVgUpdateSize, - .keyType = SDB_KEYTYPE_AUTO, + .keyType = SDB_KEY_TYPE_AUTO, .insertFp = mgmtVgroupActionInsert, .deleteFp = mgmtVgroupActionDelete, .updateFp = mgmtVgroupActionUpdate, @@ -137,47 +163,11 @@ int32_t mgmtInitVgroups() { return -1; } - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL) break; - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) continue; - - pVgroup->prev = NULL; - pVgroup->next = NULL; - - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)malloc(size); - if (pVgroup->tableList == NULL) { - mError("failed to malloc(size:%d) for the tableList of vgroups", size); - return -1; - } - - memset(pVgroup->tableList, 0, size); - - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); - if (pVgroup->idPool == NULL) { - mError("failed to taosInitIdPool for vgroups"); - free(pVgroup->tableList); - return -1; - } - - taosIdPoolReinit(pVgroup->idPool); - - if (tsIsCluster && pVgroup->vnodeGid[0].publicIp == 0) { - pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); - pVgroup->vnodeGid[0].privateIp = inet_addr(tsPrivateIp); - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); - } - - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); - } - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mgmtGetVgroupMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mgmtRetrieveVgroups); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mgmtProcessCreateVnodeRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mgmtProcessDropVnodeRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mgmtProcessVnodeCfgMsg); mTrace("vgroup is initialized"); return 0; @@ -196,32 +186,41 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg) { if (pDb == NULL) { mError("failed to create vgroup, db not found"); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB); + mgmtFreeQueuedMsg(pMsg); return; } - SVgObj *pVgroup = (SVgObj *)calloc(sizeof(SVgObj), 1); + SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj)); strcpy(pVgroup->dbName, pDb->name); pVgroup->numOfVnodes = pDb->cfg.replications; + pVgroup->createdTime = taosGetTimestampMs(); if (mgmtAllocVnodes(pVgroup) != 0) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup", pDb->name, pVgroup->numOfVnodes); free(pVgroup); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_ENOUGH_DNODES); + mgmtFreeQueuedMsg(pMsg); return; } - pVgroup->createdTime = taosGetTimestampMs(); - pVgroup->tableList = (STableInfo **) calloc(sizeof(STableInfo *), pDb->cfg.maxSessions); - pVgroup->numOfTables = 0; - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj) + }; - mgmtAddVgroupIntoDb(pDb, pVgroup); - // mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); - - sdbInsertRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + tfree(pVgroup); + code = TSDB_CODE_SDB_ERROR; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); + mgmtFreeQueuedMsg(pMsg); + return; + } mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - mPrint("vgroup:%d, dnode:%d vnode:%d", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); + mPrint("vgroup:%d, index:%d, dnode:%d vnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId, pVgroup->vnodeGid[i].vnode); } pMsg->ahandle = pVgroup; @@ -235,28 +234,12 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) { } else { mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes); mgmtSendDropVgroupMsg(pVgroup, NULL); - sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); - } -} - -void mgmtSetVgroupIdPool() { - void * pNode = NULL; - SVgObj *pVgroup = NULL; - SDbObj *pDb; - - while (1) { - pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); - if (pVgroup == NULL || pVgroup->idPool == 0) break; - - taosIdPoolSetFreeList(pVgroup->idPool); - pVgroup->numOfTables = taosIdPoolNumOfUsed(pVgroup->idPool); - - pDb = mgmtGetDb(pVgroup->dbName); - pDb->numOfTables += pVgroup->numOfTables; - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1) - mgmtAddVgroupIntoDbTail(pDb, pVgroup); - else - mgmtAddVgroupIntoDb(pDb, pVgroup); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + sdbDeleteRow(&oper); } } @@ -287,7 +270,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vgroup_status"); + strcpy(pSchema[cols].name, "vgroup status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -327,13 +310,13 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pShow->bytes[cols] = 9; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "vnode_status"); + strcpy(pSchema[cols].name, "vnode status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 16; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public_ip"); + strcpy(pSchema[cols].name, "public ip"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -445,25 +428,36 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -void mgmtUpdateVgroup(SVgObj *pVgroup) { - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_LOCAL); -} - void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables++; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) { pVgroup->tableList[pTable->sid] = pTable; + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables++; + pVgroup->pDb->numOfTables++; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); + else + mgmtAddVgroupIntoDb(pVgroup); } void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { - pVgroup->numOfTables--; - if (pTable->sid >= 0) + if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) { pVgroup->tableList[pTable->sid] = NULL; - taosFreeId(pVgroup->idPool, pTable->sid); + taosFreeId(pVgroup->idPool, pTable->sid); + pVgroup->numOfTables--; + pVgroup->pDb->numOfTables--; + } + + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + mgmtAddVgroupIntoDbTail(pVgroup); + else + mgmtAddVgroupIntoDb(pVgroup); } SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); + SDbObj *pDb = pVgroup->pDb; if (pDb == NULL) return NULL; SMDCreateVnodeMsg *pVnode = rpcMallocCont(sizeof(SMDCreateVnodeMsg)); @@ -487,6 +481,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SVnodeDesc *vpeerDesc = pVnode->vpeerDesc; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { + vpeerDesc[j].vnode = htonl(pVgroup->vnodeGid[j].vnode); vpeerDesc[j].ip = htonl(pVgroup->vnodeGid[j].privateIp); } @@ -576,13 +571,23 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { newMsg->contLen = queueMsg->contLen; newMsg->pCont = rpcMallocCont(newMsg->contLen); memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); + queueMsg->pCont = NULL; mgmtAddToShellQueue(newMsg); } else { - sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); } - free(queueMsg); + mgmtFreeQueuedMsg(queueMsg); } static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) { @@ -632,7 +637,15 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { if (queueMsg->received != queueMsg->expected) return; - sdbDeleteRow(tsVgroupSdb, pVgroup, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsVgroupSdb, + .pObj = pVgroup + }; + int32_t code = sdbDeleteRow(&oper); + if (code != 0) { + code = TSDB_CODE_SDB_ERROR; + } SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); newMsg->msgType = queueMsg->msgType; @@ -644,50 +657,64 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) { memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); mgmtAddToShellQueue(newMsg); - free(queueMsg); + queueMsg->pCont = NULL; + mgmtFreeQueuedMsg(queueMsg); } -void mgmtUpdateVgroupIp(SDnodeObj *pDnode) { - void * pNode = NULL; +static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + + SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); + if (pVgroup == NULL) { + mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); +} + +void mgmtDropAllVgroups(SDbObj *pDropDb) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; + while (1) { pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *vnodeGid = pVgroup->vnodeGid + i; - if (vnodeGid->dnodeId == pDnode->dnodeId) { - mPrint("vgroup:%d, dnode:%d, privateIp:%s change to %s, publicIp:%s change to %s", - pVgroup->vgId, vnodeGid->dnodeId, pDnode->privateIp, taosIpStr(vnodeGid->privateIp), - pDnode->publicIp, taosIpStr(vnodeGid->publicIp)); - vnodeGid->publicIp = pDnode->publicIp; - vnodeGid->privateIp = pDnode->privateIp; - sdbUpdateRow(tsVgroupSdb, pVgroup, tsVgUpdateSize, SDB_OPER_GLOBAL); - } + if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsVgroupSdb, + .pObj = pVgroup, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; } } + + mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables); } -//static void mgmtProcessVnodeCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *pConn) { -// if (!sdbMaster) { -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// -// SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); -// if (pVgroup == NULL) { -// mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_NOT_ACTIVE_VNODE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(pConn, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateVnodeMsg(pVgroup, pCfg->vnode, &ipSet, NULL); -//} -// \ No newline at end of file +void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { + assert(ahandle != NULL); + + if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { + // TODO: + // mgmtSendAlterVgroupMsg(pVgroup, NULL); + } else { + mgmtAddToShellQueue(ahandle); + } +} \ No newline at end of file diff --git a/src/util/inc/tidpool.h b/src/util/inc/tidpool.h index 08f9d36467..bf35251631 100644 --- a/src/util/inc/tidpool.h +++ b/src/util/inc/tidpool.h @@ -34,11 +34,7 @@ void taosIdPoolCleanUp(void *handle); int taosIdPoolNumOfUsed(void *handle); -void taosIdPoolReinit(void *handle); - -void taosIdPoolMarkStatus(void *handle, int id, int status); - -void taosIdPoolSetFreeList(void *handle); +void taosIdPoolMarkStatus(void *handle, int id); #ifdef __cplusplus } diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index c50c38aa3c..c3f429ac43 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -25,78 +25,74 @@ typedef struct { } id_pool_t; void *taosInitIdPool(int maxId) { - id_pool_t *pIdPool; - int * idList, i; - - if (maxId < 3) maxId = 3; - - pIdPool = (id_pool_t *)malloc(sizeof(id_pool_t)); + id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); - if (idList == NULL) { + pIdPool->freeList = malloc(sizeof(int) * (size_t)maxId); + if (pIdPool->freeList == NULL) { free(pIdPool); return NULL; } - memset(pIdPool, 0, sizeof(id_pool_t)); pIdPool->maxId = maxId; - pIdPool->numOfFree = maxId - 1; + pIdPool->numOfFree = maxId; pIdPool->freeSlot = 0; - pIdPool->freeList = idList; pthread_mutex_init(&pIdPool->mutex, NULL); - for (i = 1; i < maxId; ++i) idList[i - 1] = i; + for (int i = 0; i < maxId; ++i) { + pIdPool->freeList[i] = -1; + } pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); - return (void *)pIdPool; + return pIdPool; } int taosAllocateId(void *handle) { - id_pool_t *pIdPool; - int id = -1; - if (handle == NULL) return id; - - pIdPool = (id_pool_t *)handle; - - if (pIdPool->maxId < 3) pError("pool:%p is messed up, maxId:%d", pIdPool, pIdPool->maxId); - - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); - - if (pIdPool->numOfFree > 0) { - id = pIdPool->freeList[pIdPool->freeSlot]; - pIdPool->freeSlot = (pIdPool->freeSlot + 1) % pIdPool->maxId; - pIdPool->numOfFree--; + id_pool_t *pIdPool = handle; + if (handle == NULL) { + return -1; } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + int slot = -1; + pthread_mutex_lock(&pIdPool->mutex); - return id; + if (pIdPool->numOfFree > 0) { + for (int i = 0; i < pIdPool->maxId; ++i) { + slot = (i + pIdPool->freeSlot) % pIdPool->maxId; + if (pIdPool->freeList[slot] == -1) { + pIdPool->freeList[slot] = slot; + pIdPool->freeSlot = slot + 1; + pIdPool->numOfFree--; + break; + } + } + } + + pthread_mutex_unlock(&pIdPool->mutex); + return slot; } void taosFreeId(void *handle, int id) { - id_pool_t *pIdPool; - int slot; + id_pool_t *pIdPool = handle; + if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; - if (pIdPool->freeList == NULL || pIdPool->maxId == 0) return; - if (id <= 0 || id >= pIdPool->maxId) return; - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); + pthread_mutex_lock(&pIdPool->mutex); - slot = (pIdPool->freeSlot + pIdPool->numOfFree) % pIdPool->maxId; - pIdPool->freeList[slot] = id; - pIdPool->numOfFree++; + int slot = id % pIdPool->maxId; + if (pIdPool->freeList[slot] != -1) { + pIdPool->freeList[slot] = -1; + pIdPool->numOfFree++; + } - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); } void taosIdPoolCleanUp(void *handle) { - id_pool_t *pIdPool; + id_pool_t *pIdPool = handle; - if (handle == NULL) return; - pIdPool = (id_pool_t *)handle; + if (pIdPool == NULL) return; pTrace("pool:%p is cleaned", pIdPool); @@ -110,42 +106,21 @@ void taosIdPoolCleanUp(void *handle) { } int taosIdPoolNumOfUsed(void *handle) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - return pIdPool->maxId - pIdPool->numOfFree - 1; + id_pool_t *pIdPool = handle; + return pIdPool->maxId - pIdPool->numOfFree; } -void taosIdPoolReinit(void *handle) { - id_pool_t *pIdPool; +void taosIdPoolMarkStatus(void *handle, int id) { + id_pool_t *pIdPool = handle; + pthread_mutex_lock(&pIdPool->mutex); - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 0; i < pIdPool->maxId; ++i) pIdPool->freeList[i] = 0; -} - -void taosIdPoolMarkStatus(void *handle, int id, int status) { - id_pool_t *pIdPool = (id_pool_t *)handle; - - pIdPool->freeList[id] = status; -} - -void taosIdPoolSetFreeList(void *handle) { - id_pool_t *pIdPool; - int pos = 0; - - pIdPool = (id_pool_t *)handle; - pIdPool->numOfFree = 0; - pIdPool->freeSlot = 0; - - for (int i = 1; i < pIdPool->maxId; ++i) { - if (pIdPool->freeList[i] == 0) { - pIdPool->freeList[pos] = i; - pIdPool->numOfFree++; - pos++; - } + int slot = id % pIdPool->maxId; + if (pIdPool->freeList[slot] == -1) { + pIdPool->freeList[slot] = slot; + pIdPool->numOfFree--; } + + pthread_mutex_unlock(&pIdPool->mutex); } int taosUpdateIdPool(id_pool_t *handle, int maxId) { @@ -154,18 +129,18 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return -1; } - int *idList, i; - idList = (int *)malloc(sizeof(int) * (size_t)maxId); + int *idList = malloc(sizeof(int) * maxId); if (idList == NULL) { return -1; } - for (i = 1; i < maxId; ++i) { - idList[i - 1] = i; + + pthread_mutex_lock(&pIdPool->mutex); + + for (int i = 0; i < maxId; ++i) { + idList[i] = -1; } - if (pthread_mutex_lock(&pIdPool->mutex) != 0) perror("lock pIdPool Mutex"); - - memcpy(idList, pIdPool->freeList, sizeof(int) * (size_t)pIdPool->maxId); + memcpy(idList, pIdPool->freeList, sizeof(int) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); pIdPool->maxId = maxId; @@ -173,7 +148,7 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { pIdPool->freeList = idList; free(oldIdList); - if (pthread_mutex_unlock(&pIdPool->mutex) != 0) perror("unlock pIdPool Mutex"); + pthread_mutex_unlock(&pIdPool->mutex); return 0; } From 51171c75de64ec31a2185c9547360a28d45ce373 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 23 Mar 2020 22:58:09 +0800 Subject: [PATCH 23/85] [TD-15] --- src/mnode/inc/mgmtChildTable.h | 1 + src/mnode/src/mgmtChildTable.c | 123 +++++++++++++++++------ src/mnode/src/mgmtDnode.c | 3 - src/mnode/src/mgmtNormalTable.c | 115 ++++++++++++++------- src/mnode/src/mgmtSuperTable.c | 170 +++++++++++++++----------------- src/mnode/src/mgmtTable.c | 43 ++++---- 6 files changed, 272 insertions(+), 183 deletions(-) diff --git a/src/mnode/inc/mgmtChildTable.h b/src/mnode/inc/mgmtChildTable.h index b16dd58f67..9252a7d485 100644 --- a/src/mnode/inc/mgmtChildTable.h +++ b/src/mnode/inc/mgmtChildTable.h @@ -39,6 +39,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); void mgmtDropAllChildTables(SDbObj *pDropDb); +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 9d3c3c634d..d717a4e5e9 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -42,14 +42,14 @@ static void mgmtDestroyChildTable(SChildTableObj *pTable) { tfree(pTable); } -static int32_t mgmtChildTableActionDestroy(void *pObj) { - SChildTableObj *pTable = (SChildTableObj *)pObj; +static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; mgmtDestroyChildTable(pTable); return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionInsert(void *pObj) { - SChildTableObj *pTable = (SChildTableObj *) pObj; +static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -84,15 +84,15 @@ static int32_t mgmtChildTableActionInsert(void *pObj) { mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); + if (pVgroup->numOfTables >= pDb->cfg.maxSessions && pDb->numOfVgroups > 1) { + mgmtMoveVgroupToTail(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionDelete(void *pObj) { - SChildTableObj *pTable = (SChildTableObj *) pObj; +static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { return TSDB_CODE_INVALID_VGROUP_ID; } @@ -121,33 +121,35 @@ static int32_t mgmtChildTableActionDelete(void *pObj) { mgmtRemoveTableFromSuperTable(pTable->superTable); if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); + mgmtMoveVgroupToHead(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionUpdate(void *pObj) { +static int32_t mgmtChildTableActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtChildTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SChildTableObj *pTable = (SChildTableObj *) pObj; - assert(pObj != NULL && pData != NULL); +static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { + SChildTableObj *pTable = pOper->pObj; + assert(pTable != NULL && pOper->rowData != NULL); - memcpy(pData, pTable, tsChildTableUpdateSize); + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); return tsChildTableUpdateSize; } -static void *mgmtChildTableActionDecode(void *pData) { - assert(pData != NULL); +static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SChildTableObj *pTable = (SChildTableObj *)calloc(sizeof(SChildTableObj), 1); - if (pTable == NULL) return NULL; + pOper->pObj = calloc(1, sizeof(SChildTableObj)); + if (pOper->pObj == NULL) { + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } - memcpy(pTable, pData, tsChildTableUpdateSize); + memcpy(pOper->pObj, pOper->rowData, tsChildTableUpdateSize); - return (void *)pTable; + return TSDB_CODE_SUCCESS; } int32_t mgmtInitChildTables() { @@ -162,7 +164,7 @@ int32_t mgmtInitChildTables() { .tableName = "ctables", .hashSessions = tsMaxTables, .maxRowSize = tsChildTableUpdateSize, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, .updateFp = mgmtChildTableActionUpdate, @@ -187,7 +189,11 @@ int32_t mgmtInitChildTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -196,7 +202,11 @@ int32_t mgmtInitChildTables() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -205,7 +215,11 @@ int32_t mgmtInitChildTables() { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -213,19 +227,27 @@ int32_t mgmtInitChildTables() { if (pVgroup->tableList == NULL) { mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); pTable->vgId = 0; - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -310,7 +332,13 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t pTable->vgId = pVgroup->vgId; pTable->superTable = pSuperTable; - if (sdbInsertRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbInsertRow(&desc); + + if (sdbInsertRow(&desc) < 0) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; @@ -453,12 +481,45 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_LOCAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all child tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); +} + +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { + void *pNode = NULL; + void *pLastNode = NULL; + int32_t numOfTables = 0; + SChildTableObj *pTable = NULL; + + while (1) { + pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); + if (pTable == NULL) { + break; + } + + if (pTable->superTable == pStable) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsChildTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); + pNode = pLastNode; + numOfTables++; + continue; + } + } + + mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->tableId, numOfTables); } \ No newline at end of file diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 0d83e262ae..0aab083471 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -547,9 +547,6 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { return ; } - uint32_t lastPrivateIp = pDnode->privateIp; - uint32_t lastPublicIp = pDnode->publicIp; - pDnode->privateIp = htonl(pStatus->privateIp); pDnode->publicIp = htonl(pStatus->publicIp); pDnode->lastReboot = htonl(pStatus->lastReboot); diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index 9140a4c752..0ffd4fb28e 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -42,14 +42,14 @@ static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { tfree(pTable); } -static int32_t mgmtNormalTableActionDestroy(void *pObj) { - SNormalTableObj *pTable = (SNormalTableObj *)pObj; +static int32_t mgmtNormalTableActionDestroy(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; mgmtDestroyNormalTable(pTable); return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionInsert(void *pObj) { - SNormalTableObj *pTable = (SNormalTableObj *) pObj; +static int32_t mgmtNormalTableActionInsert(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -82,14 +82,14 @@ static int32_t mgmtNormalTableActionInsert(void *pObj) { mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pDb, pVgroup); + mgmtMoveVgroupToTail(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionDelete(void *pObj) { - SNormalTableObj *pTable = (SNormalTableObj *) pObj; +static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { return TSDB_CODE_INVALID_VGROUP_ID; } @@ -116,13 +116,13 @@ static int32_t mgmtNormalTableActionDelete(void *pObj) { mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pDb, pVgroup); + mgmtMoveVgroupToHead(pVgroup); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionUpdate(void *pObj) { +static int32_t mgmtNormalTableActionUpdate(SSdbOperDesc *pOper) { // SNormalTableObj *pTable = (SNormalTableObj *) pObj; // memcpy(pTable, str, tsNormalTableUpdateSize); @@ -134,48 +134,49 @@ static int32_t mgmtNormalTableActionUpdate(void *pObj) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtNormalTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SNormalTableObj *pTable = (SNormalTableObj *) pObj; - assert(pObj != NULL && pData != NULL); +static int32_t mgmtNormalTableActionEncode(SSdbOperDesc *pOper) { + SNormalTableObj *pTable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { + if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { return -1; } - memcpy(pData, pTable, tsNormalTableUpdateSize); - memcpy(pData + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(pData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); + memcpy(pOper->rowData, pTable, tsNormalTableUpdateSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize, pTable->schema, schemaSize); + memcpy(pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); return tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; } -static void *mgmtNormalTableActionDecode(void *pData) { - assert(pData != NULL); +static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); if (pTable == NULL) { - return NULL; + return -1; } memset(pTable, 0, sizeof(SNormalTableObj)); - memcpy(pTable, pData, tsNormalTableUpdateSize); + memcpy(pTable, pOper->rowData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); if (pTable->schema == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; + return -1; } - memcpy(pTable->schema, pData + tsNormalTableUpdateSize, schemaSize); + memcpy(pTable->schema, pOper->rowData + tsNormalTableUpdateSize, schemaSize); pTable->sql = (char *)malloc(pTable->sqlLen); if (pTable->sql == NULL) { mgmtDestroyNormalTable(pTable); - return NULL; + return -1; } - memcpy(pTable->sql, pData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); - return (void *)pTable; + memcpy(pTable->sql, pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + + return TSDB_CODE_SUCCESS; } int32_t mgmtInitNormalTables() { @@ -190,7 +191,7 @@ int32_t mgmtInitNormalTables() { .tableName = "ntables", .hashSessions = TSDB_MAX_NORMAL_TABLES, .maxRowSize = sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtNormalTableActionInsert, .deleteFp = mgmtNormalTableActionDelete, .updateFp = mgmtNormalTableActionUpdate, @@ -213,7 +214,11 @@ int32_t mgmtInitNormalTables() { SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); if (pDb == NULL) { mError("ntable:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -222,7 +227,11 @@ int32_t mgmtInitNormalTables() { if (pVgroup == NULL) { mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -231,7 +240,12 @@ int32_t mgmtInitNormalTables() { mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } @@ -239,14 +253,18 @@ int32_t mgmtInitNormalTables() { if (pVgroup->tableList == NULL) { mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); pTable->vgId = 0; - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_DISK); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + sdbDeleteRow(&desc); pNode = pLastNode; continue; } mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable); //pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid, 1); + taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns; @@ -346,7 +364,11 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); } - if (sdbInsertRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + if (sdbInsertRow(&desc) < 0) { mError("table:%s, update sdb error", pTable->tableId); free(pTable); terrno = TSDB_CODE_SDB_ERROR; @@ -443,7 +465,14 @@ int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int3 pTable->sversion++; pAcct->acctInfo.numOfTimeSeries += ncols; - sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); + return TSDB_CODE_SUCCESS; } @@ -472,7 +501,14 @@ int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) pTable->sversion++; pAcct->acctInfo.numOfTimeSeries--; - sdbUpdateRow(tsNormalTableSdb, pTable, tsNormalTableUpdateSize, SDB_OPER_GLOBAL); + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsNormalTableUpdateSize; + sdbUpdateRow(&desc); return TSDB_CODE_SUCCESS; } @@ -533,12 +569,17 @@ void mgmtDropAllNormalTables(SDbObj *pDropDb) { if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_LOCAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsNormalTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables ++; + numOfTables++; continue; } } - mTrace("db:%s, all normal tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all normal tables:%d is dropped from sdb", pDropDb->name, numOfTables); } diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index a4949a74c4..d3a0dcb6bc 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -32,93 +32,84 @@ #include "name.h" #include "tsqlfunction.h" -static void *tsSuperTableSdb; +static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static void mgmtDestroySuperTable(SSuperTableObj *pTable) { - tfree(pTable->schema); - tfree(pTable); +static void mgmtDestroySuperTable(SSuperTableObj *pStable) { + tfree(pStable->schema); + tfree(pStable); } -static int32_t mgmtSuperTableActionDestroy(void *pObj) { - SSuperTableObj *pTable = (SSuperTableObj *) pObj; - mgmtDestroySuperTable(pTable); +static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) { + mgmtDestroySuperTable(pOper->pObj); return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionInsert(void *pObj) { - STableInfo *pTable = (STableInfo *) pObj; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionDelete(void *pObj) { - STableInfo *pTable = (STableInfo *) pObj; - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb) { +static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { + STableInfo *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + if (pDb != NULL) { mgmtRemoveSuperTableFromDb(pDb); + mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); } return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionUpdate(void *pObj) { - SSuperTableObj *pTable = (SSuperTableObj *) pObj; - memcpy(pTable, pObj, tsSuperTableUpdateSize); - - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = realloc(pTable->schema, schemaSize); - memcpy(pTable->schema, pObj + tsSuperTableUpdateSize, schemaSize); - +static int32_t mgmtSuperTableActionUpdate(SSdbOperDesc *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mgmtSuperTableActionEncode(void *pObj, void *pData, int32_t maxRowSize) { - SSuperTableObj *pTable = (SSuperTableObj *) pObj; - assert(pObj != NULL && pData != NULL); +static int32_t mgmtSuperTableActionEncode(SSdbOperDesc *pOper) { + SSuperTableObj *pStable = pOper->pObj; + assert(pOper->pObj != NULL && pOper->rowData != NULL); - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - if (maxRowSize < tsSuperTableUpdateSize + schemaSize + 1) { + if (pOper->maxRowSize < tsSuperTableUpdateSize + schemaSize) { return TSDB_CODE_INVALID_MSG_LEN; } - memcpy(pData, pTable, tsSuperTableUpdateSize); - memcpy(pData + tsSuperTableUpdateSize, pTable->schema, schemaSize); - return tsSuperTableUpdateSize + schemaSize; + memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize); + memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize); + pOper->rowSize = tsSuperTableUpdateSize + schemaSize; + + return TSDB_CODE_SUCCESS; } -static void *mgmtSuperTableActionDecode(void *pData) { - assert(pData != NULL); +static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { + assert(pOper->rowData != NULL); - SSuperTableObj *pTable = (SSuperTableObj *) malloc(sizeof(SSuperTableObj)); - if (pTable == NULL) { - return NULL; - } - memset(pTable, 0, sizeof(SSuperTableObj)); - memcpy(pTable, pData, tsSuperTableUpdateSize); + SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); + if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns + pTable->numOfTags); - pTable->schema = malloc(schemaSize); - if (pTable->schema == NULL) { - mgmtDestroySuperTable(pTable); - return NULL; + memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize); + + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); + pStable->schema = malloc(schemaSize); + if (pStable->schema == NULL) { + mgmtDestroySuperTable(pStable); + return -1; } - memcpy(pTable->schema, pData + tsSuperTableUpdateSize, schemaSize); - return (void *) pTable; + memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize); + pOper->pObj = pStable; + + return TSDB_CODE_SUCCESS; } int32_t mgmtInitSuperTables() { - void *pNode = NULL; - void *pLastNode = NULL; - SSuperTableObj *pTable = NULL; - SSuperTableObj tObj; tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; @@ -126,7 +117,7 @@ int32_t mgmtInitSuperTables() { .tableName = "stables", .hashSessions = TSDB_MAX_SUPER_TABLES, .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, - .keyType = SDB_KEYTYPE_STRING, + .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtSuperTableActionInsert, .deleteFp = mgmtSuperTableActionDelete, .updateFp = mgmtSuperTableActionUpdate, @@ -141,24 +132,6 @@ int32_t mgmtInitSuperTables() { return -1; } - pNode = NULL; - while (1) { - pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **) &pTable); - if (pTable == NULL) { - break; - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("super table:%s, failed to get db, discard it", pTable->tableId); - sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_DISK); - pNode = pLastNode; - continue; - } - - mgmtAddSuperTableIntoDb(pDb); - } - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mgmtGetShowSuperTableMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mgmtRetrieveShowSuperTables); @@ -171,7 +144,7 @@ void mgmtCleanUpSuperTables() { } int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { - SSuperTableObj *pStable = (SSuperTableObj *)calloc(sizeof(SSuperTableObj), 1); + SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -203,13 +176,21 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { tschema[col].bytes = htons(tschema[col].bytes); } - if (sdbInsertRow(tsSuperTableSdb, pStable, SDB_OPER_GLOBAL) < 0) { - mError("stable:%s, update sdb error", pStable->tableId); - return TSDB_CODE_SDB_ERROR; - } + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable, + .rowSize = sizeof(SSuperTableObj) + schemaSize + }; - mPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); - return TSDB_CODE_SUCCESS; + int32_t code = sdbInsertRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mgmtDestroySuperTable(pStable); + return TSDB_CODE_SDB_ERROR; + } else { + mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); + return TSDB_CODE_SUCCESS; + } } int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pStable) { @@ -217,10 +198,14 @@ int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pSta mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables); return TSDB_CODE_OTHERS; } else { - //TODO: drop child tables - mError("stable:%s, is dropped from sdb", pStable->tableId); - mgmtRemoveSuperTableFromDb(pDb); - return TSDB_CODE_OTHERS; + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable + }; + int32_t code = sdbDeleteRow(&oper); + mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->tableId, tstrerror(code)); + return code; } } @@ -229,10 +214,9 @@ void* mgmtGetSuperTable(char *tableId) { } void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { - //TODO get vgroup of dnodes SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); - rsp->numOfDnodes = 1; - rsp->dnodeIps[0] = 0; + rsp->numOfDnodes = htonl(1); + rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; } @@ -289,7 +273,7 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); return TSDB_CODE_SUCCESS; @@ -322,7 +306,7 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -354,7 +338,8 @@ int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagN // mgmtSuperTableActionEncode(pStable, msg, size, &rowSize); - int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + int32_t ret = 0; + // int32_t ret = sdbUpdateRow(tsSuperTableSdb, msg, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); tfree(msg); if (ret < 0) { @@ -416,7 +401,7 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 pStable->sversion++; pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -449,7 +434,7 @@ int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { pStable->schema = realloc(pStable->schema, schemaSize); pAcct->acctInfo.numOfTimeSeries -= (pStable->numOfTables); - sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); + // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); return TSDB_CODE_SUCCESS; } @@ -588,14 +573,19 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { } if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - sdbDeleteRow(tsSuperTableSdb, pTable, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_LOCAL, + .table = tsSuperTableSdb, + .pObj = pTable, + }; + sdbDeleteRow(&oper); pNode = pLastNode; numOfTables ++; continue; } } - mTrace("db:%s, all super tables:%d is dropped", pDropDb->name, numOfTables); + mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable) { diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 255b2a6f27..918547fa12 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -78,8 +78,6 @@ int32_t mgmtInitTables() { return code; } - mgmtSetVgroupIdPool(); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CREATE_TABLE, mgmtProcessCreateTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_TABLE, mgmtProcessDropTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mgmtProcessAlterTableMsg); @@ -202,13 +200,13 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table_name"); + strcpy(pSchema[cols].name, "table name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -220,7 +218,7 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "stable_name"); + strcpy(pSchema[cols].name, "stable name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -238,9 +236,6 @@ int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) return 0; } -/* - * remove the hole in result set - */ static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { if (rows < capacity) { for (int32_t i = 0; i < numOfCols; ++i) { @@ -434,11 +429,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateChildTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } else { @@ -446,11 +443,13 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { pTable = mgmtCreateNormalTable(pCreate, pVgroup, sid); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); if (pMDCreate == NULL) { mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); return; } } @@ -761,9 +760,9 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); + // sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); } else if (pTable->type == TSDB_NORMAL_TABLE){ - sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); + // sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -817,19 +816,19 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - mError("table:%s, update ctables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } + // if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + // mError("table:%s, update ctables sdb error", pTable->tableId); + // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + // free(queueMsg); + // return; + // } } else if (pTable->type == TSDB_NORMAL_TABLE){ - if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - mError("table:%s, update ntables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } + // if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { + // mError("table:%s, update ntables sdb error", pTable->tableId); + // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + // free(queueMsg); + // return; + // } } if (pVgroup->numOfTables <= 0) { From e49bd8a723c35b78f61eb0c92babc764b25d2557 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 24 Mar 2020 01:01:55 +0800 Subject: [PATCH 24/85] [td-32] fix bugs in returning data to client --- src/client/src/tscFunctionImpl.c | 4 + src/client/src/tscServer.c | 78 ++---- src/dnode/src/dnodeRead.c | 12 +- src/inc/taosmsg.h | 441 +++++++++++++++---------------- src/query/inc/queryExecutor.h | 15 +- src/query/inc/tsqlfunction.h | 2 +- src/query/src/queryExecutor.c | 392 ++++++++++++++------------- src/util/src/shash.c | 2 + src/vnode/tsdb/src/tsdbRead.c | 42 ++- 9 files changed, 497 insertions(+), 491 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 689f9715d0..c7be1b84ce 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -417,6 +417,10 @@ static void count_function(SQLFunctionCtx *pCtx) { numOfElem += 1; } } else { + /* + * when counting on the primary time stamp column and no statistics data is provided, + * simple use the size value + */ numOfElem = pCtx->size; } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 58fc32c824..3da374f152 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -659,14 +659,18 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p failed to malloc for query msg", pSql); return -1; } - + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - char *pStart = pCmd->payload + tsRpcHeadSize; - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; + + if (pQueryInfo->colList.numOfCols <= 0) { + tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); + return -1; + } + + char *pStart = pCmd->payload + tsRpcHeadSize; SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; @@ -675,11 +679,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; - - pQueryMsg->uid = pTableMeta->uid; - pQueryMsg->numOfTagsCols = 0; - - pQueryMsg->vgId = htonl(pTableMeta->vgId); + pQueryMsg->head.vgId = htonl(pTableMeta->vgId); tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); } else { // query on super table if (pTableMetaInfo->vnodeIndex < 0) { @@ -697,11 +697,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); - pQueryMsg->vgId = htons(vnodeId); + pQueryMsg->head.vgId = htons(vnodeId); } pQueryMsg->numOfTables = htonl(numOfTables); - pQueryMsg->numOfTagsCols = htons(pTableMetaInfo->numOfTags); if (pQueryInfo->order.order == TSQL_SO_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->stime); @@ -711,24 +710,15 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); } - pQueryMsg->order = htons(pQueryInfo->order.order); - pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); - - pQueryMsg->interpoType = htons(pQueryInfo->interpoType); - - pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); - pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); - - pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); - - if (pQueryInfo->colList.numOfCols <= 0) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; - } - - pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); + pQueryMsg->order = htons(pQueryInfo->order.order); + pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); + pQueryMsg->interpoType = htons(pQueryInfo->interpoType); + pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); + pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); + pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); + pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; - pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); if (pQueryInfo->intervalTime < 0) { tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); @@ -776,7 +766,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colList[i].colId = htons(pColSchema->colId); pQueryMsg->colList[i].bytes = htons(pColSchema->bytes); - pQueryMsg->colList[i].type = htons(pColSchema->type); + pQueryMsg->colList[i].type = htons(pColSchema->type); pQueryMsg->colList[i].numOfFilters = htons(pCol->numOfFilters); // append the filter information after the basic column information @@ -824,11 +814,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return -1; } - pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); + pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); pSqlFuncExpr->colInfo.colIdx = htons(pExpr->colInfo.colIdx); - pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); + pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); - pSqlFuncExpr->functionId = htons(pExpr->functionId); + pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); @@ -866,25 +856,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colNameLen = htonl(len); // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->vgId), pMsg); - - // only include the required tag column schema. If a tag is not required, it won't be sent to vnode - if (pTableMetaInfo->numOfTags > 0) { - // always transfer tag schema to vnode if exists - SSchema *pTagSchema = tscGetTableTagSchema(pTableMeta); - - for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - if (pTableMetaInfo->tagColumnIndex[j] == TSDB_TBNAME_COLUMN_INDEX) { - SSchema tbSchema = { - .bytes = TSDB_TABLE_NAME_LEN, .colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY}; - memcpy(pMsg, &tbSchema, sizeof(SSchema)); - } else { - memcpy(pMsg, &pTagSchema[pTableMetaInfo->tagColumnIndex[j]], sizeof(SSchema)); - } - - pMsg += sizeof(SSchema); - } - } + pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->head.vgId), pMsg); SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr; if (pGroupbyExpr->numOfGroupCols != 0) { @@ -948,8 +920,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = msgLen; pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; - pQueryMsg->contLen = htonl(msgLen); - + pQueryMsg->head.contLen = htonl(msgLen); assert(msgLen + minMsgSize() <= size); return TSDB_CODE_SUCCESS; @@ -2358,7 +2329,6 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { } pRes->row = 0; - tscTrace("%p numOfRows:%d, offset:%d", pSql, pRes->numOfRows, pRes->offset); return 0; diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index 887845b00e..bfe717c367 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -284,16 +284,10 @@ static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { //todo free qinfo } else { - contLen = 100; + SRetrieveTableRsp* pRsp = NULL; - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - pRsp->numOfRows = htonl(1); - pRsp->precision = htons(0); - pRsp->offset = htobe64(0); - pRsp->useconds = htobe64(0); - - // todo set the data - *(int64_t*) pRsp->data = 1000; + int32_t code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); + //todo check code rpcRsp = (SRpcMsg) { .handle = pMsg->rpcMsg.handle, diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index cc17df9bec..2fcc91a7bb 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -20,122 +20,122 @@ extern "C" { #endif -#include #include +#include #include "taosdef.h" #include "taoserror.h" #include "trpc.h" // message type -#define TSDB_MSG_TYPE_REG 1 -#define TSDB_MSG_TYPE_REG_RSP 2 -#define TSDB_MSG_TYPE_SUBMIT 3 -#define TSDB_MSG_TYPE_SUBMIT_RSP 4 -#define TSDB_MSG_TYPE_QUERY 5 -#define TSDB_MSG_TYPE_QUERY_RSP 6 -#define TSDB_MSG_TYPE_RETRIEVE 7 -#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 +#define TSDB_MSG_TYPE_REG 1 +#define TSDB_MSG_TYPE_REG_RSP 2 +#define TSDB_MSG_TYPE_SUBMIT 3 +#define TSDB_MSG_TYPE_SUBMIT_RSP 4 +#define TSDB_MSG_TYPE_QUERY 5 +#define TSDB_MSG_TYPE_QUERY_RSP 6 +#define TSDB_MSG_TYPE_RETRIEVE 7 +#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 // message from mnode to dnode -#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 -#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 -#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 -#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 -#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 -#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 -#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 -#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 +#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 +#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 +#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 +#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 +#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 +#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 // message from client to mnode -#define TSDB_MSG_TYPE_CM_CONNECT 31 -#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 -#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 -#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 -#define TSDB_MSG_TYPE_CM_CREATE_USER 39 -#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 -#define TSDB_MSG_TYPE_CM_ALTER_USER 41 -#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 -#define TSDB_MSG_TYPE_CM_DROP_USER 43 -#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 -#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 -#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP -#define TSDB_MSG_TYPE_CM_CREATE_DB 49 -#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 -#define TSDB_MSG_TYPE_CM_DROP_DB 51 -#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 -#define TSDB_MSG_TYPE_CM_USE_DB 53 -#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 -#define TSDB_MSG_TYPE_CM_ALTER_DB 55 -#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 -#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 -#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 -#define TSDB_MSG_TYPE_CM_TABLE_META 63 -#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 -#define TSDB_MSG_TYPE_CM_STABLE_META 65 -#define TSDB_MSG_TYPE_CM_STABLE_META_RSP 66 -#define TSDB_MSG_TYPE_CM_TABLES_META 67 -#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 -#define TSDB_MSG_TYPE_CM_SHOW 71 -#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 -#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 -#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 -#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 -#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 -#define TSDB_MSG_TYPE_CM_KILL_CONN 77 -#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 -#define TSDB_MSG_TYPE_CM_HEARTBEAT 79 -#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80 +#define TSDB_MSG_TYPE_CM_CONNECT 31 +#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 +#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 +#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 +#define TSDB_MSG_TYPE_CM_CREATE_USER 39 +#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 +#define TSDB_MSG_TYPE_CM_ALTER_USER 41 +#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 +#define TSDB_MSG_TYPE_CM_DROP_USER 43 +#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 +#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 +#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP +#define TSDB_MSG_TYPE_CM_CREATE_DB 49 +#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 +#define TSDB_MSG_TYPE_CM_DROP_DB 51 +#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 +#define TSDB_MSG_TYPE_CM_USE_DB 53 +#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 +#define TSDB_MSG_TYPE_CM_ALTER_DB 55 +#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 +#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 +#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 +#define TSDB_MSG_TYPE_CM_TABLE_META 63 +#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 +#define TSDB_MSG_TYPE_CM_STABLE_META 65 +#define TSDB_MSG_TYPE_CM_STABLE_META_RSP 66 +#define TSDB_MSG_TYPE_CM_TABLES_META 67 +#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 +#define TSDB_MSG_TYPE_CM_SHOW 71 +#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 +#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 +#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 +#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 +#define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 +#define TSDB_MSG_TYPE_CM_KILL_CONN 77 +#define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 +#define TSDB_MSG_TYPE_CM_HEARTBEAT 79 +#define TSDB_MSG_TYPE_CM_HEARTBEAT_RSP 80 // message from dnode to mnode -#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91 -#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92 -#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93 -#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94 -#define TSDB_MSG_TYPE_DM_STATUS 95 -#define TSDB_MSG_TYPE_DM_STATUS_RSP 96 -#define TSDB_MSG_TYPE_DM_GRANT 97 -#define TSDB_MSG_TYPE_DM_GRANT_RSP 98 +#define TSDB_MSG_TYPE_DM_CONFIG_TABLE 91 +#define TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP 92 +#define TSDB_MSG_TYPE_DM_CONFIG_VNODE 93 +#define TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP 94 +#define TSDB_MSG_TYPE_DM_STATUS 95 +#define TSDB_MSG_TYPE_DM_STATUS_RSP 96 +#define TSDB_MSG_TYPE_DM_GRANT 97 +#define TSDB_MSG_TYPE_DM_GRANT_RSP 98 -#define TSDB_MSG_TYPE_SDB_SYNC 101 -#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102 -#define TSDB_MSG_TYPE_SDB_FORWARD 103 -#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104 +#define TSDB_MSG_TYPE_SDB_SYNC 101 +#define TSDB_MSG_TYPE_SDB_SYNC_RSP 102 +#define TSDB_MSG_TYPE_SDB_FORWARD 103 +#define TSDB_MSG_TYPE_SDB_FORWARD_RSP 104 -#define TSDB_MSG_TYPE_MAX 105 +#define TSDB_MSG_TYPE_MAX 105 // IE type -#define TSDB_IE_TYPE_SEC 1 -#define TSDB_IE_TYPE_META 2 -#define TSDB_IE_TYPE_MGMT_IP 3 -#define TSDB_IE_TYPE_DNODE_CFG 4 -#define TSDB_IE_TYPE_NEW_VERSION 5 -#define TSDB_IE_TYPE_DNODE_EXT 6 -#define TSDB_IE_TYPE_DNODE_STATE 7 +#define TSDB_IE_TYPE_SEC 1 +#define TSDB_IE_TYPE_META 2 +#define TSDB_IE_TYPE_MGMT_IP 3 +#define TSDB_IE_TYPE_DNODE_CFG 4 +#define TSDB_IE_TYPE_NEW_VERSION 5 +#define TSDB_IE_TYPE_DNODE_EXT 6 +#define TSDB_IE_TYPE_DNODE_STATE 7 enum _mgmt_table { TSDB_MGMT_TABLE_ACCT, @@ -157,38 +157,38 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MAX, }; -#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 -#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 -#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 -#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 +#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 +#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 +#define TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN 3 +#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 -#define TSDB_ALTER_TABLE_ADD_COLUMN 5 -#define TSDB_ALTER_TABLE_DROP_COLUMN 6 +#define TSDB_ALTER_TABLE_ADD_COLUMN 5 +#define TSDB_ALTER_TABLE_DROP_COLUMN 6 -#define TSDB_INTERPO_NONE 0 -#define TSDB_INTERPO_NULL 1 -#define TSDB_INTERPO_SET_VALUE 2 -#define TSDB_INTERPO_LINEAR 3 -#define TSDB_INTERPO_PREV 4 +#define TSDB_INTERPO_NONE 0 +#define TSDB_INTERPO_NULL 1 +#define TSDB_INTERPO_SET_VALUE 2 +#define TSDB_INTERPO_LINEAR 3 +#define TSDB_INTERPO_PREV 4 -#define TSDB_ALTER_USER_PASSWD 0x1 -#define TSDB_ALTER_USER_PRIVILEGES 0x2 +#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_PRIVILEGES 0x2 -#define TSDB_KILL_MSG_LEN 30 +#define TSDB_KILL_MSG_LEN 30 -#define TSDB_VN_READ_ACCCESS ((char)0x1) +#define TSDB_VN_READ_ACCCESS ((char)0x1) #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u +#define TSDB_COL_TAG 0x1u +#define TSDB_COL_JOIN 0x2u extern char *taosMsg[]; #pragma pack(push, 1) -//typedef struct { +// typedef struct { // int32_t vnode; // int32_t sid; // int32_t sversion; @@ -206,7 +206,7 @@ typedef struct SMsgHead { int32_t vgId; } SMsgHead; -//typedef struct { +// typedef struct { // SMsgDesc desc; // SMsgHead header; // int16_t import; @@ -216,37 +216,37 @@ typedef struct SMsgHead { // Submit message for one table typedef struct SSubmitBlk { - int64_t uid; // table unique id - int32_t tid; // table id - int32_t padding; // TODO just for padding here - int32_t sversion; // data schema version - int32_t len; // data part length, not including the SSubmitBlk head - int16_t numOfRows; // total number of rows in current submit block - char data[]; + int64_t uid; // table unique id + int32_t tid; // table id + int32_t padding; // TODO just for padding here + int32_t sversion; // data schema version + int32_t len; // data part length, not including the SSubmitBlk head + int16_t numOfRows; // total number of rows in current submit block + char data[]; } SSubmitBlk; // Submit message for this TSDB typedef struct SSubmitMsg { SMsgHead header; int32_t length; - int32_t compressed:2; - int32_t numOfBlocks:30; + int32_t compressed : 2; + int32_t numOfBlocks : 30; SSubmitBlk blocks[]; } SSubmitMsg; typedef struct { - int32_t index; // index of failed block in submit blocks - int32_t vnode; // vnode index of failed block - int32_t sid; // table index of failed block - int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table + int32_t index; // index of failed block in submit blocks + int32_t vnode; // vnode index of failed block + int32_t sid; // table index of failed block + int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table } SShellSubmitRspBlock; typedef struct { - int32_t code; // 0-success, > 0 error code - int32_t numOfRows; // number of records the client is trying to write - int32_t affectedRows; // number of records actually written - int32_t failedRows; // number of failed records (exclude duplicate records) - int32_t numOfFailedBlocks; + int32_t code; // 0-success, > 0 error code + int32_t numOfRows; // number of records the client is trying to write + int32_t affectedRows; // number of records actually written + int32_t failedRows; // number of failed records (exclude duplicate records) + int32_t numOfFailedBlocks; SShellSubmitRspBlock failedBlocks[]; } SShellSubmitRspMsg; @@ -258,38 +258,38 @@ typedef struct SSchema { } SSchema; typedef struct { - int32_t vnode; //the index of vnode + int32_t vnode; // the index of vnode uint32_t ip; } SVnodeDesc; typedef struct { - int32_t contLen; - int32_t vgId; - int8_t tableType; - int16_t numOfColumns; - int16_t numOfTags; - int32_t sid; - int32_t sversion; - int32_t tagDataLen; - int32_t sqlDataLen; - uint64_t uid; - uint64_t superTableUid; - uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; - char data[]; + int32_t contLen; + int32_t vgId; + int8_t tableType; + int16_t numOfColumns; + int16_t numOfTags; + int32_t sid; + int32_t sversion; + int32_t tagDataLen; + int32_t sqlDataLen; + uint64_t uid; + uint64_t superTableUid; + uint64_t createdTime; + char tableId[TSDB_TABLE_ID_LEN]; + char superTableId[TSDB_TABLE_ID_LEN]; + char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; - int8_t igExists; - int16_t numOfTags; - int16_t numOfColumns; - int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string - int32_t contLen; - int8_t reserved[16]; - char schema[]; + char tableId[TSDB_TABLE_ID_LEN]; + char db[TSDB_DB_NAME_LEN]; + int8_t igExists; + int16_t numOfTags; + int16_t numOfColumns; + int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string + int32_t contLen; + int8_t reserved[16]; + char schema[]; } SCMCreateTableMsg; typedef struct { @@ -331,7 +331,7 @@ typedef struct { int64_t maxQueryTime; // In unit of hour int64_t maxInbound; int64_t maxOutbound; - int8_t accessState; // Configured only by command + int8_t accessState; // Configured only by command } SAcctCfg; typedef struct { @@ -356,11 +356,11 @@ typedef struct { } SMgmtHead; typedef struct { - int32_t contLen; - int32_t vgId; - int32_t sid; - uint64_t uid; - char tableId[TSDB_TABLE_ID_LEN + 1]; + int32_t contLen; + int32_t vgId; + int32_t sid; + uint64_t uid; + char tableId[TSDB_TABLE_ID_LEN + 1]; } SMDDropTableMsg; typedef struct { @@ -371,7 +371,7 @@ typedef struct { } SMDDropSTableMsg; typedef struct { - int32_t vgId; + int32_t vgId; } SMDDropVnodeMsg; typedef struct SColIndexEx { @@ -386,7 +386,7 @@ typedef struct SColIndexEx { */ int16_t colIdx; int16_t colIdxInBuf; - uint16_t flag; // denote if it is a tag or not + uint16_t flag; // denote if it is a tag or not char name[TSDB_COL_NAME_LEN]; } SColIndexEx; @@ -458,7 +458,7 @@ typedef struct SColumnInfo { typedef struct STableIdInfo { int32_t sid; int64_t uid; - TSKEY key; // last accessed ts, for subscription + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -472,47 +472,30 @@ typedef struct STimeWindow { * the outputCols will be 3 while the numOfCols is 1. */ typedef struct { - int32_t contLen; // msg header - int16_t vgId; - - int32_t numOfTables; - uint64_t uid; + SMsgHead head; STimeWindow window; + int32_t numOfTables; + int16_t order; + int16_t orderColId; + int16_t numOfCols; // the number of columns will be load from vnode + int64_t intervalTime; // time interval for aggregation, in million second + int64_t intervalOffset; // start offset for interval query + int64_t slidingTime; // value for sliding window + char slidingTimeUnit; // time interval type, for revisement of interval(1d) + int16_t tagLength; // tag length in current query + int16_t numOfGroupCols; // num of group by columns + int16_t orderByIdx; + int16_t orderType; // used in group by xx order by xxx + uint64_t groupbyTagIds; + int64_t limit; + int64_t offset; + int16_t queryType; // denote another query process + int16_t numOfOutputCols; // final output columns numbers + int16_t interpoType; // interpolate type + uint64_t defaultVal; // default value array list - int16_t order; - int16_t orderColId; - - int16_t numOfCols; // the number of columns will be load from vnode - char slidingTimeUnit; // time interval type, for revisement of interval(1d) - - int64_t intervalTime; // time interval for aggregation, in million second - int64_t slidingTime; // value for sliding window - - // tag schema, used to parse tag information in pSidExtInfo - uint64_t pTagSchema; - - int16_t numOfTagsCols; // required number of tags - int16_t tagLength; // tag length in current query - - int16_t numOfGroupCols; // num of group by columns - int16_t orderByIdx; - int16_t orderType; // used in group by xx order by xxx - uint64_t groupbyTagIds; - - int64_t limit; - int64_t offset; - - int16_t queryType; // denote another query process - int16_t numOfOutputCols; // final output columns numbers - - int16_t interpoType; // interpolate type - uint64_t defaultVal; // default value array list - - int32_t colNameLen; - int64_t colNameList; - - int64_t pSqlFuncExprs; - + int32_t colNameLen; + int64_t colNameList; int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed int32_t tsLen; // total length of ts comp block int32_t tsNumOfBlocks; // ts comp block numbers @@ -533,9 +516,9 @@ typedef struct { typedef struct SRetrieveTableRsp { int32_t numOfRows; - int8_t completed; // all results are returned to client + int8_t completed; // all results are returned to client int16_t precision; - int64_t offset; // updated offset value for multi-vnode projection query + int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; char data[]; } SRetrieveTableRsp; @@ -615,11 +598,11 @@ typedef struct { char dnodeName[TSDB_DNODE_NAME_LEN]; uint32_t privateIp; uint32_t publicIp; - uint32_t lastReboot; // time stamp for last reboot - uint16_t numOfTotalVnodes; // from config file + uint32_t lastReboot; // time stamp for last reboot + uint16_t numOfTotalVnodes; // from config file uint16_t openVnodes; uint16_t numOfCores; - float diskAvailable; // GB + float diskAvailable; // GB uint8_t alternativeRole; uint8_t reserve[15]; SVnodeLoad load[]; @@ -648,7 +631,7 @@ typedef struct { } SCMMultiTableInfoMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; + char tableId[TSDB_TABLE_ID_LEN + 1]; } SCMSuperTableInfoMsg; typedef struct { @@ -707,18 +690,18 @@ typedef struct STableMetaMsg { uint8_t tableType; int16_t numOfColumns; int16_t sversion; - - int8_t numOfVpeers; + + int8_t numOfVpeers; SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT]; - int32_t sid; - int32_t vgId; - uint64_t uid; - SSchema schema[]; + int32_t sid; + int32_t vgId; + uint64_t uid; + SSchema schema[]; } STableMetaMsg; typedef struct SMultiTableMeta { - int32_t numOfTables; - int32_t contLen; + int32_t numOfTables; + int32_t contLen; STableMetaMsg metas[]; } SMultiTableMeta; @@ -756,7 +739,7 @@ typedef struct { typedef struct { uint32_t dnode; - int32_t vnode; + int32_t vnode; } SDMConfigVnodeMsg; typedef struct { @@ -783,13 +766,13 @@ typedef struct { } SStreamDesc; typedef struct { - int32_t numOfQueries; - SQueryDesc qdesc[]; + int32_t numOfQueries; + SQueryDesc qdesc[]; } SQqueryList; typedef struct { - int32_t numOfStreams; - SStreamDesc sdesc[]; + int32_t numOfStreams; + SStreamDesc sdesc[]; } SStreamList; typedef struct { diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 346170cde8..b10d869780 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -142,7 +142,7 @@ typedef struct SQuery { SResultRec rec; int32_t pos; int64_t pointsOffset; // the number of points offset to save read data - SData** sdata; + SData** sdata; int32_t capacity; SSingleColumnFilterInfo* pFilterInfo; } SQuery; @@ -171,11 +171,10 @@ typedef struct SQueryRuntimeEnv { typedef struct SQInfo { uint64_t signature; - void* pVnode; + void* pVnode; TSKEY startTime; int64_t elapsedTime; SResultRec rec; - int32_t pointsReturned; int32_t pointsInterpo; int32_t code; // error code to returned to client int32_t killed; // denotes if current query is killed @@ -184,7 +183,6 @@ typedef struct SQInfo { SQueryRuntimeEnv runtimeEnv; int32_t subgroupIdx; int32_t offset; /* offset in group result set of subgroup */ -// tSidSet* pSidSet; T_REF_DECLARE() /* @@ -226,7 +224,12 @@ void qSuperTableQuery(void* pReadMsg); */ int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize); - -//int32_t qBuildQueryResult(SQInfo* pQInfo, void* pBuf); +/** + * + * @param pQInfo + * @param pRsp + * @return + */ +int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen); #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 04ff9e56f8..c7aa57200c 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -219,7 +219,7 @@ typedef struct SQLAggFuncElem { void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus); + int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; typedef struct SPatternCompareInfo { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 04626227dd..af7d6f9ab0 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -119,7 +119,7 @@ static int32_t flushFromResultBuf(SQInfo *pQInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); -static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, +static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); @@ -427,15 +427,9 @@ static bool isTopBottomQuery(SQuery *pQuery) { return false; } -static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - int32_t columnIndex) { - // no SField info exist, or column index larger than the output column, no result. - if (pStatis == NULL) { - return NULL; - } - +static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) { // for a tag column, no corresponding field info - SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[columnIndex].pBase.colInfo; + SColIndexEx *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -453,13 +447,31 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo return NULL; } +/** + * @param pQuery + * @param col + * @param pDataBlockInfo + * @param pStatis + * @param pColStatis + * @return + */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[col].pBase.colInfo.flag) || pStatis == NULL) { + SColIndexEx* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } - - *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + + // query on primary timestamp column, not null value at all + if (pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return false; + } + + *pColStatis = NULL; + if (pStatis != NULL) { + *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + } + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { return false; } @@ -887,7 +899,6 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * the remain meter may not have the required column in cache actually. * So, the validation of required column in cache with the corresponding meter schema is reinforced. */ - if (pDataBlock == NULL) { return NULL; } @@ -939,10 +950,11 @@ static int32_t blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataSt int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *tpField = NULL; - bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); - char * dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->size, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, (char *)primaryKeyCol, pDataBlockInfo->size, functionId, tpField, + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->size, functionId, tpField, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); } @@ -1381,7 +1393,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl return 0; } -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char *primaryColumnData, int32_t size, +void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) { pCtx->scanFlag = scanFlag; @@ -1396,15 +1408,15 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char * pCtx->preAggVals.isSet = false; } - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (primaryColumnData != NULL)) { - pCtx->ptsList = (int64_t *)(primaryColumnData); + if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (tsCol != NULL)) { + pCtx->ptsList = tsCol; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null - pCtx->ptsList = (int64_t *)(primaryColumnData); + pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* @@ -1420,7 +1432,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, char * pTWAInfo->EKey = pQuery->window.ekey; } - pCtx->ptsList = (int64_t *)(primaryColumnData); + pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; @@ -2524,22 +2536,18 @@ static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow) { } SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo, SDataStatis **pStatis) { - SQuery * pQuery = pRuntimeEnv->pQuery; - tsdb_query_handle_t pQueryHandle = pRuntimeEnv->pQueryHandle; + SQuery *pQuery = pRuntimeEnv->pQuery; uint32_t r = 0; SArray * pDataBlock = NULL; - // STimeWindow *w = &pQueryHandle->window; - if (pQuery->numOfFilterCols > 0) { r = BLK_DATA_ALL_NEEDED; } else { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; - - // r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], w->skey, w->ekey, colId); + r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); } if (pRuntimeEnv->pTSBuf > 0 || isIntervalQuery(pQuery)) { @@ -2553,7 +2561,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl // pBlock->keyFirst, pBlock->keyLast, pBlock->numOfPoints); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { - // return DISK_DATA_LOAD_FAILED; +// return DISK_DATA_LOAD_FAILED; } if (pStatis == NULL) { @@ -3036,7 +3044,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { total += pData->numOfElems; } - pQuery->sdata[0]->num = total; + int32_t rows = total; int32_t offset = 0; for (int32_t num = 0; num < list.size; ++num) { @@ -3044,7 +3052,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - char * pDest = pQuery->sdata[i]->data; + char * pDest = pQuery->sdata[i]; memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->numOfElems, bytes * pData->numOfElems); @@ -3055,7 +3063,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { assert(pQuery->rec.pointsRead == 0); - pQuery->rec.pointsRead += pQuery->sdata[0]->num; + pQuery->rec.pointsRead += rows; pQInfo->offset += 1; } @@ -3222,7 +3230,8 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. int32_t pageId = -1; - int32_t remain = pQuery->sdata[0]->num; + assert(0); + int32_t remain = 0;//pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { @@ -3253,12 +3262,12 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; + pCtx[k].aOutputBuf = pQuery->sdata[k] - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; pCtx[k].resultInfo = &pResultInfo[k]; - pQuery->sdata[k]->num = 0; + pQuery->sdata[k] = 0; } } @@ -4080,11 +4089,6 @@ int32_t vnodeGetResultSize(void *thandle, int32_t *numOfRows) { } } -int64_t vnodeGetOffsetVal(void *thandle) { - SQInfo *pQInfo = (SQInfo *)thandle; - return pQInfo->runtimeEnv.pQuery->limit.offset; -} - bool vnodeHasRemainResults(void *handle) { SQInfo *pQInfo = (SQInfo *)handle; @@ -4162,62 +4166,13 @@ static int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **p } static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { -#if 0 - SMeterObj *pObj = pQInfo->pObj; - SQuery * pQuery = &pQInfo->query; - - int tnumOfRows = vnodeList[pObj->vnode].cfg.rowsInFileBlock; - - // for metric query, bufIndex always be 0. - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { // pQInfo->bufIndex == 0 + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { int32_t bytes = pQuery->pSelectExpr[col].resBytes; memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; } -#endif -} - -/** - * Copy the result data/file to output message buffer. - * If the result is in file format, read file from disk and copy to output buffer, compression is not involved since - * data in file is already compressed. - * In case of other result in buffer, compress the result before copy once the tsComressMsg is set. - * - * @param handle - * @param data - * @param numOfRows the number of rows that are not returned in current retrieve - * @return - */ -int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows) { - SQInfo *pQInfo = (SQInfo *)handle; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - assert(pQuery->pSelectExpr != NULL && pQuery->numOfOutputCols > 0); - - // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - - // make sure file exist - if (FD_VALID(fd)) { - size_t s = lseek(fd, 0, SEEK_END); - dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - - lseek(fd, 0, SEEK_SET); - read(fd, data, s); - close(fd); - - unlink(pQuery->sdata[0]->data); - } else { - dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, - pQuery->sdata[0]->data, strerror(errno)); - } - } else { - doCopyQueryResultToMsg(pQInfo, numOfRows, data); - } - - return numOfRows; } int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage **pDataSrc, int32_t numOfRows, @@ -4308,7 +4263,7 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param, void* tsdb) { +int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; @@ -4342,7 +4297,6 @@ int32_t vnodeQueryTablePrepare(SQInfo *pQInfo, void *param, void* tsdb) { taosArrayPush(cols, &pQuery->colList[i]); } - pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; @@ -4993,8 +4947,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { // vnodePrintQueryStatistics(pSupporter); } - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->rec.pointsRead, - pQInfo->rec.pointsRead, pQInfo->pointsReturned); + dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.pointsRead, pQInfo->rec.pointsTotal); return; } #if 0 @@ -5294,7 +5247,7 @@ void qTableQuery(SQInfo *pQInfo) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - (pQuery->intervalTime > 0 && pQInfo->pointsReturned < pQuery->limit.limit)) { + (pQuery->intervalTime > 0 && pQInfo->rec.pointsTotal < pQuery->limit.limit)) { // todo limit the output for interval query? pQuery->rec.pointsRead = 0; pQInfo->subgroupIdx = 0; // always start from 0 @@ -5436,11 +5389,6 @@ static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { return -1; } - if (pQueryTableMsg->numOfTagsCols < 0 || pQueryTableMsg->numOfTagsCols > TSDB_MAX_TAGS + 1) { - dError("qmsg:%p illegal value of numOfTagsCols %d", pQueryTableMsg, pQueryTableMsg->numOfTagsCols); - return -1; - } - if (pQueryTableMsg->numOfCols <= 0 || pQueryTableMsg->numOfCols > TSDB_MAX_COLUMNS) { dError("qmsg:%p illegal value of numOfCols %d", pQueryTableMsg, pQueryTableMsg->numOfCols); return -1; @@ -5496,8 +5444,15 @@ static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArra return pMsg; } -static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList) { - pQueryTableMsg->vgId = htons(pQueryTableMsg->vgId); +/** + * pQueryTableMsg->head has been converted before this function is called. + * + * @param pQueryTableMsg + * @param pTableIdList + * @param pExpr + * @return + */ +static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr) { pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); @@ -5513,7 +5468,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); - pQueryTableMsg->numOfTagsCols = htons(pQueryTableMsg->numOfTagsCols); pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); @@ -5572,27 +5526,23 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId bool hasArithmeticFunction = false; - /* - * 1. simple projection query on meters, we only record the pSqlFuncExprs[i].colIdx value - * 2. for complex queries, whole SqlExprs object is required. - */ - pQueryTableMsg->pSqlFuncExprs = (int64_t)malloc(POINTER_BYTES * pQueryTableMsg->numOfOutputCols); + *pExpr = calloc(pQueryTableMsg->numOfOutputCols, POINTER_BYTES); + SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { - ((SSqlFuncExprMsg **)pQueryTableMsg->pSqlFuncExprs)[i] = pExprMsg; + (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); + pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); pExprMsg->arg[j].argBytes = htons(pExprMsg->arg[j].argBytes); if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) { @@ -5628,22 +5578,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId pMsg = createTableIdList(pQueryTableMsg, pMsg, pTableIdList); - if (pQueryTableMsg->numOfGroupCols > 0 || pQueryTableMsg->numOfTagsCols > 0) { // group by tag columns - pQueryTableMsg->pTagSchema = (uint64_t)pMsg; - SSchema *pTagSchema = (SSchema *)pQueryTableMsg->pTagSchema; - pMsg += sizeof(SSchema) * pQueryTableMsg->numOfTagsCols; - - if (pQueryTableMsg->numOfGroupCols > 0) { - pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); - } else { - pQueryTableMsg->groupbyTagIds = 0; - } + if (pQueryTableMsg->numOfGroupCols > 0) { // group by tag columns +// if (pQueryTableMsg->numOfGroupCols > 0) { +// pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); +// } else { +// pQueryTableMsg->groupbyTagIds = 0; +// } pQueryTableMsg->orderByIdx = htons(pQueryTableMsg->orderByIdx); pQueryTableMsg->orderType = htons(pQueryTableMsg->orderType); pMsg += sizeof(SColIndexEx) * pQueryTableMsg->numOfGroupCols; } else { - pQueryTableMsg->pTagSchema = 0; pQueryTableMsg->groupbyTagIds = 0; } @@ -5657,13 +5602,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId } } - dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 - ", numOfGroupbyTagCols:%d, numOfTagCols:%d, timestamp order:%d, " - "tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 + dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " + "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64, pQueryTableMsg, pQueryTableMsg->numOfTables, pQueryTableMsg->window.skey, pQueryTableMsg->window.ekey, - pQueryTableMsg->numOfGroupCols, pQueryTableMsg->numOfTagsCols, pQueryTableMsg->order, - pQueryTableMsg->orderType, pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, + pQueryTableMsg->numOfGroupCols, pQueryTableMsg->order, pQueryTableMsg->orderType, + pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, pQueryTableMsg->numOfCols, pQueryTableMsg->intervalTime, pQueryTableMsg->interpoType, pQueryTableMsg->tsLen, pQueryTableMsg->limit, pQueryTableMsg->offset); @@ -5721,22 +5665,20 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr) { +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, SSqlFuncExprMsg** pExprMsg) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; SSqlFunctionExpr *pExprs = (SSqlFunctionExpr *)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols); if (pExprs == NULL) { - tfree(pQueryMsg->pSqlFuncExprs); return TSDB_CODE_SERV_OUT_OF_MEMORY; } bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); int16_t tagLen = 0; - SSchema *pTagSchema = (SSchema *)pQueryMsg->pTagSchema; for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *((SSqlFuncExprMsg **)pQueryMsg->pSqlFuncExprs)[i]; + pExprs[i].pBase = *pExprMsg[i]; pExprs[i].resBytes = 0; int16_t type = 0; @@ -5744,36 +5686,24 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct SColIndexEx *pColumnIndexExInfo = &pExprs[i].pBase.colInfo; - // tag column schema is kept in pQueryMsg->pColumnModel - if (TSDB_COL_IS_TAG(pColumnIndexExInfo->flag)) { - if (pColumnIndexExInfo->colIdx >= pQueryMsg->numOfTagsCols) { + // parse the arithmetic expression + if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { + code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); + + if (code != TSDB_CODE_SUCCESS) { tfree(pExprs); - - return TSDB_CODE_INVALID_QUERY_MSG; + return code; } - type = pTagSchema[pColumnIndexExInfo->colIdx].type; - bytes = pTagSchema[pColumnIndexExInfo->colIdx].bytes; + type = TSDB_DATA_TYPE_DOUBLE; + bytes = tDataTypeDesc[type].nSize; + } else { // parse the normal column + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + assert(j < pQueryMsg->numOfCols); - } else { // parse the arithmetic expression - if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { - code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); - - if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); - return code; - } - - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypeDesc[type].nSize; - } else { // parse the normal column - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); - assert(j < pQueryMsg->numOfCols); - - SColumnInfo *pCol = &pQueryMsg->colList[j]; - type = pCol->type; - bytes = pCol->bytes; - } + SColumnInfo *pCol = &pQueryMsg->colList[j]; + type = pCol->type; + bytes = pCol->bytes; } int32_t param = pExprs[i].pBase.arg[0].argValue.i64; @@ -5793,7 +5723,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct // TODO refactor for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { - pExprs[i].pBase = *((SSqlFuncExprMsg **)pQueryMsg->pSqlFuncExprs)[i]; + pExprs[i].pBase = *pExprMsg[i]; int16_t functId = pExprs[i].pBase.functionId; if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); @@ -5810,7 +5740,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct } } - tfree(pQueryMsg->pSqlFuncExprs); + tfree(pExprMsg); *pSqlFuncExpr = pExprs; return TSDB_CODE_SUCCESS; @@ -5962,8 +5892,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i].info = pQueryMsg->colList[i]; - // SColumnInfo *pColInfo = &pQuery->colList[i].data; - // pColInfo->filters = NULL; + + SColumnInfo *pColInfo = &pQuery->colList[i].info; + pColInfo->filters = NULL; // if (colList[i].numOfFilters > 0) { // pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); // @@ -5987,7 +5918,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // prepare the result buffer - pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, sizeof(SData *)); + pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, POINTER_BYTES); if (pQuery->sdata == NULL) { goto _clean_memory; } @@ -6142,12 +6073,10 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE pQuery->window.skey = pQueryMsg->window.skey; pQuery->window.ekey = pQueryMsg->window.ekey; - - pQuery->lastKey = pQuery->window.skey; + pQuery->lastKey = pQuery->window.skey; if (sem_init(&(*pQInfo)->dataReady, 0, 0) != 0) { - // dError("QInfo:%p vid:%d sid:%d meterId:%s, init dataReady sem failed, reason:%s", pQInfo, pMeterObj->vnode, - // pMeterObj->sid, pMeterObj->meterId, strerror(errno)); + dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); code = TSDB_CODE_APP_ERROR; goto _error; } @@ -6163,7 +6092,7 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE tsBufNextPos(pTSBuf); } - if ((code = vnodeQueryTablePrepare(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { + if ((code = initQInfo(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -6188,7 +6117,8 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ int32_t code = TSDB_CODE_SUCCESS; SArray *pTableIdList = NULL; - if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList)) != TSDB_CODE_SUCCESS) { + SSqlFuncExprMsg** pExprMsg = NULL; + if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList, &pExprMsg)) != TSDB_CODE_SUCCESS) { return code; } @@ -6206,7 +6136,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ } SSqlFunctionExpr *pExprs = NULL; - if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs)) != TSDB_CODE_SUCCESS) { + if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { goto _query_over; } @@ -6261,15 +6191,119 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *ro sem_wait(&pQInfo->dataReady); SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + *numOfRows = pQInfo->rec.pointsRead; + *rowsize = pQuery->rowSize; - // *numOfRows = pQInfo->rec.pointsRead; - // *rowsize = pQuery->rowSize; - *numOfRows = 1; - - // dTrace("QInfo:%p, retrieve data info completed, precision:%d, rowsize:%d, rows:%d, code:%d", pQInfo, *timePrec, - // *rowsize, *numOfRows, pQInfo->code); + dTrace("QInfo:%p, retrieve res info, rowsize:%d, rows:%d, code:%d", pQInfo, *rowsize, *numOfRows, pQInfo->code); if (pQInfo->code < 0) { // less than 0 means there are error existed. return -pQInfo->code; } } + +static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); + } +} + +static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int32_t *size) { + // the remained number of retrieved rows, not the interpolated result + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, pQInfo->rec.pointsRead, data); + } + + pQInfo->rec.pointsTotal += pQInfo->rec.pointsRead; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQInfo->rec.pointsRead, pQInfo->rec.pointsTotal); +} + +static void addToTaskQueue(SQInfo* pQInfo) { + // no error occurred, continue retrieving data + if (pQInfo->code == TSDB_CODE_SUCCESS) { +#ifdef _TD_ARM_ + dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:doDumpQueryResult", pQInfo, pQInfo->signature); +#else + dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:%s", pQInfo, pQInfo->signature, __FUNCTION__); +#endif + + if (pQInfo->killed == 1) { + dTrace("%p freed or killed, abort query", pQInfo); + } else { + // todo add to task queue + } + } +} + +int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { + if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + return TSDB_CODE_INVALID_QHANDLE; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + size_t size = getResultSize(pQInfo, &pQInfo->rec.pointsRead); + *contLen = size + sizeof(SRetrieveTableRsp); + + // todo handle failed to allocate memory + *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); + + (*pRsp)->numOfRows = htonl(pQInfo->rec.pointsRead); + + int32_t code = pQInfo->code; + if (code == TSDB_CODE_SUCCESS) { + (*pRsp)->offset = htobe64(pQuery->limit.offset); + (*pRsp)->useconds = htobe64(pQInfo->elapsedTime); + } else { + (*pRsp)->offset = 0; + (*pRsp)->useconds = 0; + } + + if (pQInfo->rec.pointsRead > 0 && code == TSDB_CODE_SUCCESS) { + doDumpQueryResult(pQInfo, (*pRsp)->data, NULL); + addToTaskQueue(pQInfo); + return TSDB_CODE_SUCCESS; + } + + assert(code != TSDB_CODE_ACTION_IN_PROGRESS); + +// if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { +// dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); +// vnodeDecRefCount(pObj->qhandle); +// pObj->qhandle = NULL; +// } +} \ No newline at end of file diff --git a/src/util/src/shash.c b/src/util/src/shash.c index da97af84bb..525d00e81e 100644 --- a/src/util/src/shash.c +++ b/src/util/src/shash.c @@ -162,6 +162,8 @@ void taosDeleteStrHash(void *handle, char *string) { if (pObj == NULL || pObj->maxSessions == 0) return; if (string == NULL || string[0] == 0) return; + return; + hash = (*(pObj->hashFp))(pObj, string); pthread_mutex_lock(&pObj->mutex); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index d654308f99..1c4f09f4c9 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -109,7 +109,6 @@ typedef struct STsdbQueryHandle { uint16_t flag; // denotes reversed scan of data or not int16_t order; STimeWindow window; // the primary query time window that applies to all queries - TSKEY lastKey; int32_t blockBufferSize; SCompBlock *pBlock; int32_t numOfBlocks; @@ -264,8 +263,20 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond pQueryHandle->pColumns = pColumnInfo; pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; - - pQueryHandle->lastKey = pQueryHandle->window.skey; // ascending query + + // only support table query + assert(taosArrayGetSize(idList) == 1); + + pQueryHandle->pTableQueryInfo = calloc(1, sizeof(STableQueryRec)); + STableQueryRec* pTableQRec = pQueryHandle->pTableQueryInfo; + + pTableQRec->lastKey = pQueryHandle->window.skey; + + STableIdInfo* idInfo = taosArrayGet(pQueryHandle->pTableIdList, 0); + + STableId tableId = {.uid = idInfo->uid, .tid = idInfo->sid}; + STable *pTable = tsdbIsValidTableToInsert(tsdbGetMeta(pQueryHandle->pTsdb), tableId); + pTableQRec->pTableObj = pTable; // malloc buffer in order to load data from file int32_t numOfCols = taosArrayGetSize(pColumnInfo); @@ -295,14 +306,21 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond return (tsdb_query_handle_t)pQueryHandle; } -static int32_t next = 1; bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - if (next == 0) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + STable *pTable = pHandle->pTableQueryInfo->pTableObj; + + // no data in cache, abort + if (pTable->mem == NULL && pTable->imem == NULL) { return false; - } else { - next = 0; - return true; } + + // all data in mem are checked already. + if (pHandle->pTableQueryInfo->lastKey > pTable->mem->keyLast) { + return false; + } + + return true; } static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, @@ -344,9 +362,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; STableIdInfo* idInfo = taosArrayGet(pHandle->pTableIdList, 0); - STableId tableId = {.uid = idInfo->uid, .tid = idInfo->sid}; - STable *pTable = tsdbIsValidTableToInsert(tsdbGetMeta(pHandle->pTsdb), tableId); - assert(pTable != NULL); + STable *pTable = pHandle->pTableQueryInfo->pTableObj; TSKEY skey = 0, ekey = 0; int32_t rows = 0; @@ -357,8 +373,8 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { } SDataBlockInfo blockInfo = { - .uid = tableId.uid, - .sid = tableId.tid, + .uid = idInfo->uid, + .sid = idInfo->sid, .size = rows, .window = {.skey = skey, .ekey = ekey} }; From eed1d76b3f8b08ef060048803690cadcd53f1abe Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 24 Mar 2020 11:01:04 +0800 Subject: [PATCH 25/85] TD-34 --- src/vnode/tsdb/src/tsdbCache.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index f51c7c12d4..3496f6a5c8 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -17,8 +17,9 @@ #include "tsdb.h" #include "tsdbCache.h" -static int tsdbAllocBlockFromPool(STsdbCache *pCache); -static void tsdbFreeBlockList(SCacheMem *mem); +static int tsdbAllocBlockFromPool(STsdbCache *pCache); +static void tsdbFreeBlockList(SList *list); +static void tsdbFreeCacheMem(SCacheMem *mem); STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo) { STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); @@ -60,8 +61,8 @@ _err: } void tsdbFreeCache(STsdbCache *pCache) { - tsdbFreeBlockList(pCache->imem); - tsdbFreeBlockList(pCache->mem); + tsdbFreeCacheMem(pCache->imem); + tsdbFreeCacheMem(pCache->mem); tsdbFreeBlockList(pCache->pool.memPool); free(pCache); } @@ -90,9 +91,7 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { return ptr; } -static void tsdbFreeBlockList(SCacheMem *mem) { - if (mem == NULL) return; - SList * list = mem->list; +static void tsdbFreeBlockList(SList *list) { SListNode * node = NULL; STsdbCacheBlock *pBlock = NULL; while ((node = tdListPopHead(list)) != NULL) { @@ -101,6 +100,12 @@ static void tsdbFreeBlockList(SCacheMem *mem) { listNodeFree(node); } tdListFree(list); +} + +static void tsdbFreeCacheMem(SCacheMem *mem) { + if (mem == NULL) return; + SList *list = mem->list; + tsdbFreeBlockList(list); free(mem); } From af5b304f19d560c4e0f585ca74bd6c72c629ae99 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 24 Mar 2020 11:22:39 +0800 Subject: [PATCH 26/85] [td-32] fix bugs in returning data to client --- src/client/src/tscServer.c | 14 +++++------ src/dnode/src/dnodeRead.c | 44 ++++++++++++----------------------- src/query/src/queryExecutor.c | 11 ++++----- src/vnode/tsdb/src/tsdbRead.c | 3 +++ 4 files changed, 30 insertions(+), 42 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3da374f152..605c7a22cc 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -273,8 +273,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } pSql->retry = 0; - pRes->rspLen = 0; + if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_NETWORK_UNAVAIL; } else { @@ -283,9 +283,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { assert(rpcMsg->msgType == pCmd->msgType + 1); - pRes->code = (int32_t)rpcMsg->code; + pRes->code = rpcMsg->code; pRes->rspType = rpcMsg->msgType; - pRes->rspLen = rpcMsg->contLen; + pRes->rspLen = rpcMsg->contLen; char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); if (tmp == NULL) { @@ -389,7 +389,7 @@ int tscProcessSql(SSqlObj *pSql) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = NULL; - uint16_t type = 0; + uint16_t type = 0; if (pQueryInfo != NULL) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -2302,9 +2302,9 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { pRes->numOfRows = htonl(pRetrieve->numOfRows); pRes->precision = htons(pRetrieve->precision); - pRes->offset = htobe64(pRetrieve->offset); - pRes->useconds = htobe64(pRetrieve->useconds); - pRes->data = pRetrieve->data; + pRes->offset = htobe64(pRetrieve->offset); + pRes->useconds = htobe64(pRetrieve->useconds); + pRes->data = pRetrieve->data; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); tscSetResultPointer(pQueryInfo, pRes); diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index bfe717c367..9e50804583 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -254,7 +254,7 @@ static void dnodeProcessQueryMsg(SReadMsg *pMsg) { static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { SRetrieveTableMsg *pRetrieve = pMsg->pCont; - void *pQInfo = htobe64(pRetrieve->qhandle); + void *pQInfo = (void*) htobe64(pRetrieve->qhandle); dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId); @@ -263,40 +263,26 @@ static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { int32_t contLen = 0; SRpcMsg rpcRsp = {0}; + SRetrieveTableRsp *pRsp = NULL; int32_t code = qRetrieveQueryResultInfo(pQInfo, &numOfRows, &rowSize); if (code != TSDB_CODE_SUCCESS) { contLen = sizeof(SRetrieveTableRsp); - - SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); - pRsp->numOfRows = 0; - pRsp->precision = 0; - pRsp->offset = 0; - pRsp->useconds = 0; - - rpcRsp = (SRpcMsg) { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = contLen, - .code = code, - .msgType = 0 - }; - - //todo free qinfo + + pRsp = (SRetrieveTableRsp *)rpcMallocCont(contLen); + memset(pRsp, 0, sizeof(SRetrieveTableRsp)); } else { - SRetrieveTableRsp* pRsp = NULL; - - int32_t code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); - //todo check code - - rpcRsp = (SRpcMsg) { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = contLen, - .code = code, - .msgType = 0 - }; + // todo check code and handle error in build result set + code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); } + rpcRsp = (SRpcMsg) { + .handle = pMsg->rpcMsg.handle, + .pCont = pRsp, + .contLen = contLen, + .code = code, + .msgType = 0 + }; + rpcSendResponse(&rpcRsp); } diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index af7d6f9ab0..696d8c37c9 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2698,10 +2698,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); - // dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, rows:%d, - // checked:%d", - // GET_QINFO_ADDR(pQuery), blockInfo.window.skey, blockInfo.window.ekey, pQueryHandle->cur.fileId, - // pQueryHandle->cur.slot, pQuery->pos, blockInfo.size, forwardStep); + dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.size); // save last access position cnt += forwardStep; @@ -6189,12 +6187,13 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *ro } sem_wait(&pQInfo->dataReady); - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + *numOfRows = pQInfo->rec.pointsRead; *rowsize = pQuery->rowSize; - dTrace("QInfo:%p, retrieve res info, rowsize:%d, rows:%d, code:%d", pQInfo, *rowsize, *numOfRows, pQInfo->code); + dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, *rowsize, *numOfRows, pQInfo->code); if (pQInfo->code < 0) { // less than 0 means there are error existed. return -pQInfo->code; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 1c4f09f4c9..31cdd70f36 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -379,6 +379,9 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { .window = {.skey = skey, .ekey = ekey} }; + // update the last key value + pHandle->pTableQueryInfo->lastKey = ekey + 1; + return blockInfo; } From d2e3ed8b7ddd7d6a3f6932c30584931a638284db Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 24 Mar 2020 12:49:22 +0800 Subject: [PATCH 27/85] [TD-15] refact code of sdb --- src/inc/mnode.h | 20 ++--- src/mnode/src/mgmtChildTable.c | 49 ++++-------- src/mnode/src/mgmtDb.c | 4 +- src/mnode/src/mgmtNormalTable.c | 66 ++++------------- src/mnode/src/mgmtSdb.c | 39 +++++----- src/mnode/src/mgmtSuperTable.c | 2 +- src/mnode/src/mgmtTable.c | 127 +++++++++++++++++++++----------- src/mnode/src/mgmtUser.c | 4 +- src/mnode/src/mgmtVgroup.c | 8 +- src/util/src/tidpool.c | 35 ++++----- 10 files changed, 165 insertions(+), 189 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index cde26dad4a..792a8ec1cc 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -53,8 +53,8 @@ typedef struct { int8_t numOfMnodes; int32_t numOfDnodes; char mnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; int syncFd; void *hbTimer; void *pSync; @@ -78,8 +78,8 @@ typedef struct { float lbScore; // calc in balance function int32_t customScore; // config by user char dnodeName[TSDB_DNODE_NAME_LEN + 1]; - char reserved[7]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; SVnodeLoad vload[TSDB_MAX_VNODES]; int32_t status; uint32_t lastReboot; // time stamp for last reboot @@ -120,7 +120,7 @@ typedef struct SSuperTableObj { int32_t sversion; int32_t numOfColumns; int32_t numOfTags; - int8_t reserved[5]; + int8_t reserved[15]; int8_t updateEnd[1]; int32_t numOfTables; int16_t nextColId; @@ -152,7 +152,7 @@ typedef struct { int32_t sversion; int32_t numOfColumns; int32_t sqlLen; - int8_t reserved[3]; + int8_t reserved[7]; int8_t updateEnd[1]; char* sql; //null-terminated string int16_t nextColId; @@ -167,7 +167,6 @@ typedef struct _vg_obj { int64_t createdTime; SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; int32_t numOfVnodes; - int32_t numOfTables; int32_t lbIp; int32_t lbTime; int8_t lbStatus; @@ -175,6 +174,7 @@ typedef struct _vg_obj { int8_t updateEnd[1]; struct _vg_obj *prev, *next; struct _db_obj *pDb; + int32_t numOfTables; void * idPool; STableInfo ** tableList; } SVgObj; @@ -184,8 +184,8 @@ typedef struct _db_obj { int8_t dirty; int64_t createdTime; SDbCfg cfg; - char reserved[15]; - char updateEnd[1]; + int8_t reserved[15]; + int8_t updateEnd[1]; struct _db_obj *prev, *next; int32_t numOfVgroups; int32_t numOfTables; @@ -203,7 +203,7 @@ typedef struct _user_obj { int64_t createdTime; int8_t superAuth; int8_t writeAuth; - int8_t reserved[16]; + int8_t reserved[13]; int8_t updateEnd[1]; struct _user_obj *prev, *next; struct _acctObj * pAcct; diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index d717a4e5e9..9076681caa 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -43,8 +43,7 @@ static void mgmtDestroyChildTable(SChildTableObj *pTable) { } static int32_t mgmtChildTableActionDestroy(SSdbOperDesc *pOper) { - SChildTableObj *pTable = pOper->pObj; - mgmtDestroyChildTable(pTable); + mgmtDestroyChildTable(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -69,14 +68,6 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { return TSDB_CODE_INVALID_ACCT; } - if (!mgmtIsMaster()) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("ctable:%s, sid:%d is not matched from the master:%d", pTable->tableId, sid, pTable->sid); - return TSDB_CODE_INVALID_SESSION_ID; - } - } - pTable->superTable = mgmtGetSuperTable(pTable->superTableId); mgmtAddTableIntoSuperTable(pTable->superTable); @@ -84,10 +75,6 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pVgroup); - } - return TSDB_CODE_SUCCESS; } @@ -117,13 +104,8 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - mgmtRemoveTableFromSuperTable(pTable->superTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pVgroup); - } - return TSDB_CODE_SUCCESS; } @@ -136,7 +118,9 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { assert(pTable != NULL && pOper->rowData != NULL); memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); - return tsChildTableUpdateSize; + pOper->rowSize = tsChildTableUpdateSize; + + return TSDB_CODE_SUCCESS; } static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { @@ -158,7 +142,7 @@ int32_t mgmtInitChildTables() { SChildTableObj *pTable = NULL; SChildTableObj tObj; - tsChildTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "ctables", @@ -236,9 +220,6 @@ int32_t mgmtInitChildTables() { continue; } - pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); if (pSuperTable == NULL) { mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); @@ -251,12 +232,6 @@ int32_t mgmtInitChildTables() { pNode = pLastNode; continue; } - - pTable->superTable = pSuperTable; - mgmtAddTableIntoSuperTable(pSuperTable); - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); } mTrace("child table is initialized"); @@ -268,8 +243,13 @@ void mgmtCleanUpChildTables() { } void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { - char *pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; - int32_t tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + char * pTagData = NULL; + int32_t tagDataLen = 0; + if (pMsg != NULL) { + pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; + tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + } + int32_t totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen; @@ -302,7 +282,10 @@ void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTab pSchema++; } - memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + if (pMsg != NULL) { + memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + } + return pCreate; } diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index e8ab5aaa8c..0cc793f2c0 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -117,7 +117,7 @@ static int32_t mgmtDbActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitDbs() { SDbObj tObj; - tsDbUpdateSize = tObj.updateEnd - (char *)&tObj; + tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "dbs", @@ -276,8 +276,6 @@ static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS; } - pCreate->maxSessions++; - return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index 0ffd4fb28e..4e5e4ce7cc 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -33,7 +33,7 @@ #include "mgmtTable.h" #include "mgmtVgroup.h" -void *tsNormalTableSdb; +void *tsNormalTableSdb; int32_t tsNormalTableUpdateSize; static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { @@ -43,8 +43,7 @@ static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { } static int32_t mgmtNormalTableActionDestroy(SSdbOperDesc *pOper) { - SNormalTableObj *pTable = pOper->pObj; - mgmtDestroyNormalTable(pTable); + mgmtDestroyNormalTable(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -53,38 +52,26 @@ static int32_t mgmtNormalTableActionInsert(SSdbOperDesc *pOper) { SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("id:%s not in vgroup:%d", pTable->tableId, pTable->vgId); + mError("ntable:%s not in vgroup:%d", pTable->tableId, pTable->vgId); return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("account not exists"); + mError("ntable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - if (!mgmtIsMaster()) { - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid != pTable->sid) { - mError("sid:%d is not matched from the master:%d", sid, pTable->sid); - return TSDB_CODE_INVALID_SESSION_ID; - } - } - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); mgmtAddTableIntoDb(pDb); mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables >= pDb->cfg.maxSessions - 1 && pDb->numOfVgroups > 1) { - mgmtMoveVgroupToTail(pVgroup); - } - return TSDB_CODE_SUCCESS; } @@ -101,7 +88,7 @@ static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("vgroup:%d not in DB:%s", pVgroup->vgId, pVgroup->dbName); + mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } @@ -115,22 +102,10 @@ static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - if (pVgroup->numOfTables > 0) { - mgmtMoveVgroupToHead(pVgroup); - } - return TSDB_CODE_SUCCESS; } static int32_t mgmtNormalTableActionUpdate(SSdbOperDesc *pOper) { - // SNormalTableObj *pTable = (SNormalTableObj *) pObj; - // memcpy(pTable, str, tsNormalTableUpdateSize); - - // int32_t schemaSize = sizeof(SSchema) * (pTable->numOfColumns) + pTable->sqlLen; - // pTable->schema = realloc(pTable->schema, schemaSize); - // pTable->sql = (char*)pTable->schema + sizeof(SSchema) * (pTable->numOfColumns); - // memcpy(pTable->schema, str + tsNormalTableUpdateSize, schemaSize); - return TSDB_CODE_SUCCESS; } @@ -139,25 +114,24 @@ static int32_t mgmtNormalTableActionEncode(SSdbOperDesc *pOper) { assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize + 1) { - return -1; + if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; } memcpy(pOper->rowData, pTable, tsNormalTableUpdateSize); memcpy(pOper->rowData + tsNormalTableUpdateSize, pTable->schema, schemaSize); memcpy(pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - - return tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + + pOper->rowSize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; + return TSDB_CODE_SUCCESS; } static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { assert(pOper->rowData != NULL); - SNormalTableObj *pTable = (SNormalTableObj *)malloc(sizeof(SNormalTableObj)); - if (pTable == NULL) { - return -1; - } - memset(pTable, 0, sizeof(SNormalTableObj)); + SNormalTableObj *pTable = (SNormalTableObj *)calloc(1, sizeof(SNormalTableObj)); + if (pTable == NULL) TSDB_CODE_SERV_OUT_OF_MEMORY; + memcpy(pTable, pOper->rowData, tsNormalTableUpdateSize); int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); @@ -176,6 +150,7 @@ static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { } memcpy(pTable->sql, pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); + pOper->pObj = pTable; return TSDB_CODE_SUCCESS; } @@ -185,7 +160,7 @@ int32_t mgmtInitNormalTables() { SNormalTableObj *pTable = NULL; SNormalTableObj tObj; - tsNormalTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsNormalTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "ntables", @@ -261,15 +236,6 @@ int32_t mgmtInitNormalTables() { pNode = pLastNode; continue; } - - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *)pTable); - //pVgroup->tableList[pTable->sid] = (STableInfo*)pTable; - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); - - pTable->sql = (char *)pTable->schema + sizeof(SSchema) * pTable->numOfColumns; - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); } mTrace("ntables is initialized"); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index eee8c0f1a0..e950761a31 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -309,11 +309,11 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); pTable->numOfRows++; - sdbTrace("table:%s, read new record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read new record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } } } else { @@ -325,8 +325,8 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { (*pTable->destroyFp)(&oper); (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, rowHead->data); pTable->numOfRows--; - sdbTrace("table:%s, read deleted record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read deleted record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } else { SRowMeta rowMeta; rowMeta.version = rowHead->version; @@ -346,11 +346,11 @@ static int32_t sdbInitTableByFile(SSdbTable *pTable) { if (code == TSDB_CODE_SUCCESS) { rowMeta.row = oper.pObj; (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, rowMeta.row, &rowMeta); - sdbTrace("table:%s, read updated record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, read updated record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } else { - sdbError("table:%s, failed to decode record:%s, numOfRows:%d version:%" PRId64 , - pTable->tableName, sdbGetkeyStr(pTable, rowHead->data), pTable->numOfRows, pTable->version); + sdbTrace("table:%s, version:%" PRId64 " numOfRows:%d, failed to decode record:%s", + pTable->tableName, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, rowHead->data)); } } numOfChanged++; @@ -416,8 +416,8 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->tableId = sdbNumOfTables++; sdbTableList[pTable->tableId] = pTable; - sdbTrace("table:%s, is initialized, numOfRows:%d numOfTables:%d version:%" PRId64 " sdbversion:%" PRId64, - pTable->tableName, pTable->numOfRows, sdbNumOfTables, pTable->version, sdbVersion); + sdbTrace("table:%s, is initialized, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d numOfTables:%d", + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbNumOfTables); return pTable; } @@ -531,8 +531,8 @@ int32_t sdbInsertRow(SSdbOperDesc *pOper) { pthread_mutex_unlock(&pTable->mutex); - sdbTrace("table:%s, inserte record:%s, sdbversion:%" PRId64 " version:%" PRId64 " rowSize:%d numOfRows:%d fileSize:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pOper->rowSize, pTable->numOfRows, pTable->fileSize); + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, insert record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); (*pTable->insertFp)(pOper); @@ -614,8 +614,9 @@ int32_t sdbDeleteRow(SSdbOperDesc *pOper) { tfree(rowHead); - sdbTrace("table:%s, delete record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d", - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); + + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, delete record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); // Delete from current layer (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); @@ -702,9 +703,9 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { pTable->fileSize += real_size; sdbFinishCommit(pTable); - sdbTrace("table:%s, update record:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%" PRId64, - pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj), sdbVersion, pTable->version, pTable->numOfRows); - + sdbTrace("table:%s, sdbversion:%" PRId64 " version:%" PRId64 " numOfRows:%d, update record:%s, rowSize:%d fileSize:%" PRId64, + pTable->tableName, sdbVersion, pTable->version, pTable->numOfRows, sdbGetkeyStr(pTable, pOper->pObj), pOper->rowSize, pTable->fileSize); + pMeta->version = pTable->version; pMeta->offset = pTable->fileSize; pMeta->rowSize = rowHead->rowSize; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index d3a0dcb6bc..f2b54ec000 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -111,7 +111,7 @@ static int32_t mgmtSuperTableActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitSuperTables() { SSuperTableObj tObj; - tsSuperTableUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "stables", diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 918547fa12..675de57924 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -22,6 +22,7 @@ #include "mgmtDClient.h" #include "mgmtDb.h" #include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtGrant.h" #include "mgmtMnode.h" #include "mgmtNormalTable.h" @@ -61,6 +62,7 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle); +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); int32_t mgmtInitTables() { int32_t code = mgmtInitSuperTables(); @@ -90,6 +92,7 @@ int32_t mgmtInitTables() { mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg); return TSDB_CODE_SUCCESS; } @@ -760,9 +763,19 @@ static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->code != TSDB_CODE_SUCCESS) { if (pTable->type == TSDB_CHILD_TABLE) { - // sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else if (pTable->type == TSDB_NORMAL_TABLE){ - // sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL); + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); } else {} mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); @@ -816,19 +829,31 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { } if (pTable->type == TSDB_CHILD_TABLE) { - // if (sdbDeleteRow(tsChildTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - // mError("table:%s, update ctables sdb error", pTable->tableId); - // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - // free(queueMsg); - // return; - // } + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, update ctables sdb error", pTable->tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + free(queueMsg); + return; + } } else if (pTable->type == TSDB_NORMAL_TABLE){ - // if (sdbDeleteRow(tsNormalTableSdb, pTable, SDB_OPER_GLOBAL) < 0) { - // mError("table:%s, update ntables sdb error", pTable->tableId); - // mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - // free(queueMsg); - // return; - // } + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsNormalTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, update ntables sdb error", pTable->tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + free(queueMsg); + return; + } } if (pVgroup->numOfTables <= 0) { @@ -844,32 +869,48 @@ static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); } -// -// -//static void mgmtProcessTableCfgMsg(int8_t msgType, int8_t *pCont, int32_t contLen, void *thandle) { -// SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) pCont; -// pCfg->dnode = htonl(pCfg->dnode); -// pCfg->vnode = htonl(pCfg->vnode); -// pCfg->sid = htonl(pCfg->sid); -// mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// -// if (!sdbMaster) { -// mError("dnode:%s, vnode:%d, sid:%d, not master, redirect it", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_REDIRECT, NULL, 0); -// return; -// } -// -// STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); -// if (pTable == NULL) { -// mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_INVALID_TABLE, NULL, 0); -// return; -// } -// -// mgmtSendRspToDnode(thandle, msgType + 1, TSDB_CODE_SUCCESS, NULL, 0); -// -// //TODO -// SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); -// mgmtSendCreateTableMsg(NULL, &ipSet, NULL); -//} -// \ No newline at end of file +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + pCfg->sid = htonl(pCfg->sid); + mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + + STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); + if (pTable == NULL) { + mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SMDCreateTableMsg *pMDCreate = NULL; + if (pTable->type == TSDB_CHILD_TABLE) { + mTrace("table:%s, is a child table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateChildTableMsg(NULL, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else if (pTable->type == TSDB_NORMAL_TABLE) { + mTrace("table:%s, is a normal table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + } else { + mError("table:%s, invalid msg type, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); + } + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SRpcMsg rpcRsp = { + .handle = NULL, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + mgmtSendMsgToDnode(&ipSet, &rpcRsp); +} diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index e55ba9ea60..576b326669 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -94,7 +94,7 @@ static int32_t mgmtUserActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitUsers() { SUserObj tObj; - tsUserUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "users", @@ -471,7 +471,7 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { if (hasRight) { code = mgmtDropUser(pUser->pAcct, pDrop->user); if (code == TSDB_CODE_SUCCESS) { - mLPrint("user:%s is dropped by %s", pDrop->user, pUser->user); + mLPrint("user:%s is dropped by %s, result:%d", pUser->user, pOperUser->user, tstrerror(code)); } } else { code = TSDB_CODE_NO_RIGHTS; diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index b37b363077..9e52574b8c 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -142,7 +142,7 @@ static int32_t mgmtVgroupActionDecode(SSdbOperDesc *pOper) { int32_t mgmtInitVgroups() { SVgObj tObj; - tsVgUpdateSize = tObj.updateEnd - (int8_t *)&tObj; + tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; SSdbTableDesc tableDesc = { .tableName = "vgroups", @@ -433,13 +433,10 @@ void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { pVgroup->tableList[pTable->sid] = pTable; taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); pVgroup->numOfTables++; - pVgroup->pDb->numOfTables++; } if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) mgmtAddVgroupIntoDbTail(pVgroup); - else - mgmtAddVgroupIntoDb(pVgroup); } void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { @@ -447,13 +444,10 @@ void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { pVgroup->tableList[pTable->sid] = NULL; taosFreeId(pVgroup->idPool, pTable->sid); pVgroup->numOfTables--; - pVgroup->pDb->numOfTables--; } if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) mgmtAddVgroupIntoDbTail(pVgroup); - else - mgmtAddVgroupIntoDb(pVgroup); } SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index c3f429ac43..5ed6b8cf23 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -15,12 +15,13 @@ #include "os.h" #include "tlog.h" +#include typedef struct { int maxId; int numOfFree; int freeSlot; - int * freeList; + bool * freeList; pthread_mutex_t mutex; } id_pool_t; @@ -28,7 +29,7 @@ void *taosInitIdPool(int maxId) { id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - pIdPool->freeList = malloc(sizeof(int) * (size_t)maxId); + pIdPool->freeList = calloc(maxId, sizeof(bool)); if (pIdPool->freeList == NULL) { free(pIdPool); return NULL; @@ -40,10 +41,6 @@ void *taosInitIdPool(int maxId) { pthread_mutex_init(&pIdPool->mutex, NULL); - for (int i = 0; i < maxId; ++i) { - pIdPool->freeList[i] = -1; - } - pTrace("pool:%p is setup, maxId:%d", pIdPool, pIdPool->maxId); return pIdPool; @@ -61,8 +58,8 @@ int taosAllocateId(void *handle) { if (pIdPool->numOfFree > 0) { for (int i = 0; i < pIdPool->maxId; ++i) { slot = (i + pIdPool->freeSlot) % pIdPool->maxId; - if (pIdPool->freeList[slot] == -1) { - pIdPool->freeList[slot] = slot; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; pIdPool->freeSlot = slot + 1; pIdPool->numOfFree--; break; @@ -71,7 +68,7 @@ int taosAllocateId(void *handle) { } pthread_mutex_unlock(&pIdPool->mutex); - return slot; + return slot + 1; } void taosFreeId(void *handle, int id) { @@ -80,9 +77,9 @@ void taosFreeId(void *handle, int id) { pthread_mutex_lock(&pIdPool->mutex); - int slot = id % pIdPool->maxId; - if (pIdPool->freeList[slot] != -1) { - pIdPool->freeList[slot] = -1; + int slot = (id - 1) % pIdPool->maxId; + if (pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = false; pIdPool->numOfFree++; } @@ -114,9 +111,9 @@ void taosIdPoolMarkStatus(void *handle, int id) { id_pool_t *pIdPool = handle; pthread_mutex_lock(&pIdPool->mutex); - int slot = id % pIdPool->maxId; - if (pIdPool->freeList[slot] == -1) { - pIdPool->freeList[slot] = slot; + int slot = (id - 1) % pIdPool->maxId; + if (!pIdPool->freeList[slot]) { + pIdPool->freeList[slot] = true; pIdPool->numOfFree--; } @@ -129,18 +126,14 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return -1; } - int *idList = malloc(sizeof(int) * maxId); + int *idList = calloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } pthread_mutex_lock(&pIdPool->mutex); - for (int i = 0; i < maxId; ++i) { - idList[i] = -1; - } - - memcpy(idList, pIdPool->freeList, sizeof(int) * pIdPool->maxId); + memcpy(idList, pIdPool->freeList, sizeof(bool) * pIdPool->maxId); pIdPool->numOfFree += (maxId - pIdPool->maxId); pIdPool->maxId = maxId; From 305523f47ab9710f81624590fa3144c6e25a4c0a Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 24 Mar 2020 15:10:41 +0800 Subject: [PATCH 28/85] TD-34 --- src/vnode/tsdb/inc/tsdbFile.h | 5 ---- src/vnode/tsdb/src/tsdbMain.c | 50 ++++++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 7f3acb6624..385d1f59ea 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -36,11 +36,6 @@ typedef enum { extern const char *tsdbFileSuffix[]; -typedef struct { - int64_t size; - int64_t tombSize; -} SFileInfo; - typedef struct { int8_t type; char fname[128]; diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 2df7974844..afd57947b7 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -782,6 +782,39 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max return numOfRows; } +static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { + if (iters == NULL) return; + + for (int tid = 0; tid < maxTables; tid++) { + if (iters[tid] == NULL) continue; + tSkipListDestroy(iters[tid]); + } + + free(iters); +} + +static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) { + SSkipListIterator **iters = (SSkipListIterator *)calloc(maxTables, sizeof(SSkipListIterator *)); + if (iters == NULL) return NULL; + + for (int tid = 0; tid < maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL || pTable->imem == NULL) continue; + + iters[tid] = tSkipListCreateIter(pTable->imem->pData); + if (iters[tid] == NULL) { + tsdbDestroyTableIters(iters, maxTables); + return NULL; + } + + if (!tSkipListIterNext(iters[tid])) { + assert(false); + } + } + + return iters; +} + // Commit to file static void *tsdbCommitToFile(void *arg) { // TODO @@ -791,10 +824,8 @@ static void *tsdbCommitToFile(void *arg) { STsdbCfg * pCfg = &(pRepo->config); if (pCache->imem == NULL) return; - int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); - int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); - - SSkipListIterator **iters = (SSkipListIterator **)calloc(pCfg->maxTables, sizeof(SSkipListIterator *)); + // Create the iterator to read from cache + SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); if (iters == NULL) { // TODO: deal with the error return NULL; @@ -805,10 +836,15 @@ static void *tsdbCommitToFile(void *arg) { SDataCol **cols = (SDataCol **)malloc(sizeof(SDataCol *) * maxCols); void *buf = malloc((maxBytes + sizeof(SDataCol)) * pCfg->maxRowsPerFileBlock); + int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + for (int fid = sfid; fid <= efid; fid++) { TSKEY minKey = 0, maxKey = 0; tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + // tsdbOpenFileForWrite(pRepo, fid); + for (int tid = 0; tid < pCfg->maxTables; tid++) { STable *pTable = pMeta->tables[tid]; if (pTable == NULL || pTable->imem == NULL) continue; @@ -837,14 +873,10 @@ static void *tsdbCommitToFile(void *arg) { } } - // Free the iterator - for (int tid = 0; tid < pCfg->maxTables; tid++) { - if (iters[tid] != NULL) tSkipListDestroyIter(iters[tid]); - } + tsdbDestroyTableIters(iters, pCfg->maxTables); free(buf); free(cols); - free(iters); tsdbLockRepo(arg); tdListMove(pCache->imem->list, pCache->pool.memPool); From c541d35462d5884f174fdb9808265981e4d9e79d Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 24 Mar 2020 15:34:22 +0800 Subject: [PATCH 29/85] [TD-15] sdb may crash while received close signal --- src/mnode/src/mgmtSdb.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index e950761a31..0b1ac3be97 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -722,18 +722,20 @@ int32_t sdbUpdateRow(SSdbOperDesc *pOper) { void sdbCloseTable(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; void * pNode = NULL; - void * row; - + if (pTable == NULL) return; while (1) { - pNode = sdbFetchRow(handle, pNode, &row); - if (row == NULL) break; + SRowMeta * pMeta; + pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); + if (pMeta == NULL) break; SSdbOperDesc oper = { + .pObj = pMeta->row, .table = pTable, - .rowData = row, + .version = pMeta->version, }; + (*pTable->destroyFp)(&oper); } From de5fb15206036f8d4eb575e2379e022aef978c91 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 24 Mar 2020 16:13:11 +0800 Subject: [PATCH 30/85] [td-32] fix bugs and refactor codes --- src/client/inc/tsclient.h | 9 ++- src/client/src/tscServer.c | 14 ++-- src/client/src/tscSql.c | 3 +- src/dnode/src/dnodeRead.c | 30 ++++--- src/query/inc/queryExecutor.h | 12 +-- src/query/src/queryExecutor.c | 145 ++++++++++++++++------------------ src/util/src/shash.c | 2 - 7 files changed, 103 insertions(+), 112 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 2dd580dda0..fdc0ae9095 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -294,20 +294,21 @@ typedef struct SResRec { struct STSBuf; typedef struct { - int32_t code; int64_t numOfRows; // num of results in current retrieved int64_t numOfTotal; // num of total results int64_t numOfTotalInCurrentClause; // num of total result in current subclause char * pRsp; - int rspType; - int rspLen; + int32_t rspType; + int32_t rspLen; uint64_t qhandle; int64_t uid; int64_t useconds; int64_t offset; // offset value from vnode during projection query of stable - int row; + int32_t row; int16_t numOfCols; int16_t precision; + bool completed; + int32_t code; int32_t numOfGroups; SResRec * pGroupRec; char * data; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 605c7a22cc..348c0709f1 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -209,7 +209,6 @@ int tscSendMsgToServer(SSqlObj *pSql) { } void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { - tscTrace("response:%s is received, len:%d error:%s", taosMsg[rpcMsg->msgType], rpcMsg->contLen, tstrerror(rpcMsg->code)); SSqlObj *pSql = (SSqlObj *)rpcMsg->handle; if (pSql == NULL || pSql->signature != pSql) { tscError("%p sql is already released, signature:%p", pSql, pSql->signature); @@ -256,7 +255,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcFreeCont(rpcMsg->pCont); return; } else { - tscTrace("%p it shall renew meter meta, code:%d", pSql, rpcMsg->code); + tscTrace("%p it shall renew meter meta, code:%d", pSql, tstrerror(rpcMsg->code)); pSql->maxRetry = TSDB_VNODES_SUPPORT * 2; pSql->res.code = rpcMsg->code; // keep the previous error code @@ -278,7 +277,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_NETWORK_UNAVAIL; } else { - tscTrace("%p query is cancelled, code:%d", pSql, pRes->code); + tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code)); } if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { @@ -318,19 +317,17 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code, *(int32_t *)pRes->pRsp, pRes->rspLen); } else { - tscTrace("%p cmd:%d code:%d rsp len:%d", pSql, pCmd->command, pRes->code, pRes->rspLen); + tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); } } if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - + if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { int command = pCmd->command; void *taosres = tscKeepConn[command] ? pSql : NULL; - rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; - - tscTrace("%p Async SQL result:%d res:%p", pSql, rpcMsg->code, taosres); + tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), taosres); /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj @@ -2304,6 +2301,7 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { pRes->precision = htons(pRetrieve->precision); pRes->offset = htobe64(pRetrieve->offset); pRes->useconds = htobe64(pRetrieve->useconds); + pRes->completed = (pRetrieve->completed == 1); pRes->data = pRetrieve->data; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 63612d0f5f..fa50551548 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -729,7 +729,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { + if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT || + pRes->completed) { return NULL; } diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index 9e50804583..3419128c72 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -93,8 +93,8 @@ void dnodeRead(SRpcMsg *pMsg) { while (leftLen > 0) { SMsgHead *pHead = (SMsgHead *) pCont; - pHead->vgId = 1;//htonl(pHead->vgId); - pHead->contLen = pMsg->contLen; //htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + pHead->contLen = htonl(pHead->contLen); void *pVnode = dnodeGetVnode(pHead->vgId); if (pVnode == NULL) { @@ -104,12 +104,13 @@ void dnodeRead(SRpcMsg *pMsg) { } // put message into queue - SReadMsg readMsg; - readMsg.rpcMsg = *pMsg; - readMsg.pCont = pCont; - readMsg.contLen = pHead->contLen; - readMsg.pRpcContext = pRpcContext; - readMsg.pVnode = pVnode; + SReadMsg readMsg = { + .rpcMsg = *pMsg, + .pCont = pCont, + .contLen = pHead->contLen, + .pRpcContext = pRpcContext, + .pVnode = pVnode, + }; taos_queue queue = dnodeGetVnodeRworker(pVnode); taosWriteQitem(queue, &readMsg); @@ -177,8 +178,6 @@ static void *dnodeProcessReadQueue(void *param) { } else { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } - - dnodeProcessReadResult(&readMsg); } return NULL; @@ -252,17 +251,19 @@ static void dnodeProcessQueryMsg(SReadMsg *pMsg) { qTableQuery(pQInfo); } +static int32_t c = 0; static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { SRetrieveTableMsg *pRetrieve = pMsg->pCont; void *pQInfo = (void*) htobe64(pRetrieve->qhandle); dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId); - + if ((++c)%2 == 0) { + int32_t k = 1; + } int32_t rowSize = 0; int32_t numOfRows = 0; int32_t contLen = 0; - SRpcMsg rpcRsp = {0}; SRetrieveTableRsp *pRsp = NULL; int32_t code = qRetrieveQueryResultInfo(pQInfo, &numOfRows, &rowSize); @@ -276,7 +277,7 @@ static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); } - rpcRsp = (SRpcMsg) { + SRpcMsg rpcRsp = (SRpcMsg) { .handle = pMsg->rpcMsg.handle, .pCont = pRsp, .contLen = contLen, @@ -285,4 +286,7 @@ static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { }; rpcSendResponse(&rpcRsp); + + //todo merge result should be done here + //dnodeProcessReadResult(&readMsg); } diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index b10d869780..4ce606f599 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -33,7 +33,7 @@ typedef struct SData { } SData; enum { - ST_QUERY_KILLED = 0, // query killed +// ST_QUERY_KILLED = 0, // query killed ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer ST_QUERY_COMPLETED = 2, // query completed }; @@ -142,8 +142,8 @@ typedef struct SQuery { SResultRec rec; int32_t pos; int64_t pointsOffset; // the number of points offset to save read data - SData** sdata; - int32_t capacity; + SData** sdata; + int32_t capacity; SSingleColumnFilterInfo* pFilterInfo; } SQuery; @@ -170,14 +170,14 @@ typedef struct SQueryRuntimeEnv { } SQueryRuntimeEnv; typedef struct SQInfo { - uint64_t signature; + void* signature; void* pVnode; TSKEY startTime; - int64_t elapsedTime; + TSKEY elapsedTime; SResultRec rec; int32_t pointsInterpo; int32_t code; // error code to returned to client - int32_t killed; // denotes if current query is killed +// int32_t killed; // denotes if current query is killed sem_t dataReady; SArray* pTableIdList; // table list SQueryRuntimeEnv runtimeEnv; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 696d8c37c9..cbce5097ec 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -64,38 +64,24 @@ typedef struct SPointInterpoSupporter { } SPointInterpoSupporter; typedef enum { - - /* - * the program will call this function again, if this status is set. - * used to transfer from QUERY_RESBUF_FULL - */ + // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, - /* - * output buffer is full, so, the next query will be employed, - * in this case, we need to set the appropriated start scan point for - * the next query. - * - * this status is only exist in group-by clause and - * diff/add/division/multiply/ query. + /* result output buffer is full, current query is paused. + * this status is only exist in group-by clause and diff/add/division/multiply/ query. */ QUERY_RESBUF_FULL = 0x2u, - /* - * query is over - * 1. this status is used in one row result query process, e.g., - * count/sum/first/last/ - * avg...etc. - * 2. when the query range on timestamp is satisfied, it is also denoted as - * query_compeleted + /* query is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed */ QUERY_COMPLETED = 0x4u, - - /* - * all data has been scanned, so current search is stopped, - * At last, the function will transfer this status to QUERY_COMPLETED + + /* when the result is not completed return to client, this status will be + * usually used in case of interval query with interpolation option */ - QUERY_NO_DATA_TO_CHECK = 0x8u, + QUERY_OVER = 0x8u, } vnodeQueryStatus; static void setQueryStatus(SQuery *pQuery, int8_t status); @@ -1301,7 +1287,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat if (pRuntimeEnv->pTSBuf != NULL) { // if timestamp filter list is empty, quit current query if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); + setQueryStatus(pQuery, QUERY_COMPLETED); break; } } @@ -1621,10 +1607,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } -bool isQueryKilled(SQuery *pQuery) { - return false; - - SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); +static bool isQueryKilled(SQuery *pQuery) { #if 0 /* * check if the queried meter is going to be deleted. @@ -1638,9 +1621,14 @@ bool isQueryKilled(SQuery *pQuery) { return (pQInfo->killed == 1); #endif + return 0; } +static bool setQueryKilled(SQInfo* pQInfo) { + pQInfo->code = TSDB_CODE_QUERY_CANCELLED; +} + bool isFixedOutputQuery(SQuery *pQuery) { if (pQuery->intervalTime != 0) { return false; @@ -2664,7 +2652,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check if (isQueryKilled(pQuery)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return cnt; } @@ -2714,7 +2701,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } if (isIntervalQuery(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP; closeAllTimeWindow(&pRuntimeEnv->windowResInfo); @@ -3631,7 +3618,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { /* check if query is killed or not */ if (isQueryKilled(pQuery)) { - setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); +// setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } } @@ -4111,7 +4098,7 @@ bool vnodeHasRemainResults(void *handle) { } // query has completed - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { TSKEY ekey = taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); // int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY @@ -4272,7 +4259,7 @@ int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { pQuery->window.ekey, pQuery->order.order); sem_post(&pQInfo->dataReady); - pQInfo->killed = 1; + setQueryStatus(pQuery, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; } @@ -5024,7 +5011,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); // assert(pQuery->pointsRead <= pQuery->pointsToRead && - // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)); + // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED)); // must be top/bottom query if offset > 0 if (pQuery->limit.offset > 0) { @@ -5128,7 +5115,7 @@ static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= c; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { break; } @@ -5178,7 +5165,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.pointsRead, &numOfInterpo); dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.pointsRead); - if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { doRevisedResultsByLimit(pQInfo); break; } @@ -5206,17 +5193,20 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { } void qTableQuery(SQInfo *pQInfo) { - assert(pQInfo != NULL); - - if (pQInfo->killed) { + if (pQInfo == NULL || pQInfo->signature != pQInfo) { + dTrace("%p freed abort query", pQInfo); + return; + } + + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQuery *pQuery = pRuntimeEnv->pQuery; + if (isQueryKilled(pQuery)) { dTrace("QInfo:%p it is already killed, abort", pQInfo); return; } - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - // dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pQInfo); + + dTrace("QInfo:%p query task is launched", pQInfo); if (vnodeHasRemainResults(pQInfo)) { /* @@ -5242,7 +5232,7 @@ void qTableQuery(SQInfo *pQInfo) { } // here we have scan all qualified data in both data file and cache - if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (pQuery->intervalTime > 0 && pQInfo->rec.pointsTotal < pQuery->limit.limit)) { @@ -5303,10 +5293,8 @@ void qTableQuery(SQInfo *pQInfo) { /* check if query is killed or not */ if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query is killed", pQInfo); - // pQInfo->over = 1; } else { - // dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, - // pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); + dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.pointsRead); } sem_post(&pQInfo->dataReady); @@ -5989,21 +5977,16 @@ bool isQInfoValid(void *param) { return (sig == (uint64_t)pQInfo); } -void vnodeFreeQInfo(SQInfo *pQInfo, bool decQueryRef) { +void vnodeFreeQInfo(SQInfo *pQInfo) { if (!isQInfoValid(pQInfo)) { return; } - pQInfo->killed = 1; + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + setQueryKilled(pQInfo); + dTrace("QInfo:%p start to free SQInfo", pQInfo); - - if (decQueryRef) { - vnodeDecMeterRefcnt(pQInfo); - } - - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - for (int col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { tfree(pQuery->sdata[col]); } @@ -6049,7 +6032,7 @@ void vnodeFreeQInfo(SQInfo *pQInfo, bool decQueryRef) { tfree(pQuery->pGroupbyExpr); tfree(pQuery); - // dTrace("QInfo:%p vid:%d sid:%d meterId:%s, QInfo is freed", pQInfo, pObj->vnode, pObj->sid, pObj->meterId); + dTrace("QInfo:%p QInfo is freed", pQInfo); // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); @@ -6105,7 +6088,7 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE _error: // table query ref will be decrease during error handling - vnodeFreeQInfo(*pQInfo, false); + vnodeFreeQInfo(*pQInfo); return code; } @@ -6176,28 +6159,25 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *ro if (pQInfo == NULL || !isQInfoValid(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - - if (pQInfo->killed) { + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); if (pQInfo->code == TSDB_CODE_SUCCESS) { return TSDB_CODE_QUERY_CANCELLED; } else { // in case of not TSDB_CODE_SUCCESS, return the code to client - return abs(pQInfo->code); + return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } } sem_wait(&pQInfo->dataReady); - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - *numOfRows = pQInfo->rec.pointsRead; *rowsize = pQuery->rowSize; dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, *rowsize, *numOfRows, pQInfo->code); - - if (pQInfo->code < 0) { // less than 0 means there are error existed. - return -pQInfo->code; - } + + return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { @@ -6250,6 +6230,11 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int32_t *size) { pQInfo->rec.pointsTotal += pQInfo->rec.pointsRead; dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQInfo->rec.pointsRead, pQInfo->rec.pointsTotal); + + setQueryStatus(pQuery, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; + + // todo if interpolation exists, the result may be dump to client by several rounds } static void addToTaskQueue(SQInfo* pQInfo) { @@ -6261,11 +6246,7 @@ static void addToTaskQueue(SQInfo* pQInfo) { dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:%s", pQInfo, pQInfo->signature, __FUNCTION__); #endif - if (pQInfo->killed == 1) { - dTrace("%p freed or killed, abort query", pQInfo); - } else { - // todo add to task queue - } + // todo add to task queue } } @@ -6293,12 +6274,20 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c } if (pQInfo->rec.pointsRead > 0 && code == TSDB_CODE_SUCCESS) { - doDumpQueryResult(pQInfo, (*pRsp)->data, NULL); + code = doDumpQueryResult(pQInfo, (*pRsp)->data, NULL); + + // has more data to return or need next round to execute addToTaskQueue(pQInfo); - return TSDB_CODE_SUCCESS; + } else if (isQueryKilled(pQuery)) { + code = TSDB_CODE_QUERY_CANCELLED; } - assert(code != TSDB_CODE_ACTION_IN_PROGRESS); + if (isQueryKilled(pQuery) || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + (*pRsp)->completed = 1; // notify no more result to client + vnodeFreeQInfo(pQInfo); + } + + return code; // if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { // dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); diff --git a/src/util/src/shash.c b/src/util/src/shash.c index 525d00e81e..da97af84bb 100644 --- a/src/util/src/shash.c +++ b/src/util/src/shash.c @@ -162,8 +162,6 @@ void taosDeleteStrHash(void *handle, char *string) { if (pObj == NULL || pObj->maxSessions == 0) return; if (string == NULL || string[0] == 0) return; - return; - hash = (*(pObj->hashFp))(pObj, string); pthread_mutex_lock(&pObj->mutex); From b4e4e5a50b612a95211b9da4c78dcacd1bd550a5 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 24 Mar 2020 16:18:43 +0800 Subject: [PATCH 31/85] [td-32] fix bugs and refactor codes --- src/client/src/tscSql.c | 62 +++-------------------------------------- 1 file changed, 4 insertions(+), 58 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index fa50551548..4885cf7cc3 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -653,62 +653,6 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { return pRes->tsrow; } -TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) { - SSqlObj *pSql = (SSqlObj *)res; - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - return NULL; - } - - if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscTrace("%p data from all subqueries have been retrieved to client", pSql); - return tscBuildResFromSubqueries(pSql); - } else { - tscTrace("%p retrieve data from subquery failed, code:%d", pSql, pRes->code); - return NULL; - } - - } else if (pRes->row >= pRes->numOfRows) { - /** - * NOT a join query - * - * If the data block of current result set have been consumed already, try fetch next result - * data block from virtual node. - */ - tscResetForNextRetrieve(pRes); - - if (pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); // retrieve data from virtual node - - // if failed to retrieve data from current virtual node, try next one if exists - if (hasMoreVnodesToTry(pSql)) { - tscTryQueryNextVnode(pSql, NULL); - } - - /* - * local reducer has handle this case, - * so no need to add the pRes->numOfRows for super table query - */ - if (pCmd->command != TSDB_SQL_RETRIEVE_METRIC) { - pRes->numOfTotalInCurrentClause += pRes->numOfRows; - } - - if (pRes->numOfRows == 0) { - return NULL; - } - } - - return doSetResultRowData(pSql); -} - static void asyncFetchCallback(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; if (numOfRows < 0) { @@ -729,8 +673,10 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT || - pRes->completed) { + if (pRes->qhandle == 0 || + pRes->completed || + pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + pCmd->command == TSDB_SQL_INSERT) { return NULL; } From e0863b4c0b8428aaa894a0004c1cd6ae6ee633e7 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 24 Mar 2020 17:48:29 +0800 Subject: [PATCH 32/85] TD-34 --- src/vnode/tsdb/inc/tsdbFile.h | 59 +++++++-- src/vnode/tsdb/src/tsdbFile.c | 236 +++++++++++++++++----------------- src/vnode/tsdb/src/tsdbMain.c | 66 +++++----- 3 files changed, 201 insertions(+), 160 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 385d1f59ea..9a4d94c58f 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -34,15 +34,22 @@ typedef enum { TSDB_FILE_TYPE_MAX } TSDB_FILE_TYPE; +#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX) + extern const char *tsdbFileSuffix[]; typedef struct { int8_t type; + int fd; char fname[128]; int64_t size; // total size of the file int64_t tombSize; // unused file size + int32_t totalBlocks; + int32_t totalSubBlocks; } SFile; +#define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1) + typedef struct { int32_t fileId; SFile files[TSDB_FILE_TYPE_MAX]; @@ -50,14 +57,26 @@ typedef struct { // TSDB file handle typedef struct { - int32_t daysPerFile; - int32_t keep; - int32_t minRowPerFBlock; - int32_t maxRowsPerFBlock; - int32_t maxTables; + int maxFGroups; + int numOfFGroups; + SFileGroup fGroup[]; } STsdbFileH; +#define TSDB_MIN_FILE_ID(fh) (fh)->fGroup[0].fileId +#define TSDB_MAX_FILE_ID(fh) (fh)->fGroup[(fh)->numOfFGroups - 1].fileId + +STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles); +void tsdbCloseFileH(STsdbFileH *pFileH); +int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); +int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); + +typedef struct { + int32_t len; + int32_t padding; // For padding purpose + int64_t offset; +} SCompIdx; + /** * if numOfSubBlocks == -1, then the SCompBlock is a sub-block * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to @@ -78,14 +97,32 @@ typedef struct { TSKEY keyLast; } SCompBlock; -#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX) +typedef struct { + int32_t delimiter; // For recovery usage + int32_t checksum; // TODO: decide if checksum logic in this file or make it one API + int64_t uid; + int32_t padding; // For padding purpose + int32_t numOfBlocks; // TODO: make the struct padding + SCompBlock blocks[]; +} SCompInfo; -STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, - int32_t maxRowsPerFBlock, int32_t maxTables); +// TODO: take pre-calculation into account +typedef struct { + int16_t colId; // Column ID + int16_t len; // Column length + int32_t type : 8; + int32_t offset : 24; +} SCompCol; -void tsdbCloseFile(STsdbFileH *pFileH); -int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables); -void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); +// TODO: Take recover into account +typedef struct { + int32_t delimiter; // For recovery usage + int32_t numOfCols; // For recovery usage + int64_t uid; // For recovery usage + SCompCol cols[]; +} SCompData; + +void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } #endif diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 98562be0cc..f622c38b5f 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -27,72 +27,126 @@ #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F -typedef struct { - int32_t len; - int32_t padding; // For padding purpose - int64_t offset; -} SCompIdx; - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t checksum; // TODO: decide if checksum logic in this file or make it one API - int64_t uid; - int32_t padding; // For padding purpose - int32_t numOfBlocks; // TODO: make the struct padding - SCompBlock blocks[]; -} SCompInfo; - -// TODO: take pre-calculation into account -typedef struct { - int16_t colId; // Column ID - int16_t len; // Column length - int32_t type : 8; - int32_t offset : 24; -} SCompCol; - -// TODO: Take recover into account -typedef struct { - int32_t delimiter; // For recovery usage - int32_t numOfCols; // For recovery usage - int64_t uid; // For recovery usage - SCompCol cols[]; -} SCompData; - const char *tsdbFileSuffix[] = { ".head", // TSDB_FILE_TYPE_HEAD ".data", // TSDB_FILE_TYPE_DATA ".last" // TSDB_FILE_TYPE_LAST }; -static int tsdbWriteFileHead(int fd, SFile *pFile) { +static int compFGroupKey(const void *key, const void *fgroup); +static int compFGroup(const void *arg1, const void *arg2); +static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname); +static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile); +static int tsdbWriteFileHead(SFile *pFile); +static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); + +STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { + STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * maxFiles); + if (pFileH == NULL) { // TODO: deal with ERROR here + return NULL; + } + + pFileH->maxFGroups = maxFiles; + + DIR *dir = opendir(dataDir); + if (dir == NULL) { + free(pFileH); + return NULL; + } + + struct dirent *dp; + while ((dp = readdir(dir)) != NULL) { + if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 1) == 0) continue; + // TODO + } + + return pFileH; +} + +void tsdbCloseFileH(STsdbFileH *pFileH) { free(pFileH); } + +int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { + if (pFileH->numOfFGroups >= pFileH->maxFGroups) return -1; + + SFileGroup fGroup; + SFileGroup *pFGroup = &fGroup; + if (fid < TSDB_MIN_FILE_ID(pFileH) || fid > TSDB_MAX_FILE_ID(pFileH) || + bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey) == + NULL) { + pFGroup->fileId = fid; + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbCreateFile(dataDir, fid, type, maxTables, &(pFGroup->files[type])) < 0) { + // TODO: deal with the ERROR here, remove those creaed file + return -1; + } + } + + pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; + qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); + } + return 0; +} + +int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { + SFileGroup *pGroup = + bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); + if (pGroup == NULL) return -1; + + // Remove from disk + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + remove(pGroup->files[type].fname); + } + + // Adjust the memory + int filesBehind = pFileH->numOfFGroups - (((char *)pGroup - (char *)(pFileH->fGroup)) / sizeof(SFileGroup) + 1); + if (filesBehind > 0) { + memmove((void *)pGroup, (void *)((char *)pGroup + sizeof(SFileGroup)), sizeof(SFileGroup) * filesBehind); + } + pFileH->numOfFGroups--; + + return 0; +} + +static int compFGroupKey(const void *key, const void *fgroup) { + int fid = *(int *)key; + SFileGroup *pFGroup = (SFileGroup *)fgroup; + return (fid - pFGroup->fileId); +} + +static int compFGroup(const void *arg1, const void *arg2) { + return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId; +} + +static int tsdbWriteFileHead(SFile *pFile) { char head[TSDB_FILE_HEAD_SIZE] = "\0"; pFile->size += TSDB_FILE_HEAD_SIZE; // TODO: write version and File statistic to the head - lseek(fd, 0, SEEK_SET); - if (write(fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1; + lseek(pFile->fd, 0, SEEK_SET); + if (write(pFile->fd, head, TSDB_FILE_HEAD_SIZE) < 0) return -1; return 0; } -static int tsdbWriteHeadFileIdx(int fd, int maxTables, SFile *pFile) { +static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { int size = sizeof(SCompIdx) * maxTables; void *buf = calloc(1, size); if (buf == NULL) return -1; - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { free(buf); return -1; } - if (write(fd, buf, size) < 0) { + if (write(pFile->fd, buf, size) < 0) { free(buf); return -1; } pFile->size += size; + free(buf); return 0; } @@ -104,12 +158,27 @@ static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname) return 0; } -/** - * Create a file and set the SFile object - */ +static int tsdbOpenFileForWrite(SFile *pFile, int oflag) { // TODO: change the function + if (TSDB_IS_FILE_OPENED(pFile)) return -1; + + pFile->fd = open(pFile->fname, oflag, 0755); + if (pFile->fd < 0) return -1; + + return 0; +} + +static int tsdbCloseFile(SFile *pFile) { + if (!TSDB_IS_FILE_OPENED(pFile)) return -1; + int ret = close(pFile->fd); + pFile->fd = -1; + + return ret; +} + static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile) { memset((void *)pFile, 0, sizeof(SFile)); pFile->type = type; + pFile->fd = -1; tsdbGetFileName(dataDir, fileId, type, pFile->fname); if (access(pFile->fname, F_OK) == 0) { @@ -117,93 +186,28 @@ static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, return -1; } - int fd = open(pFile->fname, O_WRONLY | O_CREAT, 0755); - if (fd < 0) return -1; - - if (type == TSDB_FILE_TYPE_HEAD) { - if (tsdbWriteHeadFileIdx(fd, maxTables, pFile) < 0) { - close(fd); - return -1; - } - } - - if (tsdbWriteFileHead(fd, pFile) < 0) { - close(fd); + if (tsdbOpenFileForWrite(pFile, O_WRONLY | O_CREAT) < 0) { + // TODO: deal with the ERROR here return -1; } - close(fd); - - return 0; -} - -static int tsdbRemoveFile(SFile *pFile) { - if (pFile == NULL) return -1; - return remove(pFile->fname); -} - -// Create a file group with fileId and return a SFileGroup object -int tsdbCreateFileGroup(char *dataDir, int fileId, SFileGroup *pFGroup, int maxTables) { - if (dataDir == NULL || pFGroup == NULL) return -1; - - memset((void *)pFGroup, 0, sizeof(SFileGroup)); - - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(dataDir, fileId, type, maxTables, &(pFGroup->files[type])) < 0) { - // TODO: deal with the error here, remove the created files + if (type == TSDB_FILE_TYPE_HEAD) { + if (tsdbWriteHeadFileIdx(pFile, maxTables) < 0) { + tsdbCloseFile(pFile); return -1; } } - pFGroup->fileId = fileId; + if (tsdbWriteFileHead(pFile) < 0) { + tsdbCloseFile(pFile); + return -1; + } + + tsdbCloseFile(pFile); return 0; } -/** - * Initialize the TSDB file handle - */ -STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32_t minRowsPerFBlock, - int32_t maxRowsPerFBlock, int32_t maxTables) { - STsdbFileH *pTsdbFileH = - (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * tsdbGetMaxNumOfFiles(keep, daysPerFile)); - if (pTsdbFileH == NULL) return NULL; - - pTsdbFileH->daysPerFile = daysPerFile; - pTsdbFileH->keep = keep; - pTsdbFileH->minRowPerFBlock = minRowsPerFBlock; - pTsdbFileH->maxRowsPerFBlock = maxRowsPerFBlock; - pTsdbFileH->maxTables = maxTables; - - // Open the directory to read information of each file - DIR *dir = opendir(dataDir); - if (dir == NULL) { - free(pTsdbFileH); - return NULL; - } - - char fname[256]; - - struct dirent *dp; - while ((dp = readdir(dir)) != NULL) { - if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; - if (true /* check if the file is the .head file */) { - int fileId = 0; - int vgId = 0; - sscanf(dp->d_name, "v%df%d.head", &vgId, &fileId); - // TODO - - // Open head file - - // Open data file - - // Open last file - } - } - - return pTsdbFileH; -} - void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index afd57947b7..07ea9bd11a 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -182,7 +182,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO char dataDir[128] = "\0"; tsdbGetDataDirName(pRepo, dataDir); pRepo->tsdbFileH = - tsdbInitFile(dataDir, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->maxTables); + tsdbInitFileH(dataDir, pCfg->maxTables); if (pRepo->tsdbFileH == NULL) { free(pRepo->rootDir); tsdbFreeCache(pRepo->tsdbCache); @@ -787,7 +787,7 @@ static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { for (int tid = 0; tid < maxTables; tid++) { if (iters[tid] == NULL) continue; - tSkipListDestroy(iters[tid]); + tSkipListDestroyIter(iters[tid]); } free(iters); @@ -836,42 +836,42 @@ static void *tsdbCommitToFile(void *arg) { SDataCol **cols = (SDataCol **)malloc(sizeof(SDataCol *) * maxCols); void *buf = malloc((maxBytes + sizeof(SDataCol)) * pCfg->maxRowsPerFileBlock); - int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); - int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + // int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); + // int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); - for (int fid = sfid; fid <= efid; fid++) { - TSKEY minKey = 0, maxKey = 0; - tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + // for (int fid = sfid; fid <= efid; fid++) { + // TSKEY minKey = 0, maxKey = 0; + // tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - // tsdbOpenFileForWrite(pRepo, fid); + // // tsdbOpenFileForWrite(pRepo, fid); - for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL || pTable->imem == NULL) continue; - if (iters[tid] == NULL) { // create table iterator - iters[tid] = tSkipListCreateIter(pTable->imem->pData); - // TODO: deal with the error - if (iters[tid] == NULL) break; - if (!tSkipListIterNext(iters[tid])) { - // assert(0); - } - } + // for (int tid = 0; tid < pCfg->maxTables; tid++) { + // STable *pTable = pMeta->tables[tid]; + // if (pTable == NULL || pTable->imem == NULL) continue; + // if (iters[tid] == NULL) { // create table iterator + // iters[tid] = tSkipListCreateIter(pTable->imem->pData); + // // TODO: deal with the error + // if (iters[tid] == NULL) break; + // if (!tSkipListIterNext(iters[tid])) { + // // assert(0); + // } + // } - // Init row data part - cols[0] = (SDataCol *)buf; - for (int col = 1; col < schemaNCols(pTable->schema); col++) { - cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); - } + // // Init row data part + // cols[0] = (SDataCol *)buf; + // for (int col = 1; col < schemaNCols(pTable->schema); col++) { + // cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); + // } - // Loop the iterator - int rowsRead = 0; - while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > - 0) { - // printf("rowsRead:%d-----------\n", rowsRead); - int k = 0; - } - } - } + // // Loop the iterator + // int rowsRead = 0; + // while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > + // 0) { + // // printf("rowsRead:%d-----------\n", rowsRead); + // int k = 0; + // } + // } + // } tsdbDestroyTableIters(iters, pCfg->maxTables); From 377517eec46cc5f3d12f6f38fdf3e589ccf67640 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Tue, 24 Mar 2020 18:30:25 +0800 Subject: [PATCH 33/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 63 +++++++++++++++--------------- src/vnode/tsdb/tests/tsdbTests.cpp | 2 +- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 07ea9bd11a..af3a923d90 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -818,6 +818,7 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) // Commit to file static void *tsdbCommitToFile(void *arg) { // TODO + printf("Starting to commit....\n"); STsdbRepo * pRepo = (STsdbRepo *)arg; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCache *pCache = pRepo->tsdbCache; @@ -836,42 +837,42 @@ static void *tsdbCommitToFile(void *arg) { SDataCol **cols = (SDataCol **)malloc(sizeof(SDataCol *) * maxCols); void *buf = malloc((maxBytes + sizeof(SDataCol)) * pCfg->maxRowsPerFileBlock); - // int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); - // int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); - // for (int fid = sfid; fid <= efid; fid++) { - // TSKEY minKey = 0, maxKey = 0; - // tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + for (int fid = sfid; fid <= efid; fid++) { + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - // // tsdbOpenFileForWrite(pRepo, fid); + // tsdbOpenFileForWrite(pRepo, fid); - // for (int tid = 0; tid < pCfg->maxTables; tid++) { - // STable *pTable = pMeta->tables[tid]; - // if (pTable == NULL || pTable->imem == NULL) continue; - // if (iters[tid] == NULL) { // create table iterator - // iters[tid] = tSkipListCreateIter(pTable->imem->pData); - // // TODO: deal with the error - // if (iters[tid] == NULL) break; - // if (!tSkipListIterNext(iters[tid])) { - // // assert(0); - // } - // } + for (int tid = 0; tid < pCfg->maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL || pTable->imem == NULL) continue; + if (iters[tid] == NULL) { // create table iterator + iters[tid] = tSkipListCreateIter(pTable->imem->pData); + // TODO: deal with the error + if (iters[tid] == NULL) break; + if (!tSkipListIterNext(iters[tid])) { + // assert(0); + } + } - // // Init row data part - // cols[0] = (SDataCol *)buf; - // for (int col = 1; col < schemaNCols(pTable->schema); col++) { - // cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); - // } + // Init row data part + cols[0] = (SDataCol *)buf; + for (int col = 1; col < schemaNCols(pTable->schema); col++) { + cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); + } - // // Loop the iterator - // int rowsRead = 0; - // while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > - // 0) { - // // printf("rowsRead:%d-----------\n", rowsRead); - // int k = 0; - // } - // } - // } + // Loop the iterator + int rowsRead = 0; + while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > + 0) { + // printf("rowsRead:%d-----------\n", rowsRead); + int k = 0; + } + } + } tsdbDestroyTableIters(iters, pCfg->maxTables); diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index a76aef2d41..bc6532984f 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -142,7 +142,7 @@ TEST(TsdbTest, DISABLED_openRepo) { TEST(TsdbTest, DISABLED_createFileGroup) { SFileGroup fGroup; - ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0); + // ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0); int k = 0; } \ No newline at end of file From 2cf04d23b396ad7b762e418a6e8fef9b79d0e3e0 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 24 Mar 2020 19:30:45 +0800 Subject: [PATCH 34/85] [TD-10] fix compile error in dnode --- src/dnode/src/dnodeRead.c | 8 +++---- src/dnode/src/dnodeWrite.c | 47 +++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index a238f38671..c928f5ee91 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -164,7 +164,7 @@ static void *dnodeProcessReadQueue(void *param) { void *pVnode; while (1) { - if (taosReadQitemFromQset(qset, &type, &pReadMsg, &pVnode) == 0) { + if (taosReadQitemFromQset(qset, &type, (void **)&pReadMsg, (void **)&pVnode) == 0) { dnodeHandleIdleReadWorker(); continue; } @@ -228,11 +228,11 @@ static void dnodeProcessReadResult(void *pVnode, SReadMsg *pRead) { rpcFreeCont(pRead->rpcMsg.pCont); // free the received message } -static void dnodeProcessQueryMsg(SReadMsg *pMsg) { +static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont; SQInfo* pQInfo = NULL; - void* tsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void* tsdb = dnodeGetVnodeTsdb(pVnode); int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); @@ -254,7 +254,7 @@ static void dnodeProcessQueryMsg(SReadMsg *pMsg) { } static int32_t c = 0; -static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { +static void dnodeProcessRetrieveMsg(void *pVnode, SReadMsg *pMsg) { SRetrieveTableMsg *pRetrieve = pMsg->pCont; void *pQInfo = (void*) htobe64(pRetrieve->qhandle); diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index bb93c1cb34..7ec206f366 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -53,12 +53,12 @@ typedef struct _thread_obj { static void (*dnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(void *, SWriteMsg *); static void *dnodeProcessWriteQueue(void *param); static void dnodeHandleIdleWorker(SWriteWorker *pWorker); -static void dnodeProcessWriteResult(SWriteMsg *pWrite); -static void dnodeProcessSubmitMsg(SWriteMsg *pMsg); -static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg); -static void dnodeProcessDropTableMsg(SWriteMsg *pMsg); -static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg); -static void dnodeProcessDropStableMsg(SWriteMsg *pMsg); +static void dnodeProcessWriteResult(void *pVnode, SWriteMsg *pWrite); +static void dnodeProcessSubmitMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessCreateTableMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessDropTableMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessAlterTableMsg(void *pVnode, SWriteMsg *pMsg); +static void dnodeProcessDropStableMsg(void *pVnode, SWriteMsg *pMsg); SWriteWorkerPool wWorkerPool; @@ -193,20 +193,20 @@ static void *dnodeProcessWriteQueue(void *param) { continue; } - for (int32_t i=0; iwhandle, writeMsg.rpcMsg.msgType, writeMsg.pCont, writeMsg.contLen); } - + // flush WAL file // walFsync(pVnode->whandle); // browse all items, and process them one by one taosResetQitems(qall); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, &type, &pWriteMsg); + taosGetQitem(qall, &type, (void **)&pWriteMsg); terrno = 0; if (dnodeProcessWriteMsgFp[pWriteMsg->rpcMsg.msgType]) { @@ -218,7 +218,6 @@ static void *dnodeProcessWriteQueue(void *param) { dnodeProcessWriteResult(pVnode, pWriteMsg); taosFreeQitem(pWriteMsg); } - } taosFreeQall(qall); @@ -270,7 +269,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { } } -static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { +static void dnodeProcessSubmitMsg(void *pVnode, SWriteMsg *pMsg) { dTrace("submit msg is disposed"); SShellSubmitRspMsg *pRsp = rpcMallocCont(sizeof(SShellSubmitRspMsg)); @@ -279,7 +278,7 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { pRsp->affectedRows = htonl(1); pRsp->numOfFailedBlocks = 0; - void* tsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void* tsdb = dnodeGetVnodeTsdb(pVnode); assert(tsdb != NULL); tsdbInsertData(tsdb, pMsg->pCont); @@ -295,7 +294,7 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) { rpcSendResponse(&rpcRsp); } -static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg) { +static void dnodeProcessCreateTableMsg(void *pVnode, SWriteMsg *pMsg) { SMDCreateTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -344,16 +343,16 @@ static void dnodeProcessCreateTableMsg(SWriteMsg *pMsg) { tsdbTableSetTagValue(&tCfg, dataRow, false); } - void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void *pTsdb = dnodeGetVnodeTsdb(pVnode); rpcRsp.code = tsdbCreateTable(pTsdb, &tCfg); - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("table:%s, create table result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); } -static void dnodeProcessDropTableMsg(SWriteMsg *pMsg) { +static void dnodeProcessDropTableMsg(void *pVnode, SWriteMsg *pMsg) { SMDDropTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -363,16 +362,16 @@ static void dnodeProcessDropTableMsg(SWriteMsg *pMsg) { .tid = htonl(pTable->sid) }; - void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void *pTsdb = dnodeGetVnodeTsdb(pVnode); rpcRsp.code = tsdbDropTable(pTsdb, tableId); - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("table:%s, drop table result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); } -static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg) { +static void dnodeProcessAlterTableMsg(void *pVnode, SWriteMsg *pMsg) { SMDCreateTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -421,16 +420,16 @@ static void dnodeProcessAlterTableMsg(SWriteMsg *pMsg) { tsdbTableSetTagValue(&tCfg, dataRow, false); } - void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + void *pTsdb = dnodeGetVnodeTsdb(pVnode); rpcRsp.code = tsdbAlterTable(pTsdb, &tCfg); - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("table:%s, alter table result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); } -static void dnodeProcessDropStableMsg(SWriteMsg *pMsg) { +static void dnodeProcessDropStableMsg(void *pVnode, SWriteMsg *pMsg) { SMDDropSTableMsg *pTable = pMsg->rpcMsg.pCont; SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; @@ -442,7 +441,7 @@ static void dnodeProcessDropStableMsg(SWriteMsg *pMsg) { //rpcRsp.code = tsdbDropSTable(pTsdb, pTable->uid); rpcRsp.code = TSDB_CODE_SUCCESS; - dnodeReleaseVnode(pMsg->pVnode); + dnodeReleaseVnode(pVnode); dTrace("stable:%s, drop stable result:%s", pTable->tableId, tstrerror(rpcRsp.code)); rpcSendResponse(&rpcRsp); From c2a3c50f07bf238849c6f5f26dc6e0f2f9890e0a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 24 Mar 2020 22:59:00 +0800 Subject: [PATCH 35/85] [td-32] fix memory leaks, and fix bugs in select * query. --- src/client/src/tscParseInsert.c | 11 ---- src/client/src/tscSQLParser.c | 22 ++++--- src/client/src/tscServer.c | 10 +-- src/client/src/tscSql.c | 7 +-- src/client/src/tscStream.c | 1 - src/client/src/tscUtil.c | 22 +------ src/query/src/queryExecutor.c | 105 ++++++++++++++++++-------------- src/vnode/tsdb/src/tsdbRead.c | 9 +-- 8 files changed, 83 insertions(+), 104 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ccd734ec7b..5651e5aa38 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1238,22 +1238,11 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _clean; } - // submit to more than one vnode if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { goto _error_clean; } - - STableDataBlocks *pDataBlock = pCmd->pDataBlocks->pData[0]; - if ((code = tscCopyDataBlockToPayload(pSql, pDataBlock)) != TSDB_CODE_SUCCESS) { - goto _error_clean; - } - - pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - - // set the next sent data vnode index in data block arraylist - pTableMetaInfo->vnodeIndex = 1; } else { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e0850a7139..6427578e82 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1489,11 +1489,9 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, resColIdx, functionID, pColIndex, type, bytes, bytes); strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); - // for point interpolation/last_row query, we need the timestamp column to be loaded + // for all querie, the timestamp column meeds to be loaded SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { - tscColumnBaseInfoInsert(pQueryInfo, &index); - } + tscColumnBaseInfoInsert(pQueryInfo, &index); SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); insertResultField(pQueryInfo, resColIdx, &ids, bytes, type, columnName, pExpr); @@ -1581,7 +1579,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); } } - + + SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + return TSDB_CODE_SUCCESS; } case TK_SUM: @@ -1689,7 +1690,10 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); } } - + + SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + return TSDB_CODE_SUCCESS; } case TK_FIRST: @@ -1708,7 +1712,6 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } /* in first/last function, multiple columns can be add to resultset */ - for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) { tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { @@ -1753,7 +1756,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt } } } - + return TSDB_CODE_SUCCESS; } else { // select * from xxx int32_t numOfFields = 0; @@ -1773,6 +1776,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); } + return TSDB_CODE_SUCCESS; } } @@ -1891,6 +1895,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIdx, tSQLExprIt default: return TSDB_CODE_INVALID_SQL; } + + } // todo refactor diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 348c0709f1..9e1efcd6ff 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -341,14 +341,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { (*pSql->fp)(pSql->param, taosres, rpcMsg->code); if (shouldFree) { - // If it is failed, all objects allocated during execution taos_connect_a should be released - if (command == TSDB_SQL_CONNECT) { - taos_close(pObj); - tscTrace("%p Async sql close failed connection", pSql); - } else { - tscFreeSqlObj(pSql); - tscTrace("%p Async sql is automatically freed", pSql); - } + tscFreeSqlObj(pSql); + tscTrace("%p Async sql is automatically freed", pSql); } } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 4885cf7cc3..d62dac088b 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -594,11 +594,7 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { } if (numOfTableHasRes >= 2) { // do merge result - success = (doSetResultRowData(pSql->pSubs[0]) != NULL) && (doSetResultRowData(pSql->pSubs[1]) != NULL); - // TSKEY key1 = *(TSKEY *)pRes1->tsrow[0]; - // TSKEY key2 = *(TSKEY *)pRes2->tsrow[0]; - // printf("first:%" PRId64 ", second:%" PRId64 "\n", key1, key2); } else { // only one subquery SSqlObj *pSub = pSql->pSubs[0]; if (pSub == NULL) { @@ -674,14 +670,13 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlRes *pRes = &pSql->res; if (pRes->qhandle == 0 || - pRes->completed || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { return NULL; } // current data are exhausted, fetch more data - if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows && + if (pRes->data == NULL || (pRes->data != NULL && pRes->row >= pRes->numOfRows && pRes->completed != true && (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_METRIC || pCmd->command == TSDB_SQL_FETCH))) { taos_fetch_rows_a(res, asyncFetchCallback, pSql->pTscObj); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 46e3ac2e60..0b464c362b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -504,7 +504,6 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p } tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->emptyRspSem, 0, 1); SSqlInfo SQLInfo = {0}; tSQLParse(&SQLInfo, pSql->sqlstr); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 2a9673b192..d079494dde 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -423,9 +423,6 @@ void tscFreeResData(SSqlObj* pSql) { } void tscFreeSqlResult(SSqlObj* pSql) { - //TODO not free - return; - tfree(pSql->res.pRsp); pSql->res.row = 0; pSql->res.numOfRows = 0; @@ -469,8 +466,6 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { tscFreeSqlCmdData(pCmd); tscTrace("%p free sqlObj partial completed", pSql); - - tscFreeSqlCmdData(pCmd); } void tscFreeSqlObj(SSqlObj* pSql) { @@ -489,10 +484,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pCmd->allocSize = 0; - if (pSql->fp == NULL) { - tsem_destroy(&pSql->rspSem); - tsem_destroy(&pSql->emptyRspSem); - } + tsem_destroy(&pSql->rspSem); free(pSql); } @@ -1751,16 +1743,8 @@ bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { } int32_t command = pSql->cmd.command; - if (pTscObj->pSql == pSql) { - /* - * in case of taos_connect_a query, the object should all be released, even it is the - * master sql object. Otherwise, the master sql should not be released - */ - if (command == TSDB_SQL_CONNECT && pSql->res.code != TSDB_CODE_SUCCESS) { - return true; - } - - return false; + if (command == TSDB_SQL_CONNECT) { + return true; } if (command == TSDB_SQL_INSERT) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index cbce5097ec..f233ac0d99 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -53,9 +53,9 @@ /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].info.bytes) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.bytes) #define GET_COLUMN_TYPE(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdxInBuf].info.type) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIdx].info.type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -1498,16 +1498,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel SColIndexEx * pColIndexEx = &pSqlFuncMsg->colInfo; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - - if (TSDB_COL_IS_TAG(pSqlFuncMsg->colInfo.flag)) { // process tag column info - SSchema *pSchema = getColumnModelSchema(pTagsSchema, pColIndexEx->colIdx); - - pCtx->inputType = pSchema->type; - pCtx->inputBytes = pSchema->bytes; - } else { - pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); - pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); - } + pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); + pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); pCtx->ptsOutputBuf = NULL; @@ -1891,8 +1883,6 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { pQuery->checkBufferInLoop = hasMultioutput ? 1 : 0; } - - // pQuery->pointsOffset = pQuery->pointsToRead; } /* @@ -2552,7 +2542,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl // return DISK_DATA_LOAD_FAILED; } - if (pStatis == NULL) { + if (*pStatis == NULL) { pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); } } else { @@ -5025,11 +5015,8 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { } static void tableMultiOutputProcessor(SQInfo *pQInfo) { -#if 0 - SQuery * pQuery = &pQInfo->query; - SMeterObj *pMeterObj = pQInfo->pObj; - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pTableQuerySupporter->runtimeEnv; + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { @@ -5044,8 +5031,8 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { return; } - pQuery->pointsRead = getNumOfResult(pRuntimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->pointsRead > 0) { + pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.pointsRead > 0) { doSkipResults(pRuntimeEnv); } @@ -5053,40 +5040,31 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { * 1. if pQuery->pointsRead == 0, pQuery->limit.offset >= 0, still need to check data * 2. if pQuery->pointsRead > 0, pQuery->limit.offset must be 0 */ - if (pQuery->pointsRead > 0 || Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED | QUERY_NO_DATA_TO_CHECK)) { + if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { break; } - TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); - assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); - dTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->limit.offset, pQuery->lastKey, - pQuery->ekey); + pQInfo, pQuery->limit.offset, pQuery->lastKey); resetCtxOutputBuf(pRuntimeEnv); } doRevisedResultsByLimit(pQInfo); - pQInfo->pointsRead += pQuery->pointsRead; + pQInfo->rec.pointsRead += pQuery->rec.pointsRead; - if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)) { - TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); - assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); - - dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, - pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->lastKey, pQuery->ekey); + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { +// dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, +// pQInfo, pQuery->lastKey, pQuery->ekey); } - dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, - pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); +// dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, +// pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); - pQuery->pointsOffset = pQuery->pointsToRead; // restore the available buffer - if (!isTSCompQuery(pQuery)) { - assert(pQuery->pointsRead <= pQuery->pointsToRead); - } - -#endif +// pQuery->pointsOffset = pQuery->pointsToRead; //restore the available buffer +// if (!isTSCompQuery(pQuery)) { +// assert(pQuery->pointsRead <= pQuery->pointsToRead); +// } } static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { @@ -5127,10 +5105,8 @@ static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { } } -/* handle time interval query on single table */ +// handle time interval query on table static void tableIntervalProcessor(SQInfo *pQInfo) { - // STable *pMeterObj = pQInfo->pObj; - SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); SQuery * pQuery = pRuntimeEnv->pQuery; @@ -5839,6 +5815,39 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_SUCCESS; } +static void doUpdateExprColumnIndex(SQuery* pQuery) { + assert(pQuery->pSelectExpr != NULL && pQuery != NULL); +// int32_t i = 0, j = 0; +// while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { +// if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { +// pQuery->colList[i++].colIdx = (int16_t)j++; +// } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { +// pQuery->colList[i++].colIdx = -1; +// } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { +// j++; +// } +// } + +// while (i < pQuery->numOfCols) { +// pQuery->colList[i++].colIdx = -1; // not such column in current meter +// } + + for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; + if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { + continue; + } + + SColIndexEx* pColIndexEx = &pSqlExprMsg->colInfo; + for(int32_t f = 0; f < pQuery->numOfCols; ++f) { + if (pColIndexEx->colId == pQuery->colList[f].info.colId) { + pColIndexEx->colIdx = f; + break; + } + } + } +} + static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, SArray *pTableIdList) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); @@ -5897,6 +5906,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou assert(pExprs[col].resBytes > 0); pQuery->rowSize += pExprs[col].resBytes; } + + doUpdateExprColumnIndex(pQuery); int32_t ret = vnodeCreateFilterInfo(pQInfo, pQuery); if (ret != TSDB_CODE_SUCCESS) { @@ -5933,7 +5944,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // to make sure third party won't overwrite this structure - pQInfo->signature = (uint64_t)pQInfo; + pQInfo->signature = pQInfo; pQInfo->pTableIdList = pTableIdList; pQuery->pos = -1; diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 31cdd70f36..36472857fe 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -335,7 +335,6 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; - // Convert row data to column data if (*skey == INT64_MIN) { *skey = dataRowKey(row); @@ -345,13 +344,13 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max int32_t offset = 0; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, 0); + SColumnInfoEx* pColInfo = taosArrayGet(pHandle->pColumns, i); memcpy(pColInfo->pData + numOfRows*pColInfo->info.bytes, dataRowTuple(row) + offset, pColInfo->info.bytes); offset += pColInfo->info.bytes; } numOfRows++; - if (numOfRows > maxRowsToRead) break; + if (numOfRows >= maxRowsToRead) break; }; return numOfRows; @@ -392,7 +391,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SData } SArray *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle, SArray *pIdList) { - + // in case of data in cache, all data has been kept in column info object. + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + return pHandle->pColumns; } int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, STimeWindow *window, tsdbpos_t position, int16_t order) {} From 737d4c1072e222dc55a0e281feec2704713c2780 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 24 Mar 2020 23:01:06 +0800 Subject: [PATCH 36/85] [td-32] suppress warnings --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d079494dde..e33fdc70a4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -713,7 +713,7 @@ static void trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); - SSubmitBlk* pBlock = pTableDataBlock->pData; + SSubmitBlk* pBlock = (SSubmitBlk*) pTableDataBlock->pData; int32_t rows = htons(pBlock->numOfRows); for(int32_t i = 0; i < rows; ++i) { From db3bea8f38bf1877aa16b6f3887f3991a7567e25 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 24 Mar 2020 23:13:02 +0800 Subject: [PATCH 37/85] [TD-10] fix memery corrupt after tqueue changed --- src/dnode/src/dnodeMgmt.c | 16 ++++++++-------- src/dnode/src/dnodeRead.c | 14 +++++++++----- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index f6976aa2cc..4d335d7353 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -225,17 +225,17 @@ static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir) { vnodeObj.status = TSDB_VN_STATUS_NOT_READY; vnodeObj.refCount = 1; vnodeObj.version = 0; - vnodeObj.wworker = dnodeAllocateWriteWorker(&vnodeObj); - vnodeObj.rworker = dnodeAllocateReadWorker(&vnodeObj); vnodeObj.wal = NULL; vnodeObj.tsdb = pTsdb; vnodeObj.replica = NULL; vnodeObj.events = NULL; vnodeObj.cq = NULL; - taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj)); + SVnodeObj *pVnode = (SVnodeObj *)taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *)(&vnodeObj)); + pVnode->wworker = dnodeAllocateWriteWorker(pVnode); + pVnode->rworker = dnodeAllocateReadWorker(pVnode); - dTrace("open vnode:%d in %s", vnodeObj.vgId, rootDir); + dTrace("open vnode:%d in %s", pVnode->vgId, rootDir); return TSDB_CODE_SUCCESS; } @@ -314,17 +314,17 @@ static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) { vnodeObj.status = TSDB_VN_STATUS_NOT_READY; vnodeObj.refCount = 1; vnodeObj.version = 0; - vnodeObj.wworker = dnodeAllocateWriteWorker(&vnodeObj); - vnodeObj.rworker = dnodeAllocateReadWorker(&vnodeObj); vnodeObj.wal = NULL; vnodeObj.tsdb = pTsdb; vnodeObj.replica = NULL; vnodeObj.events = NULL; vnodeObj.cq = NULL; - taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *) (&vnodeObj)); + SVnodeObj *pVnode = (SVnodeObj *)taosAddIntHash(tsDnodeVnodesHash, vnodeObj.vgId, (char *)(&vnodeObj)); + pVnode->wworker = dnodeAllocateWriteWorker(pVnode); + pVnode->rworker = dnodeAllocateReadWorker(pVnode); - dPrint("vgroup:%d, vnode:%d is created", vnodeObj.vgId, vnodeObj.vgId); + dPrint("vgroup:%d, vnode:%d is created", pVnode->vgId, pVnode->vgId); return TSDB_CODE_SUCCESS; } diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index c928f5ee91..d6ecf2fa66 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -220,11 +220,15 @@ static void dnodeProcessReadResult(void *pVnode, SReadMsg *pRead) { code = terrno; } - SRpcMsg rsp; - rsp.handle = pRead->rpcMsg.handle; - rsp.code = code; - rsp.pCont = NULL; - rpcSendResponse(&rsp); + //TODO: query handle is returned by dnodeProcessQueryMsg + if (0) { + SRpcMsg rsp; + rsp.handle = pRead->rpcMsg.handle; + rsp.code = code; + rsp.pCont = NULL; + rpcSendResponse(&rsp); + } + rpcFreeCont(pRead->rpcMsg.pCont); // free the received message } From 1a52be002fdbcd089df27a5d13c65551d326a9a0 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 24 Mar 2020 23:54:22 +0800 Subject: [PATCH 38/85] =?UTF-8?q?[TD-9]=20update=20show=20tables=E2=80=99?= =?UTF-8?q?=20tableId=20len?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/inc/taosdef.h | 2 +- src/mnode/inc/mgmtSuperTable.h | 3 ++- src/mnode/src/mgmtSuperTable.c | 5 +++++ src/mnode/src/mgmtTable.c | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 570e2e1acd..d6a9447e3d 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -176,7 +176,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_COLUMNS 256 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns -#define TSDB_DNODE_NAME_LEN 63 +#define TSDB_DNODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 192 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 diff --git a/src/mnode/inc/mgmtSuperTable.h b/src/mnode/inc/mgmtSuperTable.h index 922aafed7f..6d0c565c30 100644 --- a/src/mnode/inc/mgmtSuperTable.h +++ b/src/mnode/inc/mgmtSuperTable.h @@ -45,7 +45,8 @@ void * mgmtGetSuperTableVgroup(SSuperTableObj *pStable); int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pTable, const char *tagName); int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable); -void mgmtDropAllSuperTables(SDbObj *pDropDb); +void mgmtDropAllSuperTables(SDbObj *pDropDb); +int32_t mgmtExtractTableName(const char* tableId, char* name); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index f2b54ec000..7c98556e99 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -624,3 +624,8 @@ int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg return TSDB_CODE_SUCCESS; } +int32_t mgmtExtractTableName(const char* tableId, char* name) { + char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2); + return copy(name, r, TS_PATH_DELIMITER[0]); +} + diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 675de57924..491e65979a 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -310,7 +310,7 @@ int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void * numOfRead++; // pattern compare for meter name - extractTableName(tableId, tableName); + mgmtExtractTableName(tableId, tableName); if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { @@ -333,7 +333,7 @@ int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; if (superTableId != NULL) { - extractTableName(superTableId, pWrite); + mgmtExtractTableName(superTableId, pWrite); } cols++; From 1a048a102881b7bae22b46630449ed4d0a590ce1 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 25 Mar 2020 00:04:58 +0800 Subject: [PATCH 39/85] [TD-9] extract table name --- src/mnode/src/mgmtSuperTable.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index 7c98556e99..e8161deb4d 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -524,7 +524,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v } memset(stableName, 0, tListLen(stableName)); - extractTableName(pTable->tableId, stableName); + mgmtExtractTableName(pTable->tableId, stableName); if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) @@ -625,7 +625,16 @@ int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg } int32_t mgmtExtractTableName(const char* tableId, char* name) { - char* r = skipSegments(tableId, TS_PATH_DELIMITER[0], 2); - return copy(name, r, TS_PATH_DELIMITER[0]); + int pos = -1; + int num = 0; + for (pos = 0; tableId[pos] != 0; ++pos) { + if (tableId[pos] == '.') num++; + if (num == 2) break; + } + + if (num == 2) { + strcpy(name, tableId + pos + 1); + } + return 0; } From e1fdc40fe216344ce245e5b9e25f4ff72da92717 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 25 Mar 2020 11:01:07 +0800 Subject: [PATCH 40/85] [TD-15] fix error while create child table --- src/mnode/src/mgmtChildTable.c | 5 ++--- src/mnode/src/mgmtNormalTable.c | 2 +- src/mnode/src/mgmtTable.c | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 9076681caa..8fbf210118 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -319,9 +319,8 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t desc.type = SDB_OPER_TYPE_GLOBAL; desc.pObj = pTable; desc.table = tsChildTableSdb; - sdbInsertRow(&desc); - - if (sdbInsertRow(&desc) < 0) { + + if (sdbInsertRow(&desc) != TSDB_CODE_SUCCESS) { free(pTable); mError("ctable:%s, update sdb error", pCreate->tableId); terrno = TSDB_CODE_SDB_ERROR; diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c index 4e5e4ce7cc..fde9db2d74 100644 --- a/src/mnode/src/mgmtNormalTable.c +++ b/src/mnode/src/mgmtNormalTable.c @@ -334,7 +334,7 @@ void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t desc.type = SDB_OPER_TYPE_GLOBAL; desc.pObj = pTable; desc.table = tsNormalTableSdb; - if (sdbInsertRow(&desc) < 0) { + if (sdbInsertRow(&desc) != TSDB_CODE_SUCCESS) { mError("table:%s, update sdb error", pTable->tableId); free(pTable); terrno = TSDB_CODE_SDB_ERROR; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 491e65979a..c9a0333b17 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -287,6 +287,7 @@ int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void * createdTime = pNormalTable->createdTime; numOfColumns = pNormalTable->numOfColumns; } else { + pShow->pNode = NULL; void *pChildTableNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable); if (pTable != NULL) { SChildTableObj *pChildTable = (SChildTableObj *) pTable; From f743478363c8ff636481f8ff14c4bc266006bf1b Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 25 Mar 2020 14:19:00 +0800 Subject: [PATCH 41/85] [td-32]use code to denote if query is killed or not --- src/query/src/queryExecutor.c | 43 +++++++++++------------- src/vnode/detail/src/vnodeQueryImpl.c | 4 +-- src/vnode/detail/src/vnodeQueryProcess.c | 24 ++++++------- 3 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index f233ac0d99..a996c8467f 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -1599,7 +1599,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } -static bool isQueryKilled(SQuery *pQuery) { +static bool isQueryKilled(SQInfo *pQInfo) { + return (pQInfo->code == TSDB_CODE_QUERY_CANCELLED); #if 0 /* * check if the queried meter is going to be deleted. @@ -1613,8 +1614,6 @@ static bool isQueryKilled(SQuery *pQuery) { return (pQInfo->killed == 1); #endif - - return 0; } static bool setQueryKilled(SQInfo* pQInfo) { @@ -2641,7 +2640,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (tsdbNextDataBlock(pQueryHandle)) { // check if query is killed or not set the status of query to pass the status check - if (isQueryKilled(pQuery)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return cnt; } @@ -3607,7 +3606,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->scanFlag = REPEAT_SCAN; /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { // setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -4394,14 +4393,10 @@ static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - // SMeterObj * pTempMeter = getMeterObj(pSupporter->pMetersHashTable, pSupporter->pMeterSidExtInfo[0]->sid); - // __block_search_fn_t searchFn = vnodeSearchKeyFunc[pTempMeter->searchAlgorithm]; - // dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); - tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { break; } @@ -4583,7 +4578,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->meterIdx = start; for (int32_t k = start; k <= end; ++k, pSupporter->meterIdx++) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -4610,7 +4605,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->subgroupIdx); for (int32_t k = start; k <= end; ++k) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -4661,7 +4656,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { while (pSupporter->meterIdx < pSupporter->numOfMeters) { int32_t k = pSupporter->meterIdx; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -4955,7 +4950,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { doMultiMeterSupplementaryScan(pQInfo); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query killed, abort", pQInfo); return; } @@ -4994,7 +4989,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { vnodeScanAllData(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -5027,7 +5022,7 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { vnodeScanAllData(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -5074,7 +5069,7 @@ static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { initCtxOutputBuf(pRuntimeEnv); vnodeScanAllData(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; } @@ -5177,7 +5172,7 @@ void qTableQuery(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p it is already killed, abort", pQInfo); return; } @@ -5267,7 +5262,7 @@ void qTableQuery(SQInfo *pQInfo) { pQInfo->elapsedTime += (taosGetTimestampUs() - st); /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); } else { dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.pointsRead); @@ -5309,7 +5304,7 @@ void qSuperTableQuery(void *pReadMsg) { /* record the total elapsed time */ pQInfo->elapsedTime += (taosGetTimestampUs() - st); - pQuery->status = isQueryKilled(pQuery) ? 1 : 0; + pQuery->status = isQueryKilled(pQInfo) ? 1 : 0; // taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->pointsRead, // pQInfo->query.interpoType); @@ -6172,7 +6167,7 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *ro } SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); if (pQInfo->code == TSDB_CODE_SUCCESS) { return TSDB_CODE_QUERY_CANCELLED; @@ -6289,11 +6284,11 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c // has more data to return or need next round to execute addToTaskQueue(pQInfo); - } else if (isQueryKilled(pQuery)) { - code = TSDB_CODE_QUERY_CANCELLED; + } else { + code = pQInfo->code; } - if (isQueryKilled(pQuery) || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { (*pRsp)->completed = 1; // notify no more result to client vnodeFreeQInfo(pQInfo); } diff --git a/src/vnode/detail/src/vnodeQueryImpl.c b/src/vnode/detail/src/vnodeQueryImpl.c index 9eb3fb8b65..f3e5cc27b3 100644 --- a/src/vnode/detail/src/vnodeQueryImpl.c +++ b/src/vnode/detail/src/vnodeQueryImpl.c @@ -5312,7 +5312,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { // check if query is killed or not set the status of query to pass the status check - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return cnt; } @@ -6375,7 +6375,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->scanFlag = REPEAT_SCAN; /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } diff --git a/src/vnode/detail/src/vnodeQueryProcess.c b/src/vnode/detail/src/vnodeQueryProcess.c index cedb76b4ac..23520f35a1 100644 --- a/src/vnode/detail/src/vnodeQueryProcess.c +++ b/src/vnode/detail/src/vnodeQueryProcess.c @@ -105,7 +105,7 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo int32_t start = pSupporter->pSidSet->starterPos[groupIdx]; int32_t end = pSupporter->pSidSet->starterPos[groupIdx + 1] - 1; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -276,7 +276,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo int64_t st = taosGetTimestampUs(); while (1) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { break; } @@ -363,7 +363,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo int32_t j = QUERY_IS_ASC_QUERY(pQuery) ? 0 : numOfBlocks - 1; for (; j < numOfBlocks && j >= 0; j += step) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { break; } @@ -603,7 +603,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->meterIdx = start; for (int32_t k = start; k <= end; ++k, pSupporter->meterIdx++) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -630,7 +630,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->subgroupIdx); for (int32_t k = start; k <= end; ++k) { - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -681,7 +681,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { while (pSupporter->meterIdx < pSupporter->numOfMeters) { int32_t k = pSupporter->meterIdx; - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); return; } @@ -958,7 +958,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { doMultiMeterSupplementaryScan(pQInfo); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query killed, abort", pQInfo); return; } @@ -998,7 +998,7 @@ static void vnodeSingleTableFixedOutputProcessor(SQInfo *pQInfo) { vnodeScanAllData(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -1033,7 +1033,7 @@ static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) { vnodeScanAllData(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -1087,7 +1087,7 @@ static void vnodeSingleMeterIntervalMainLooper(STableQuerySupportObj *pSupporter initCtxOutputBuf(pRuntimeEnv); vnodeScanAllData(pRuntimeEnv); - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { return; } @@ -1301,7 +1301,7 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) { pQInfo->useconds += (taosGetTimestampUs() - st); /* check if query is killed or not */ - if (isQueryKilled(pQuery)) { + if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); pQInfo->over = 1; } else { @@ -1345,7 +1345,7 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { /* record the total elapsed time */ pQInfo->useconds += (taosGetTimestampUs() - st); - pQInfo->over = isQueryKilled(pQuery) ? 1 : 0; + pQInfo->over = isQueryKilled(pQInfo) ? 1 : 0; taosInterpoSetStartInfo(&pQInfo->pTableQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead, pQInfo->query.interpoType); From 7e5b12f7d52108ce4c25a1cbf1112c349987b8d8 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 25 Mar 2020 18:00:18 +0800 Subject: [PATCH 42/85] TD-34 --- src/common/inc/dataformat.h | 20 +++++++++++--- src/common/src/dataformat.c | 51 +++++++++++++++++++++++++++++++---- src/vnode/tsdb/src/tsdbMain.c | 28 ++++++++++++++++--- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 1f46c68abc..207ff4dbfd 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -105,11 +105,25 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int64_t len; - char data[]; + int32_t len; + void * pData; } SDataCol; -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter); +typedef struct { + TSKEY firstKey; + TSKEY lastKey; + int numOfPoints; + int numOfCols; + void * buf; + SDataCol cols[]; +} SDataCols; + +#define keyCol(cols) (&((cols)->cols[0])) // Key column + +SDataCols *tdNewDataCols(STSchema *pSchema, int nRows); +void tdFreeDataCols(SDataCols *pCols); +void tdResetDataCols(SDataCols *pCols); +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 04826e43ac..3c692f9eba 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -294,14 +294,55 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter) { - int row = *iter; +SDataCols *tdNewDataCols(STSchema *pSchema, int nRows) { + int nCols = schemaNCols(pSchema); - for (int i = 0; i < schemaNCols(pSchema); i++) { - // TODO + SDataCols *pInfo = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * nCols); + if (pInfo == NULL) return NULL; + + pInfo->numOfCols = nCols; + pInfo->firstKey = INT64_MIN; + pInfo->lastKey = INT64_MAX; + pInfo->buf = malloc(tdMaxRowBytesFromSchema(pSchema) * nRows); + if (pInfo->buf == NULL) { + free(pInfo); + return NULL; } - *iter = row + 1; + pInfo->cols[0].pData = pInfo->buf; + for (int i = 1; i < nCols; i++) { + pInfo->cols[i].pData = (char *)(pInfo->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * nRows; + } + + return pInfo; +} + +void tdFreeDataCols(SDataCols *pCols) { + if (pCols) { + if (pCols->buf) free(pCols->buf); + free(pCols); + } +} + +void tdResetDataCols(SDataCols *pCols) { + pCols->firstKey = INT64_MAX; + pCols->lastKey = INT64_MIN; + pCols->numOfPoints = 0; + for (int i = 0; i < pCols->numOfCols; i++) { + pCols->cols[i].len = 0; + } +} + +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema) { + TSKEY key = dataRowKey(row); + if (pCols->numOfPoints == 0) pCols->firstKey = key; + pCols->lastKey = key; + for (int i = 0; i < pCols->numOfCols; i++) { + SDataCol *pCol = pCols->cols + i; + memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, colOffset(schemaColAt(pSchema, i))), + colBytes(schemaColAt(pSchema, i))); + pCol->len += colBytes(schemaColAt(pSchema, i)); + } } /** diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index af3a923d90..7b1836378d 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -84,7 +84,7 @@ static int tsdbOpenMetaFile(char *tsdbDir); static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); -static void * tsdbCommitToFile(void *arg); +static void * tsdbCommitData(void *arg); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -327,7 +327,7 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { pRepo->tsdbCache->curBlock = NULL; // TODO: here should set as detached or use join for memory leak - pthread_create(&(pRepo->commitThread), NULL, tsdbCommitToFile, (void *)repo); + pthread_create(&(pRepo->commitThread), NULL, tsdbCommitData, (void *)repo); tsdbUnLockRepo(repo); return 0; @@ -816,7 +816,7 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) } // Commit to file -static void *tsdbCommitToFile(void *arg) { +static void *tsdbCommitData(void *arg) { // TODO printf("Starting to commit....\n"); STsdbRepo * pRepo = (STsdbRepo *)arg; @@ -894,4 +894,26 @@ static void *tsdbCommitToFile(void *arg) { tsdbUnLockRepo(arg); return NULL; +} + +static int tsdbCommitToFile(STsdbRepo *pRepo, SSkipListIterator **iters, int fid) { + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &pRepo->config; + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + + for (int tid = 0; tid < pCfg->maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + SSkipListIterator *pIter = iters[tid]; + + if (pIter == NULL) continue; + + // Read data + // while () { + + // } + } + + return 0; } \ No newline at end of file From 002e414caff2334ca60688aefe33b2c963ec3729 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 25 Mar 2020 22:49:54 +0800 Subject: [PATCH 43/85] TD-34 --- src/common/inc/dataformat.h | 20 +++++++++++------- src/common/src/dataformat.c | 41 ++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 207ff4dbfd..37a3ccea5a 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -105,24 +105,30 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int32_t len; - void * pData; + int8_t type; + int bytes; + int len; + void * pData; } SDataCol; typedef struct { - TSKEY firstKey; - TSKEY lastKey; + int maxRowSize; + int maxCols; // max number of columns + int maxPoints; // max number of points int numOfPoints; - int numOfCols; + int numOfCols; // Total number of cols void * buf; SDataCol cols[]; } SDataCols; #define keyCol(cols) (&((cols)->cols[0])) // Key column +#define dataColsKeyFirst(cols) ((int64_t *)(keyCol(cols)->pData))[0] +#define dataColsKeyLast(cols) ((int64_t *)(keyCol(cols)->pData))[(cols)->numOfPoints - 1] -SDataCols *tdNewDataCols(STSchema *pSchema, int nRows); -void tdFreeDataCols(SDataCols *pCols); +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema); #ifdef __cplusplus diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 3c692f9eba..84ac1efd71 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -294,27 +294,34 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } -SDataCols *tdNewDataCols(STSchema *pSchema, int nRows) { - int nCols = schemaNCols(pSchema); +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { + SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols); + if (pCols == NULL) return NULL; - SDataCols *pInfo = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * nCols); - if (pInfo == NULL) return NULL; + pCols->maxRowSize = maxRowSize; + pCols->maxCols = maxCols; + pCols->maxPoints = maxRows; - pInfo->numOfCols = nCols; - pInfo->firstKey = INT64_MIN; - pInfo->lastKey = INT64_MAX; - pInfo->buf = malloc(tdMaxRowBytesFromSchema(pSchema) * nRows); - if (pInfo->buf == NULL) { - free(pInfo); + pCols->buf = malloc(maxRowSize * maxRows); + if (pCols->buf == NULL) { + free(pCols); return NULL; } - pInfo->cols[0].pData = pInfo->buf; - for (int i = 1; i < nCols; i++) { - pInfo->cols[i].pData = (char *)(pInfo->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * nRows; + return pCols; +} + +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { + // assert(schemaNCols(pSchema) <= pCols->numOfCols); + tdResetDataCols(pCols); + pCols->numOfCols = schemaNCols(pSchema); + + pCols->cols[0].pData = pCols->buf; + for (int i = 1; i < schemaNCols(pSchema); i++) { + pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; } - return pInfo; + return pCols; } void tdFreeDataCols(SDataCols *pCols) { @@ -325,18 +332,14 @@ void tdFreeDataCols(SDataCols *pCols) { } void tdResetDataCols(SDataCols *pCols) { - pCols->firstKey = INT64_MAX; - pCols->lastKey = INT64_MIN; pCols->numOfPoints = 0; - for (int i = 0; i < pCols->numOfCols; i++) { + for (int i = 0; i < pCols->maxCols; i++) { pCols->cols[i].len = 0; } } void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema) { TSKEY key = dataRowKey(row); - if (pCols->numOfPoints == 0) pCols->firstKey = key; - pCols->lastKey = key; for (int i = 0; i < pCols->numOfCols; i++) { SDataCol *pCol = pCols->cols + i; memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, colOffset(schemaColAt(pSchema, i))), From 81f27c37a66f28e4b505cd79575cac9fb45789b6 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 25 Mar 2020 23:23:37 +0800 Subject: [PATCH 44/85] TD-34 --- src/common/inc/dataformat.h | 3 +- src/common/src/dataformat.c | 17 ++++++--- src/vnode/tsdb/src/tsdbMain.c | 69 ++++++++++------------------------- 3 files changed, 32 insertions(+), 57 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 37a3ccea5a..e123efd11e 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -108,6 +108,7 @@ typedef struct SDataCol { int8_t type; int bytes; int len; + int offset; void * pData; } SDataCol; @@ -129,7 +130,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); void tdFreeDataCols(SDataCols *pCols); -void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema); +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 84ac1efd71..da55663d0b 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -317,8 +317,13 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { pCols->numOfCols = schemaNCols(pSchema); pCols->cols[0].pData = pCols->buf; - for (int i = 1; i < schemaNCols(pSchema); i++) { - pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; + for (int i = 0; i < schemaNCols(pSchema); i++) { + if (i > 0) { + pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; + } + pCols->cols[i].type = colType(schemaColAt(pSchema, i)); + pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i)); + pCols->cols[i].offset = colOffset(schemaColAt(pSchema, i)); } return pCols; @@ -338,14 +343,14 @@ void tdResetDataCols(SDataCols *pCols) { } } -void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema) { +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) { TSKEY key = dataRowKey(row); for (int i = 0; i < pCols->numOfCols; i++) { SDataCol *pCol = pCols->cols + i; - memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, colOffset(schemaColAt(pSchema, i))), - colBytes(schemaColAt(pSchema, i))); - pCol->len += colBytes(schemaColAt(pSchema, i)); + memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, pCol->offset), pCol->bytes); + pCol->len += pCol->bytes; } + pCols->numOfPoints++; } /** diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 7b1836378d..a8e04e216a 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -85,6 +85,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -761,7 +762,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } -static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCol **cols, STSchema *pSchema) { +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { int numOfRows = 0; do { SSkipListNode *node = tSkipListIterGet(pIter); @@ -769,12 +770,8 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; - // Convert row data to column data - // for (int i = 0; i < schemaNCols(pSchema); i++) { - // STColumn *pCol = schemaColAt(pSchema, i); - // memcpy(cols[i]->data + TYPE_BYTES[colType(pCol)] * numOfRows, dataRowAt(row, pCol->offset), - // TYPE_BYTES[colType(pCol)]); - // } + + tdAppendDataRowToDataCol(row, pCols); numOfRows++; if (numOfRows > maxRowsToRead) break; @@ -823,7 +820,7 @@ static void *tsdbCommitData(void *arg) { STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCache *pCache = pRepo->tsdbCache; STsdbCfg * pCfg = &(pRepo->config); - if (pCache->imem == NULL) return; + if (pCache->imem == NULL) return NULL; // Create the iterator to read from cache SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); @@ -832,52 +829,23 @@ static void *tsdbCommitData(void *arg) { return NULL; } - int maxCols = pMeta->maxCols; - int maxBytes = pMeta->maxRowBytes; - SDataCol **cols = (SDataCol **)malloc(sizeof(SDataCol *) * maxCols); - void *buf = malloc((maxBytes + sizeof(SDataCol)) * pCfg->maxRowsPerFileBlock); + // Create a data column buffer for commit + SDataCols *pCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock); + if (pCols == NULL) { + // TODO: deal with the error + return NULL; + } int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); for (int fid = sfid; fid <= efid; fid++) { - TSKEY minKey = 0, maxKey = 0; - tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - - // tsdbOpenFileForWrite(pRepo, fid); - - for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL || pTable->imem == NULL) continue; - if (iters[tid] == NULL) { // create table iterator - iters[tid] = tSkipListCreateIter(pTable->imem->pData); - // TODO: deal with the error - if (iters[tid] == NULL) break; - if (!tSkipListIterNext(iters[tid])) { - // assert(0); - } - } - - // Init row data part - cols[0] = (SDataCol *)buf; - for (int col = 1; col < schemaNCols(pTable->schema); col++) { - cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); - } - - // Loop the iterator - int rowsRead = 0; - while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > - 0) { - // printf("rowsRead:%d-----------\n", rowsRead); - int k = 0; - } - } + tsdbCommitToFile(pRepo, fid, iters, pCols); } + tdFreeDataCols(pCols); tsdbDestroyTableIters(iters, pCfg->maxTables); - free(buf); - free(cols); tsdbLockRepo(arg); tdListMove(pCache->imem->list, pCache->pool.memPool); @@ -896,7 +864,7 @@ static void *tsdbCommitData(void *arg) { return NULL; } -static int tsdbCommitToFile(STsdbRepo *pRepo, SSkipListIterator **iters, int fid) { +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; @@ -908,11 +876,12 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, SSkipListIterator **iters, int fid SSkipListIterator *pIter = iters[tid]; if (pIter == NULL) continue; + tdInitDataCols(pCols, pTable->schema); - // Read data - // while () { - - // } + while (tsdbReadRowsFromCache(pIter, maxKey, pCfg->maxRowsPerFileBlock, pCols)) { + // TODO + int k = 0; + } } return 0; From 2180f6eb0f2232f4c9c4608303f1c36e0f60dea4 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 26 Mar 2020 00:20:40 +0800 Subject: [PATCH 45/85] [TD-9] merge normal table with child table --- src/inc/mnode.h | 80 +-- src/inc/taosmsg.h | 4 +- src/mnode/inc/mgmtChildTable.h | 17 +- src/mnode/inc/mgmtNormalTable.h | 46 -- src/mnode/inc/mgmtSuperTable.h | 22 +- src/mnode/inc/mgmtTable.h | 9 +- src/mnode/inc/mgmtUser.h | 2 +- src/mnode/inc/mgmtVgroup.h | 4 +- src/mnode/src/mgmtChildTable.c | 908 ++++++++++++++++++++++++++++---- src/mnode/src/mgmtNormalTable.c | 551 ------------------- src/mnode/src/mgmtShell.c | 10 +- src/mnode/src/mgmtSuperTable.c | 124 +++-- src/mnode/src/mgmtTable.c | 840 +++-------------------------- src/mnode/src/mgmtUser.c | 3 +- src/mnode/src/mgmtVgroup.c | 4 +- 15 files changed, 1025 insertions(+), 1599 deletions(-) delete mode 100644 src/mnode/inc/mgmtNormalTable.h delete mode 100644 src/mnode/src/mgmtNormalTable.c diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 792a8ec1cc..80a5e2b36c 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -98,67 +98,45 @@ typedef struct { } SVnodeGid; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; + char tableId[TSDB_TABLE_ID_LEN]; + int8_t type; + int8_t dirty; } STableInfo; struct _vg_obj; typedef struct SSuperTableObj { - char tableId[TSDB_TABLE_ID_LEN + 1]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; - int32_t sversion; - int32_t numOfColumns; - int32_t numOfTags; - int8_t reserved[15]; - int8_t updateEnd[1]; - int32_t numOfTables; - int16_t nextColId; - SSchema *schema; + STableInfo info; + uint64_t uid; + int64_t createdTime; + int32_t sversion; + int32_t numOfColumns; + int32_t numOfTags; + int8_t reserved[15]; + int8_t updateEnd[1]; + int32_t numOfTables; + int16_t nextColId; + SSchema * schema; } SSuperTableObj; typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; - char superTableId[TSDB_TABLE_ID_LEN + 1]; - int8_t reserved[1]; - int8_t updateEnd[1]; + STableInfo info; + uint64_t uid; + int64_t createdTime; + int32_t sversion; //used by normal table + int32_t numOfColumns; //used by normal table + int32_t sid; + int32_t vgId; + char superTableId[TSDB_TABLE_ID_LEN + 1]; + int32_t sqlLen; + int8_t reserved[1]; + int8_t updateEnd[1]; + int16_t nextColId; //used by normal table + char* sql; //used by normal table + SSchema* schema; //used by normal table SSuperTableObj *superTable; } SChildTableObj; -typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - int8_t type; - int8_t dirty; - uint64_t uid; - int32_t sid; - int32_t vgId; - int64_t createdTime; - int32_t sversion; - int32_t numOfColumns; - int32_t sqlLen; - int8_t reserved[7]; - int8_t updateEnd[1]; - char* sql; //null-terminated string - int16_t nextColId; - SSchema* schema; -} SNormalTableObj; - struct _db_obj; typedef struct _vg_obj { @@ -260,7 +238,7 @@ typedef struct { typedef struct { uint8_t msgType; - int8_t expected; + int8_t usePublicIp; int8_t received; int8_t successed; int32_t contLen; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 2fcc91a7bb..f1e07c98a8 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -94,8 +94,8 @@ extern "C" { #define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 #define TSDB_MSG_TYPE_CM_TABLE_META 63 #define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 -#define TSDB_MSG_TYPE_CM_STABLE_META 65 -#define TSDB_MSG_TYPE_CM_STABLE_META_RSP 66 +#define TSDB_MSG_TYPE_CM_STABLE_VGROUP 65 +#define TSDB_MSG_TYPE_CM_STABLE_VGROUP_RSP 66 #define TSDB_MSG_TYPE_CM_TABLES_META 67 #define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 #define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 diff --git a/src/mnode/inc/mgmtChildTable.h b/src/mnode/inc/mgmtChildTable.h index 9252a7d485..3f6b1d8b85 100644 --- a/src/mnode/inc/mgmtChildTable.h +++ b/src/mnode/inc/mgmtChildTable.h @@ -23,23 +23,18 @@ extern "C" { #include #include #include "taosdef.h" - #include "mnode.h" int32_t mgmtInitChildTables(); void mgmtCleanUpChildTables(); void * mgmtGetChildTable(char *tableId); -void *mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid); -void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pCreate, SChildTableObj *pTable); - -int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable); -int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName, char *nContent); - -int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); - -void mgmtDropAllChildTables(SDbObj *pDropDb); -void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable); +void mgmtCreateChildTable(SQueuedMsg *pMsg); +void mgmtDropChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable); +void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SChildTableObj *pTable); +void mgmtAlterChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable); +void mgmtDropAllChildTables(SDbObj *pDropDb); +void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtNormalTable.h b/src/mnode/inc/mgmtNormalTable.h deleted file mode 100644 index dd09a62bb4..0000000000 --- a/src/mnode/inc/mgmtNormalTable.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TBASE_MNODE_NORMAL_TABLE_H -#define TBASE_MNODE_NORMAL_TABLE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "mnode.h" - -int32_t mgmtInitNormalTables(); -void mgmtCleanUpNormalTables(); -void * mgmtGetNormalTable(char *tableId); - -void * mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid); -void * mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable); - -int32_t mgmtDropNormalTable(SQueuedMsg *newMsg, SNormalTableObj *pTable); -int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols); -int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName); - -int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); - -void mgmtDropAllNormalTables(SDbObj *pDropDb); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/mnode/inc/mgmtSuperTable.h b/src/mnode/inc/mgmtSuperTable.h index 6d0c565c30..73809148de 100644 --- a/src/mnode/inc/mgmtSuperTable.h +++ b/src/mnode/inc/mgmtSuperTable.h @@ -22,31 +22,19 @@ extern "C" { #include #include - #include "taosdef.h" #include "mnode.h" int32_t mgmtInitSuperTables(); void mgmtCleanUpSuperTables(); - void * mgmtGetSuperTable(char *tableId); -int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate); -int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pTable); -int32_t mgmtAddSuperTableTag(SSuperTableObj *pTable, SSchema schema[], int32_t ntags); -int32_t mgmtDropSuperTableTag(SSuperTableObj *pTable, char *tagName); -int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pTable, char *oldTagName, char *newTagName); -int32_t mgmtAddSuperTableColumn(SSuperTableObj *pTable, SSchema schema[], int32_t ncols); -int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pTable, char *colName); - -int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp); -void * mgmtGetSuperTableVgroup(SSuperTableObj *pStable); - -int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pTable, const char *tagName); -int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable); - +void mgmtCreateSuperTable(SQueuedMsg *pMsg); +void mgmtDropSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable); +void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable); +void mgmtAlterSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable); void mgmtDropAllSuperTables(SDbObj *pDropDb); -int32_t mgmtExtractTableName(const char* tableId, char* name); +int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtTable.h b/src/mnode/inc/mgmtTable.h index b145210e9b..66557b3362 100644 --- a/src/mnode/inc/mgmtTable.h +++ b/src/mnode/inc/mgmtTable.h @@ -27,13 +27,8 @@ extern "C" { int32_t mgmtInitTables(); void mgmtCleanUpTables(); -STableInfo* mgmtGetTable(char *tableId); - -STableInfo* mgmtGetTableByPos(uint32_t dnodeIp, int32_t vnode, int32_t sid); -int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp); - -void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable); -void mgmtRemoveTableFromSuperTable(SSuperTableObj *pStable); +STableInfo* mgmtGetTable(char* tableId); +void mgmtExtractTableName(char* tableId, char* tableName); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtUser.h b/src/mnode/inc/mgmtUser.h index d1f927f6de..5001bc2770 100644 --- a/src/mnode/inc/mgmtUser.h +++ b/src/mnode/inc/mgmtUser.h @@ -24,7 +24,7 @@ extern "C" { int32_t mgmtInitUsers(); void mgmtCleanUpUsers(); SUserObj *mgmtGetUser(char *name); -SUserObj *mgmtGetUserFromConn(void *pConn); +SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index 3379a93f28..aba95fb478 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -36,8 +36,8 @@ void mgmtUpdateVgroup(SVgObj *pVgroup); void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); -void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable); -void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable); +void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); +void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle); void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 8fbf210118..9b4be2fdc2 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -35,10 +35,19 @@ #include "mgmtTable.h" #include "mgmtVgroup.h" -void *tsChildTableSdb; -int32_t tsChildTableUpdateSize; +static void *tsChildTableSdb; +static int32_t tsChildTableUpdateSize; +static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg); +static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg); +static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); +static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg); +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); +static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtDestroyChildTable(SChildTableObj *pTable) { + tfree(pTable->schema); + tfree(pTable->sql); tfree(pTable); } @@ -52,28 +61,31 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("ctable:%s, not in vgroup:%d", pTable->tableId, pTable->vgId); + mError("ctable:%s, not in vgroup:%d", pTable->info.tableId, pTable->vgId); return TSDB_CODE_INVALID_VGROUP_ID; } SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("ctable:%s, vgroup:%d not in db:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); + mError("ctable:%s, vgroup:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); + mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - pTable->superTable = mgmtGetSuperTable(pTable->superTableId); - mgmtAddTableIntoSuperTable(pTable->superTable); - - mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); + if (pTable->info.type == TSDB_CHILD_TABLE) { + pTable->superTable = mgmtGetSuperTable(pTable->superTableId); + pStable->numOfTables++; + mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); + } else { + mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); + } mgmtAddTableIntoDb(pDb); - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); + mgmtAddTableIntoVgroup(pVgroup, pTable); return TSDB_CODE_SUCCESS; } @@ -91,21 +103,25 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { SDbObj *pDb = mgmtGetDb(pVgroup->dbName); if (pDb == NULL) { - mError("ctable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); + mError("ctable:%s, vgroup:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); return TSDB_CODE_INVALID_DB; } SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); if (pAcct == NULL) { - mError("ctable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); + mError("ctable:%s, account:%s not exists", pTable->info.tableId, pDb->cfg.acct); return TSDB_CODE_INVALID_ACCT; } - mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); + if (pTable->info.type == TSDB_CHILD_TABLE) { + mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); + pStable->numOfTables--; + } else { + mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); + } mgmtRemoveTableFromDb(pDb); mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - mgmtRemoveTableFromSuperTable(pTable->superTable); - + return TSDB_CODE_SUCCESS; } @@ -117,22 +133,50 @@ static int32_t mgmtChildTableActionEncode(SSdbOperDesc *pOper) { SChildTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); - memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); - pOper->rowSize = tsChildTableUpdateSize; + if (pTable->info.type == TSDB_CHILD_TABLE) { + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); + pOper->rowSize = tsChildTableUpdateSize; + } else { + int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); + if (pOper->maxRowSize < tsChildTableUpdateSize + schemaSize) { + return TSDB_CODE_INVALID_MSG_LEN; + } + memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); + memcpy(pOper->rowData + tsChildTableUpdateSize, pTable->schema, schemaSize); + memcpy(pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); + pOper->rowSize = tsChildTableUpdateSize + schemaSize + pTable->sqlLen; + } return TSDB_CODE_SUCCESS; } static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { assert(pOper->rowData != NULL); - - pOper->pObj = calloc(1, sizeof(SChildTableObj)); - if (pOper->pObj == NULL) { + SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); + if (pTable == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } - memcpy(pOper->pObj, pOper->rowData, tsChildTableUpdateSize); + memcpy(pTable, pOper->rowData, tsChildTableUpdateSize); + if (pTable->info.type != TSDB_CHILD_TABLE) { + int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); + pTable->schema = (SSchema *)malloc(schemaSize); + if (pTable->schema == NULL) { + mgmtDestroyNormalTable(pTable); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + memcpy(pTable->schema, pOper->rowData + tsChildTableUpdateSize, schemaSize); + + pTable->sql = (char *)malloc(pTable->sqlLen); + if (pTable->sql == NULL) { + mgmtDestroyNormalTable(pTable); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + memcpy(pTable->sql, pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sqlLen); + } + + pOper->pObj = pTable; return TSDB_CODE_SUCCESS; } @@ -147,7 +191,7 @@ int32_t mgmtInitChildTables() { SSdbTableDesc tableDesc = { .tableName = "ctables", .hashSessions = tsMaxTables, - .maxRowSize = tsChildTableUpdateSize, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,, .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, @@ -165,14 +209,15 @@ int32_t mgmtInitChildTables() { pNode = NULL; while (1) { + pLastNode = pNode; pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); if (pTable == NULL) { break; } - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); if (pDb == NULL) { - mError("ctable:%s, failed to get db, discard it", pTable->tableId); + mError("ctable:%s, failed to get db, discard it", pTable->info.tableId); SSdbOperDesc desc = {0}; desc.type = SDB_OPER_TYPE_LOCAL; desc.pObj = pTable; @@ -184,7 +229,7 @@ int32_t mgmtInitChildTables() { SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); + mError("ctable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; SSdbOperDesc desc = {0}; desc.type = SDB_OPER_TYPE_LOCAL; @@ -197,7 +242,7 @@ int32_t mgmtInitChildTables() { if (strcmp(pVgroup->dbName, pDb->name) != 0) { mError("ctable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", - pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); + pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; SSdbOperDesc desc = {0}; desc.type = SDB_OPER_TYPE_LOCAL; @@ -209,7 +254,7 @@ int32_t mgmtInitChildTables() { } if (pVgroup->tableList == NULL) { - mError("ctable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); + mError("ctable:%s, vgroup:%d tableList is null", pTable->info.tableId, pTable->vgId); pTable->vgId = 0; SSdbOperDesc desc = {0}; desc.type = SDB_OPER_TYPE_LOCAL; @@ -220,20 +265,30 @@ int32_t mgmtInitChildTables() { continue; } - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); - if (pSuperTable == NULL) { - mError("ctable:%s, stable:%s not exist", pTable->tableId, pTable->superTableId); - pTable->vgId = 0; - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; - desc.pObj = pTable; - desc.table = tsChildTableSdb; - sdbDeleteRow(&desc); - pNode = pLastNode; - continue; + if (pTable->info.type == TSDB_CHILD_TABLE) { + SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTable->superTableId); + if (pSuperTable == NULL) { + mError("ctable:%s, stable:%s not exist", pTable->info.tableId, pTable->superTableId); + pTable->vgId = 0; + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_LOCAL; + desc.pObj = pTable; + desc.table = tsChildTableSdb; + sdbDeleteRow(&desc); + pNode = pLastNode; + continue; + } } } + mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLES_META, mgmtProcessMultiTableMetaMsg); + mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mgmtGetShowTableMeta); + mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mgmtRetrieveShowTables); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateTableRsp); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); + mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg); + mTrace("child table is initialized"); return 0; } @@ -242,79 +297,141 @@ void mgmtCleanUpChildTables() { sdbCloseTable(tsChildTableSdb); } -void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { +static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { char * pTagData = NULL; int32_t tagDataLen = 0; - if (pMsg != NULL) { + int32_t totalCols = 0; + int32_t contLen = 0; + if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) { pTagData = pMsg->schema + TSDB_TABLE_ID_LEN + 1; tagDataLen = htonl(pMsg->contLen) - sizeof(SCMCreateTableMsg) - TSDB_TABLE_ID_LEN - 1; + totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; + contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen + pTable->sqlLen; + } else { + totalCols = pTable->numOfColumns; + contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen; } - int32_t totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; - int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + tagDataLen; - SMDCreateTableMsg *pCreate = rpcMallocCont(contLen); if (pCreate == NULL) { terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; return NULL; } - memcpy(pCreate->tableId, pTable->tableId, TSDB_TABLE_ID_LEN + 1); + memcpy(pCreate->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN + 1); memcpy(pCreate->superTableId, pTable->superTable->tableId, TSDB_TABLE_ID_LEN + 1); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); - pCreate->tableType = pTable->type; - pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); - pCreate->numOfTags = htons(pTable->superTable->numOfTags); - pCreate->sid = htonl(pTable->sid); - pCreate->sversion = htonl(pTable->superTable->sversion); - pCreate->tagDataLen = htonl(tagDataLen); - pCreate->sqlDataLen = 0; - pCreate->uid = htobe64(pTable->uid); - pCreate->superTableUid = htobe64(pTable->superTable->uid); + pCreate->tableType = pTable->info.type; pCreate->createdTime = htobe64(pTable->createdTime); - + pCreate->sid = htonl(pTable->sid); + pCreate->sqlDataLen = htonl(pTable->sqlLen); + pCreate->uid = htobe64(pTable->uid); + + if (pTable->info.type == TSDB_CHILD_TABLE) { + pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); + pCreate->numOfTags = htons(pTable->superTable->numOfTags); + pCreate->sversion = htonl(pTable->superTable->sversion); + pCreate->tagDataLen = htonl(tagDataLen); + pCreate->superTableUid = htobe64(pTable->superTable->uid); + } else { + pCreate->numOfColumns = htons(pTable->numOfColumns); + pCreate->numOfTags = 0; + pCreate->sversion = htonl(pTable->sversion); + pCreate->tagDataLen = 0; + pCreate->superTableUid = 0; + } + SSchema *pSchema = (SSchema *) pCreate->data; - memcpy(pSchema, pTable->superTable->schema, totalCols * sizeof(SSchema)); + if (pTable->info.type == TSDB_CHILD_TABLE) { + memcpy(pSchema, pTable->superTable->schema, totalCols * sizeof(SSchema)); + } else { + memcpy(pSchema, pTable->schema, totalCols * sizeof(SSchema)); + } for (int32_t col = 0; col < totalCols; ++col) { pSchema->bytes = htons(pSchema->bytes); pSchema->colId = htons(pSchema->colId); pSchema++; } - if (pMsg != NULL) { + if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) { memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData, tagDataLen); + memcpy(pCreate->data + totalCols * sizeof(SSchema) + tagDataLen, pTable->sql, pTable->sqlLen); } return pCreate; } -void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - char *pTagData = (char *) pCreate->schema; // it is a tag key - SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); - if (pSuperTable == NULL) { - mError("ctable:%s, corresponding super table does not exist", pCreate->tableId); - terrno = TSDB_CODE_INVALID_TABLE; - return NULL; - } - - SChildTableObj *pTable = (SChildTableObj *) calloc(sizeof(SChildTableObj), 1); +static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { + SChildTableObj *pTable = (SChildTableObj *) calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { mError("ctable:%s, failed to alloc memory", pCreate->tableId); terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; return NULL; } - strcpy(pTable->tableId, pCreate->tableId); - strcpy(pTable->superTableId, pSuperTable->tableId); - pTable->type = TSDB_CHILD_TABLE; + if (pCreate->numOfColumns == 0) { + pTable->info.type = TSDB_CHILD_TABLE; + } else { + pTable->info.type = TSDB_NORMAL_TABLE; + } + + strcpy(pTable->info.tableId, pCreate->tableId); pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + - (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sid = tid; pTable->vgId = pVgroup->vgId; - pTable->superTable = pSuperTable; + + if (Table->info.type == TSDB_CHILD_TABLE) { + char *pTagData = (char *) pCreate->schema; // it is a tag key + SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); + if (pSuperTable == NULL) { + mError("ctable:%s, corresponding super table does not exist", pCreate->tableId); + free(pTable); + terrno = TSDB_CODE_INVALID_TABLE; + return NULL; + } + strcpy(pTable->superTableId, pSuperTable->tableId); + pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->superTable = pSuperTable; + } else { + pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->sversion = 0; + pTable->numOfColumns = htons(pCreate->numOfColumns); + pTable->sqlLen = htons(pCreate->sqlLen); + + int32_t numOfCols = pTable->numOfColumns; + int32_t schemaSize = numOfCols * sizeof(SSchema); + pTable->schema = (SSchema *) calloc(1, schemaSize); + if (pTable->schema == NULL) { + free(pTable); + terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; + return NULL; + } + memcpy(pTable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); + + pTable->nextColId = 0; + for (int32_t col = 0; col < numOfCols; col++) { + SSchema *tschema = pTable->schema; + tschema[col].colId = pTable->nextColId++; + tschema[col].bytes = htons(tschema[col].bytes); + } + + if (pTable->sqlLen != 0) { + pTable->info.type = TSDB_STREAM_TABLE; + pTable->sql = calloc(1, pTable->sqlLen); + if (pTable->sql == NULL) { + free(pTable); + terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; + return NULL; + } + memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); + pTable->sql[pTable->sqlLen - 1] = 0; + mTrace("table:%s, stream sql len:%d sql:%s", pTable->info.tableId, pTable->sqlLen, pTable->sql); + } + } + SSdbOperDesc desc = {0}; desc.type = SDB_OPER_TYPE_GLOBAL; desc.pObj = pTable; @@ -327,24 +444,81 @@ void* mgmtCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t return NULL; } - mTrace("ctable:%s, create ctable in vgroup, uid:%" PRIu64 , pTable->tableId, pTable->uid); + mTrace("ctable:%s, create ctable in vgroup, uid:%" PRIu64 , pTable->info.tableId, pTable->uid); return pTable; } -int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable) { +void mgmtCreateChildTable(SQueuedMsg *pMsg) { + SCMCreateTableMsg *pCreate = pMsg->pCont; + + int32_t code = mgmtCheckTimeSeries(htons(pCreate->numOfColumns)); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, failed to create, timeseries exceed the limit", pCreate->tableId); + mgmtSendSimpleResp(pMsg->thandle, code); + return; + } + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + SVgObj *pVgroup = mgmtGetAvailableVgroup(pMsg->pDb); + if (pVgroup == NULL) { + mTrace("table:%s, start to create a new vgroup", pCreate->tableId); + mgmtCreateVgroup(newMsg); + return; + } + + int32_t sid = taosAllocateId(pVgroup->idPool); + if (sid < 0) { + mTrace("tables:%s, no enough sid in vgroup:%d", pVgroup->vgId); + mgmtCreateVgroup(newMsg); + return; + } + + SChildTableObj *pTable = mgmtDoCreateChildTable(pCreate, pVgroup, sid); + if (pTable == NULL) { + mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); + return; + } + + SMDCreateTableMsg *pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + mgmtSendSimpleResp(pMsg->thandle, terrno); + mgmtFreeQueuedMsg(newMsg); + return; + } + + SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); + SRpcMsg rpcMsg = { + .handle = newMsg, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + + newMsg->ahandle = pTable; + mgmtSendMsgToDnode(&ipSet, &rpcMsg); +} + +static int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable) { SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("ctable:%s, failed to drop child table, vgroup not exist", pTable->tableId); - return TSDB_CODE_OTHERS; + mError("ctable:%s, failed to drop child table, vgroup not exist", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); + return; } SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); if (pDrop == NULL) { - mError("ctable:%s, failed to drop child table, no enough memory", pTable->tableId); - return TSDB_CODE_SERV_OUT_OF_MEMORY; + mError("ctable:%s, failed to drop child table, no enough memory", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; } - strcpy(pDrop->tableId, pTable->tableId); + strcpy(pDrop->tableId, pTable->info.tableId); pDrop->vgId = htonl(pTable->vgId); pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); pDrop->sid = htonl(pTable->sid); @@ -363,8 +537,6 @@ int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable) { newMsg->ahandle = pTable; mgmtSendMsgToDnode(&ipSet, &rpcMsg); - - return TSDB_CODE_SUCCESS; } void* mgmtGetChildTable(char *tableId) { @@ -379,7 +551,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName // } // // //TODO send msg to dnode -// mTrace("Succeed to modify tag column %d of table %s", col, pTable->tableId); +// mTrace("Succeed to modify tag column %d of table %s", col, pTable->info.tableId); // return TSDB_CODE_SUCCESS; // int32_t rowSize = 0; @@ -411,42 +583,220 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName // if (pTable->isDirty) pTable->isDirty = 0; // // if (ret < 0) { -// mError("Failed to modify tag column %d of table %s", col, pTable->tableId); +// mError("Failed to modify tag column %d of table %s", col, pTable->info.tableId); // return TSDB_CODE_APP_ERROR; // } // -// mTrace("Succeed to modify tag column %d of table %s", col, pTable->tableId); +// mTrace("Succeed to modify tag column %d of table %s", col, pTable->info.tableId); // return TSDB_CODE_SUCCESS; return 0; } -int32_t mgmtGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { - pMeta->uid = htobe64(pTable->uid); - pMeta->sid = htonl(pTable->sid); - pMeta->vgId = htonl(pTable->vgId); - pMeta->sversion = htons(pTable->superTable->sversion); - pMeta->precision = pDb->cfg.precision; - pMeta->numOfTags = pTable->superTable->numOfTags; - pMeta->numOfColumns = htons(pTable->superTable->numOfColumns); - pMeta->tableType = pTable->type; - pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); - strncpy(pMeta->tableId, pTable->tableId, tListLen(pTable->tableId)); +static int32_t mgmtFindNormalTableColumnIndex(SNormalTableObj *pTable, char *colName) { + SSchema *schema = (SSchema *) pTable->schema; + for (int32_t i = 0; i < pTable->numOfColumns; i++) { + if (strcasecmp(schema[i].name, colName) == 0) { + return i; + } + } + + return -1; +} + +static int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols) { + if (ncols <= 0) { + return TSDB_CODE_APP_ERROR; + } + + for (int32_t i = 0; i < ncols; i++) { + if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { + return TSDB_CODE_APP_ERROR; + } + } + + SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); + if (pDb == NULL) { + mError("table: %s not belongs to any database", pTable->info.tableId); + return TSDB_CODE_APP_ERROR; + } + + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct == NULL) { + mError("DB: %s not belongs to andy account", pDb->name); + return TSDB_CODE_APP_ERROR; + } + + int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); + pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols); + + memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols); + + SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns); + for (int32_t i = 0; i < ncols; i++) { + tschema[i].colId = pTable->nextColId++; + } + + pTable->numOfColumns += ncols; + pTable->sversion++; + pAcct->acctInfo.numOfTimeSeries += ncols; + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsChildTableUpdateSize; + sdbUpdateRow(&desc); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) { + int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); + if (col < 0) { + return TSDB_CODE_APP_ERROR; + } + + SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); + if (pDb == NULL) { + mError("table: %s not belongs to any database", pTable->info.tableId); + return TSDB_CODE_APP_ERROR; + } + + SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + if (pAcct == NULL) { + mError("DB: %s not belongs to any account", pDb->name); + return TSDB_CODE_APP_ERROR; + } + + memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1), + sizeof(SSchema) * (pTable->numOfColumns - col - 1)); + + pTable->numOfColumns--; + pTable->sversion++; + + pAcct->acctInfo.numOfTimeSeries--; + + SSdbOperDesc desc = {0}; + desc.type = SDB_OPER_TYPE_GLOBAL; + desc.pObj = pTable; + desc.table = tsNormalTableSdb; + desc.rowData = pTable; + desc.rowSize = tsChildTableUpdateSize; + sdbUpdateRow(&desc); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) { + int32_t numOfCols = pTable->numOfColumns; + for (int32_t i = 0; i < numOfCols; ++i) { + strcpy(pSchema->name, pTable->schema[i].name); + pSchema->type = pTable->schema[i].type; + pSchema->bytes = htons(pTable->schema[i].bytes); + pSchema->colId = htons(pTable->schema[i].colId); + pSchema++; + } + + return numOfCols * sizeof(SSchema); +} + +static void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SDbObj *pDb, SChildTableObj *pTable) { + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); + if (pMeta == NULL) { + mError("table:%s, failed to get table meta, no enough memory", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + + pMeta->uid = htobe64(pTable->uid); + pMeta->sid = htonl(pTable->sid); + pMeta->vgId = htonl(pTable->vgId); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = pTable->info.type; + strncpy(pMeta->tableId, pTable->info.tableId, tListLen(pTable->info.tableId)); + + if (pTable->info.type == TSDB_CHILD_TABLE) { + pMeta->sversion = htons(pTable->superTable->sversion); + pMeta->numOfTags = htons(pTable->superTable->numOfTags); + pMeta->numOfColumns = htons(pTable->superTable->numOfColumns); + pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); + } else { + pMeta->sversion = htons(pTable->sversion); + pMeta->numOfTags = 0; + pMeta->numOfColumns = htons(pTable->numOfColumns); + pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); + } + SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { - return TSDB_CODE_INVALID_TABLE; + mError("table:%s, failed to get table meta, db not selected", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID); + return; } + for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - if (usePublicIp) { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + if (pMsg->usePublicIp) { + pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; } else { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; + pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; } pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); } pMeta->numOfVpeers = pVgroup->numOfVnodes; - return TSDB_CODE_SUCCESS; + SRpcMsg rpcRsp = { + .handle = pMsg->thandle, + .pCont = pMeta, + .contLen = pMeta->contLen, + }; + pMeta->contLen = htons(pMeta->contLen); + rpcSendResponse(&rpcRsp); + + mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid); +} + +void mgmtProcessChildTableMetaMsg(SQueuedMsg *pMsg) { + SCMTableInfoMsg *pInfo = pMsg->pCont; + SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); + if (pDb == NULL || pDb->dirty) { + mError("table:%s, failed to get table meta, db not selected", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); + return; + } + + STableInfo *pTable = mgmtGetTable(pInfo->tableId); + if (pTable == NULL) { + if (htons(pInfo->createFlag) != 1) { + mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; + } else { + //TODO: on demand create table from super table if table does not exists + int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData); + SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); + if (pCreateMsg == NULL) { + mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData)); + strcpy(pCreateMsg->tableId, pInfo->tableId); + + SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); + memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); + pMsg->pCont = NULL; + + newMsg->ahandle = newMsg->pCont; + newMsg->pCont = pCreateMsg; + mTrace("table:%s, start to create in demand", pInfo->tableId); + mgmtAddToShellQueue(newMsg); + return; + } + } + + mgmtGetChildTableMeta(pMsg, pDb, pTable); } void mgmtDropAllChildTables(SDbObj *pDropDb) { @@ -462,7 +812,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { break; } - if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { + if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOperDesc oper = { .type = SDB_OPER_TYPE_LOCAL, .table = tsChildTableSdb, @@ -504,4 +854,360 @@ void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { } mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->tableId, numOfTables); +} + +static STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { + SDnodeObj *pObj = mgmtGetDnode(dnodeId); + SVgObj *pVgroup = mgmtGetVgroup(vnode); + + if (pObj == NULL || pVgroup == NULL) { + return NULL; + } + + return pVgroup->tableList[sid]; +} + +static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { + if (mgmtCheckRedirect(rpcMsg->handle)) return; + + SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; + pCfg->dnode = htonl(pCfg->dnode); + pCfg->vnode = htonl(pCfg->vnode); + pCfg->sid = htonl(pCfg->sid); + mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + + STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); + if (pTable == NULL) { + mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE); + return; + } + + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); + + SMDCreateTableMsg *pMDCreate = NULL; + pMDCreate = mgmtBuildCreateChildTableMsg(NULL, (SChildTableObj *) pTable); + if (pMDCreate == NULL) { + return; + } + + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SRpcMsg rpcRsp = { + .handle = NULL, + .pCont = pMDCreate, + .contLen = htonl(pMDCreate->contLen), + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE + }; + mgmtSendMsgToDnode(&ipSet, &rpcRsp); +} + +static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { + if (rpcMsg->handle == NULL) return; + + SQueuedMsg *queueMsg = rpcMsg->handle; + queueMsg->received++; + + STableInfo *pTable = queueMsg->ahandle; + mTrace("table:%s, drop table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, tstrerror(rpcMsg->code)); + + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + mError("table:%s, failed to drop in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code)); + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + free(queueMsg); + return; + } + + SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); + if (pVgroup == NULL) { + mError("table:%s, failed to get vgroup", pTable->info.tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID); + free(queueMsg); + return; + } + + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + int32_t code = sdbDeleteRow(&oper); + if (code != TSDB_CODE_SUCCESS) { + mError("table:%s, update ctables sdb error", pTable->info.tableId); + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); + free(queueMsg); + return; + } + + if (pVgroup->numOfTables <= 0) { + mPrint("vgroup:%d, all tables is dropped, drop vgroup", pVgroup->vgId); + mgmtDropVgroup(pVgroup, NULL); + } + + mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SUCCESS); + free(queueMsg); +} + +static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { + mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); +} + +static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { + if (rpcMsg->handle == NULL) return; + + SQueuedMsg *queueMsg = rpcMsg->handle; + queueMsg->received++; + + STableInfo *pTable = queueMsg->ahandle; + mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, + rpcMsg->handle, tstrerror(rpcMsg->code)); + + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + SSdbOperDesc oper = { + .type = SDB_OPER_TYPE_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable + }; + sdbDeleteRow(&oper); + + mError("table:%s, failed to create in dnode, reason:%s", pTable->info.tableId, tstrerror(rpcMsg->code)); + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + } else { + mTrace("table:%s, created in dnode", pTable->info.tableId); + if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) { + SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); + newMsg->msgType = queueMsg->msgType; + newMsg->thandle = queueMsg->thandle; + newMsg->pDb = queueMsg->pDb; + newMsg->pUser = queueMsg->pUser; + newMsg->contLen = queueMsg->contLen; + newMsg->pCont = rpcMallocCont(newMsg->contLen); + memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); + mTrace("table:%s, start to get meta", pTable->info.tableId); + mgmtAddToShellQueue(newMsg); + } else { + mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); + } + } + + free(queueMsg); +} + +static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg) { + mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); +} + +static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { + SRpcConnInfo connInfo; + if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) { + mError("conn:%p is already released while get mulit table meta", pMsg->thandle); + return; + } + + bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); + SUserObj *pUser = mgmtGetUser(connInfo.user); + if (pUser == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); + return; + } + + SCMMultiTableInfoMsg *pInfo = pMsg->pCont; + pInfo->numOfTables = htonl(pInfo->numOfTables); + + int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice + SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen); + if (pMultiMeta == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + + pMultiMeta->contLen = sizeof(SMultiTableMeta); + pMultiMeta->numOfTables = 0; + + for (int t = 0; t < pInfo->numOfTables; ++t) { + char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN); + STableInfo *pTable = mgmtGetTable(tableId); + if (pTable == NULL) continue; + + SDbObj *pDb = mgmtGetDbByTableId(tableId); + if (pDb == NULL) continue; + + int availLen = totalMallocLen - pMultiMeta->contLen; + if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) { + //TODO realloc + //totalMallocLen *= 2; + //pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen); + //if (pMultiMeta == NULL) { + /// rpcSendResponse(ahandle, TSDB_CODE_SERV_OUT_OF_MEMORY, NULL, 0); + // return TSDB_CODE_SERV_OUT_OF_MEMORY; + //} else { + // t--; + // continue; + //} + } + + STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen); + int32_t code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); + if (code == TSDB_CODE_SUCCESS) { + pMultiMeta->numOfTables ++; + pMultiMeta->contLen += pMeta->contLen; + } + } + + SRpcMsg rpcRsp = {0}; + rpcRsp.handle = pMsg->thandle; + rpcRsp.pCont = pMultiMeta; + rpcRsp.contLen = pMultiMeta->contLen; + rpcSendResponse(&rpcRsp); +} + +static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + SDbObj *pDb = mgmtGetDb(pShow->db); + if (pDb == NULL) { + return TSDB_CODE_DB_NOT_SELECTED; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "table name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "stable name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = pDb->numOfTables; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { + if (rows < capacity) { + for (int32_t i = 0; i < numOfCols; ++i) { + memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); + } + } +} + +static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + SDbObj *pDb = mgmtGetDb(pShow->db); + if (pDb == NULL) return 0; + + SUserObj *pUser = mgmtGetUserFromConn(pConn); + if (pUser == NULL) return 0; + + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 && + strcmp(pUser->user, "monitor") != 0) { + return 0; + } + } + + int32_t numOfRows = 0; + SChildTableObj *pTable = NULL; + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + char prefix[64] = {0}; + strcpy(prefix, pDb->name); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = strlen(prefix); + + while (numOfRows < rows) { + pShow->pNode = sdbFetchRow(tsNormalTableSdb, pShow->pNode, (void **) &pTable); + if (pTable == NULL) break; + + // not belong to current db + if (strncmp(tableId, prefix, prefixLen)) { + continue; + } + + char tableName[TSDB_TABLE_NAME_LEN] = {0}; + memset(tableName, 0, tListLen(tableName)); + numOfRead++; + + // pattern compare for meter name + mgmtExtractTableName(pTable->info.tableId, tableName); + + if (pShow->payloadLen > 0 && + patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { + continue; + } + + int32_t cols = 0; + + char *pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + strncpy(pWrite, tableName, TSDB_TABLE_NAME_LEN); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *) pWrite = pTable->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pTable->info.type == TSDB_CHILD_TABLE) { + *(int16_t *)pWrite = pTable->superTable->numOfColumns; + } else { + *(int16_t *)pWrite = pTable->numOfColumns; + } + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pTable->info.type == TSDB_CHILD_TABLE) { + mgmtExtractTableName(pTable->superTableId, pWrite); + } + cols++; + + numOfRows++; + } + + pShow->numOfReads += numOfRead; + const int32_t NUM_OF_COLUMNS = 4; + + mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); + + return numOfRows; +} + +void mgmtAlterChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable) { + int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; + SCMAlterTableMsg *pAlter = pMsg->pCont;; + + if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { + code = mgmtModifyChildTableTagValueByName((SChildTableObj *)pTable, pAlter->schema[0].name, pAlter->tagVal); + } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { + code = mgmtAddNormalTableColumn((SNormalTableObj *)pTable, pAlter->schema, 1); + } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { + code = mgmtDropNormalTableColumnByName((SNormalTableObj *)pTable, pAlter->schema[0].name); + } else { + } + + mgmtSendSimpleResp(pMsg->thandle, code); } \ No newline at end of file diff --git a/src/mnode/src/mgmtNormalTable.c b/src/mnode/src/mgmtNormalTable.c deleted file mode 100644 index fde9db2d74..0000000000 --- a/src/mnode/src/mgmtNormalTable.c +++ /dev/null @@ -1,551 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "tscompression.h" -#include "tskiplist.h" -#include "ttime.h" -#include "tstatus.h" -#include "tutil.h" -#include "mnode.h" -#include "mgmtAcct.h" -#include "mgmtDb.h" -#include "mgmtDClient.h" -#include "mgmtGrant.h" -#include "mgmtMnode.h" -#include "mgmtNormalTable.h" -#include "mgmtSdb.h" -#include "mgmtSuperTable.h" -#include "mgmtTable.h" -#include "mgmtVgroup.h" - -void *tsNormalTableSdb; -int32_t tsNormalTableUpdateSize; - -static void mgmtDestroyNormalTable(SNormalTableObj *pTable) { - tfree(pTable->schema); - tfree(pTable->sql); - tfree(pTable); -} - -static int32_t mgmtNormalTableActionDestroy(SSdbOperDesc *pOper) { - mgmtDestroyNormalTable(pOper->pObj); - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtNormalTableActionInsert(SSdbOperDesc *pOper) { - SNormalTableObj *pTable = pOper->pObj; - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("ntable:%s not in vgroup:%d", pTable->tableId, pTable->vgId); - return TSDB_CODE_INVALID_VGROUP_ID; - } - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) { - mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return TSDB_CODE_INVALID_DB; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("ntable:%s, account:%s not exists", pTable->tableId, pDb->cfg.acct); - return TSDB_CODE_INVALID_ACCT; - } - - mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); - mgmtAddTableIntoDb(pDb); - mgmtAddTableIntoVgroup(pVgroup, (STableInfo *) pTable); - - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtNormalTableActionDelete(SSdbOperDesc *pOper) { - SNormalTableObj *pTable = pOper->pObj; - if (pTable->vgId == 0) { - return TSDB_CODE_INVALID_VGROUP_ID; - } - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - return TSDB_CODE_INVALID_VGROUP_ID; - } - - SDbObj *pDb = mgmtGetDb(pVgroup->dbName); - if (pDb == NULL) { - mError("ntable:%s, vgroup:%d not in DB:%s", pTable->tableId, pVgroup->vgId, pVgroup->dbName); - return TSDB_CODE_INVALID_DB; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("account not exists"); - return TSDB_CODE_INVALID_ACCT; - } - - mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); - mgmtRemoveTableFromDb(pDb); - mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); - - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtNormalTableActionUpdate(SSdbOperDesc *pOper) { - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtNormalTableActionEncode(SSdbOperDesc *pOper) { - SNormalTableObj *pTable = pOper->pObj; - assert(pOper->pObj != NULL && pOper->rowData != NULL); - - int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (pOper->maxRowSize < tsNormalTableUpdateSize + schemaSize) { - return TSDB_CODE_INVALID_MSG_LEN; - } - - memcpy(pOper->rowData, pTable, tsNormalTableUpdateSize); - memcpy(pOper->rowData + tsNormalTableUpdateSize, pTable->schema, schemaSize); - memcpy(pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - - pOper->rowSize = tsNormalTableUpdateSize + schemaSize + pTable->sqlLen; - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtNormalTableActionDecode(SSdbOperDesc *pOper) { - assert(pOper->rowData != NULL); - - SNormalTableObj *pTable = (SNormalTableObj *)calloc(1, sizeof(SNormalTableObj)); - if (pTable == NULL) TSDB_CODE_SERV_OUT_OF_MEMORY; - - memcpy(pTable, pOper->rowData, tsNormalTableUpdateSize); - - int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - pTable->schema = (SSchema *)malloc(schemaSize); - if (pTable->schema == NULL) { - mgmtDestroyNormalTable(pTable); - return -1; - } - - memcpy(pTable->schema, pOper->rowData + tsNormalTableUpdateSize, schemaSize); - - pTable->sql = (char *)malloc(pTable->sqlLen); - if (pTable->sql == NULL) { - mgmtDestroyNormalTable(pTable); - return -1; - } - memcpy(pTable->sql, pOper->rowData + tsNormalTableUpdateSize + schemaSize, pTable->sqlLen); - - pOper->pObj = pTable; - return TSDB_CODE_SUCCESS; -} - -int32_t mgmtInitNormalTables() { - void *pNode = NULL; - void *pLastNode = NULL; - SNormalTableObj *pTable = NULL; - - SNormalTableObj tObj; - tsNormalTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; - - SSdbTableDesc tableDesc = { - .tableName = "ntables", - .hashSessions = TSDB_MAX_NORMAL_TABLES, - .maxRowSize = sizeof(SNormalTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, - .keyType = SDB_KEY_TYPE_STRING, - .insertFp = mgmtNormalTableActionInsert, - .deleteFp = mgmtNormalTableActionDelete, - .updateFp = mgmtNormalTableActionUpdate, - .encodeFp = mgmtNormalTableActionEncode, - .decodeFp = mgmtNormalTableActionDecode, - .destroyFp = mgmtNormalTableActionDestroy, - }; - - tsNormalTableSdb = sdbOpenTable(&tableDesc); - if (tsNormalTableSdb == NULL) { - mError("failed to init ntables data"); - return -1; - } - - while (1) { - pLastNode = pNode; - pNode = sdbFetchRow(tsNormalTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) break; - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("ntable:%s, failed to get db, discard it", pTable->tableId); - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; - desc.pObj = pTable; - desc.table = tsNormalTableSdb; - sdbDeleteRow(&desc); - pNode = pLastNode; - continue; - } - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("ntable:%s, failed to get vgroup:%d sid:%d, discard it", pTable->tableId, pTable->vgId, pTable->sid); - pTable->vgId = 0; - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; - desc.pObj = pTable; - desc.table = tsNormalTableSdb; - sdbDeleteRow(&desc); - pNode = pLastNode; - continue; - } - - if (strcmp(pVgroup->dbName, pDb->name) != 0) { - mError("ntable:%s, db:%s not match with vgroup:%d db:%s sid:%d, discard it", - pTable->tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); - pTable->vgId = 0; - - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; - desc.pObj = pTable; - desc.table = tsNormalTableSdb; - sdbDeleteRow(&desc); - pNode = pLastNode; - continue; - } - - if (pVgroup->tableList == NULL) { - mError("ntable:%s, vgroup:%d tableList is null", pTable->tableId, pTable->vgId); - pTable->vgId = 0; - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_LOCAL; - desc.pObj = pTable; - desc.table = tsNormalTableSdb; - sdbDeleteRow(&desc); - pNode = pLastNode; - continue; - } - } - - mTrace("ntables is initialized"); - return 0; -} - -void mgmtCleanUpNormalTables() { - sdbCloseTable(tsNormalTableSdb); -} - -void *mgmtBuildCreateNormalTableMsg(SNormalTableObj *pTable) { - int32_t totalCols = pTable->numOfColumns; - int32_t contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen; - - SMDCreateTableMsg *pCreate = rpcMallocCont(contLen); - if (pCreate == NULL) { - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - - memcpy(pCreate->tableId, pTable->tableId, TSDB_TABLE_ID_LEN + 1); - pCreate->contLen = htonl(contLen); - pCreate->vgId = htonl(pTable->vgId); - pCreate->tableType = pTable->type; - pCreate->numOfColumns = htons(pTable->numOfColumns); - pCreate->numOfTags = 0; - pCreate->sid = htonl(pTable->sid); - pCreate->sversion = htonl(pTable->sversion); - pCreate->tagDataLen = 0; - pCreate->sqlDataLen = htonl(pTable->sqlLen); - pCreate->uid = htobe64(pTable->uid); - pCreate->superTableUid = 0; - pCreate->createdTime = htobe64(pTable->createdTime); - - SSchema *pSchema = (SSchema *) pCreate->data; - memcpy(pSchema, pTable->schema, totalCols * sizeof(SSchema)); - for (int32_t col = 0; col < totalCols; ++col) { - pSchema->bytes = htons(pSchema->bytes); - pSchema->colId = htons(pSchema->colId); - pSchema++; - } - - memcpy(pCreate + sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema), pTable->sql, pTable->sqlLen); - return pCreate; -} - -void *mgmtCreateNormalTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t sid) { - SNormalTableObj *pTable = (SNormalTableObj *) calloc(sizeof(SNormalTableObj), 1); - if (pTable == NULL) { - mError("table:%s, failed to alloc memory", pCreate->tableId); - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - - strcpy(pTable->tableId, pCreate->tableId); - pTable->type = TSDB_NORMAL_TABLE; - pTable->vgId = pVgroup->vgId; - pTable->createdTime = taosGetTimestampMs(); - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); - pTable->sid = sid; - pTable->sversion = 0; - pTable->numOfColumns = htons(pCreate->numOfColumns); - pTable->sqlLen = htons(pCreate->sqlLen); - - int32_t numOfCols = pTable->numOfColumns; - int32_t schemaSize = numOfCols * sizeof(SSchema); - pTable->schema = (SSchema *) calloc(1, schemaSize); - if (pTable->schema == NULL) { - free(pTable); - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - memcpy(pTable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); - - pTable->nextColId = 0; - for (int32_t col = 0; col < numOfCols; col++) { - SSchema *tschema = pTable->schema; - tschema[col].colId = pTable->nextColId++; - tschema[col].bytes = htons(tschema[col].bytes); - } - - if (pTable->sqlLen != 0) { - pTable->type = TSDB_STREAM_TABLE; - pTable->sql = calloc(1, pTable->sqlLen); - if (pTable->sql == NULL) { - free(pTable); - terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; - return NULL; - } - memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); - pTable->sql[pTable->sqlLen - 1] = 0; - mTrace("table:%s, stream sql len:%d sql:%s", pTable->tableId, pTable->sqlLen, pTable->sql); - } - - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_GLOBAL; - desc.pObj = pTable; - desc.table = tsNormalTableSdb; - if (sdbInsertRow(&desc) != TSDB_CODE_SUCCESS) { - mError("table:%s, update sdb error", pTable->tableId); - free(pTable); - terrno = TSDB_CODE_SDB_ERROR; - return NULL; - } - - mTrace("table:%s, create ntable in vgroup, uid:%" PRIu64 , pTable->tableId, pTable->uid); - return pTable; -} - -int32_t mgmtDropNormalTable(SQueuedMsg *newMsg, SNormalTableObj *pTable) { - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("table:%s, failed to drop normal table, vgroup not exist", pTable->tableId); - return TSDB_CODE_OTHERS; - } - - SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); - if (pDrop == NULL) { - mError("table:%s, failed to drop normal table, no enough memory", pTable->tableId); - return TSDB_CODE_SERV_OUT_OF_MEMORY; - } - - strcpy(pDrop->tableId, pTable->tableId); - pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); - pDrop->vgId = htonl(pVgroup->vgId); - pDrop->sid = htonl(pTable->sid); - pDrop->uid = htobe64(pTable->uid); - - SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); - mTrace("table:%s, send drop table msg", pDrop->tableId); - SRpcMsg rpcMsg = { - .handle = newMsg, - .pCont = pDrop, - .contLen = sizeof(SMDDropTableMsg), - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_DROP_TABLE - }; - - newMsg->ahandle = pTable; - mgmtSendMsgToDnode(&ipSet, &rpcMsg); - return TSDB_CODE_SUCCESS; -} - -void* mgmtGetNormalTable(char *tableId) { - return sdbGetRow(tsNormalTableSdb, tableId); -} - -static int32_t mgmtFindNormalTableColumnIndex(SNormalTableObj *pTable, char *colName) { - SSchema *schema = (SSchema *) pTable->schema; - for (int32_t i = 0; i < pTable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; - } - } - - return -1; -} - -int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols) { - if (ncols <= 0) { - return TSDB_CODE_APP_ERROR; - } - - for (int32_t i = 0; i < ncols; i++) { - if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; - } - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("table: %s not belongs to any database", pTable->tableId); - return TSDB_CODE_APP_ERROR; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("DB: %s not belongs to andy account", pDb->name); - return TSDB_CODE_APP_ERROR; - } - - int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols); - - memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols); - - SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns); - for (int32_t i = 0; i < ncols; i++) { - tschema[i].colId = pTable->nextColId++; - } - - pTable->numOfColumns += ncols; - pTable->sversion++; - pAcct->acctInfo.numOfTimeSeries += ncols; - - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_GLOBAL; - desc.pObj = pTable; - desc.table = tsNormalTableSdb; - desc.rowData = pTable; - desc.rowSize = tsNormalTableUpdateSize; - sdbUpdateRow(&desc); - - return TSDB_CODE_SUCCESS; -} - -int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) { - int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; - } - - SDbObj *pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL) { - mError("table: %s not belongs to any database", pTable->tableId); - return TSDB_CODE_APP_ERROR; - } - - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); - if (pAcct == NULL) { - mError("DB: %s not belongs to any account", pDb->name); - return TSDB_CODE_APP_ERROR; - } - - memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pTable->numOfColumns - col - 1)); - - pTable->numOfColumns--; - pTable->sversion++; - - pAcct->acctInfo.numOfTimeSeries--; - - SSdbOperDesc desc = {0}; - desc.type = SDB_OPER_TYPE_GLOBAL; - desc.pObj = pTable; - desc.table = tsNormalTableSdb; - desc.rowData = pTable; - desc.rowSize = tsNormalTableUpdateSize; - sdbUpdateRow(&desc); - - return TSDB_CODE_SUCCESS; -} - -static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SNormalTableObj *pTable) { - int32_t numOfCols = pTable->numOfColumns; - for (int32_t i = 0; i < numOfCols; ++i) { - strcpy(pSchema->name, pTable->schema[i].name); - pSchema->type = pTable->schema[i].type; - pSchema->bytes = htons(pTable->schema[i].bytes); - pSchema->colId = htons(pTable->schema[i].colId); - pSchema++; - } - - return numOfCols * sizeof(SSchema); -} - -int32_t mgmtGetNormalTableMeta(SDbObj *pDb, SNormalTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { - pMeta->uid = htobe64(pTable->uid); - pMeta->sid = htonl(pTable->sid); - pMeta->vgId = htonl(pTable->vgId); - pMeta->sversion = htons(pTable->sversion); - pMeta->precision = pDb->cfg.precision; - pMeta->numOfTags = 0; - pMeta->numOfColumns = htons(pTable->numOfColumns); - pMeta->tableType = pTable->type; - pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); - - strncpy(pMeta->tableId, pTable->tableId, tListLen(pTable->tableId)); - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - return TSDB_CODE_INVALID_TABLE; - } - for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - if (usePublicIp) { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; - } else { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; - } - - pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); - } - pMeta->numOfVpeers = pVgroup->numOfVnodes; - - return TSDB_CODE_SUCCESS; -} - -void mgmtDropAllNormalTables(SDbObj *pDropDb) { - void *pNode = NULL; - void *pLastNode = NULL; - int32_t numOfTables = 0; - int32_t dbNameLen = strlen(pDropDb->name); - SNormalTableObj *pTable = NULL; - - while (1) { - pNode = sdbFetchRow(tsNormalTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) break; - - if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { - SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_LOCAL, - .table = tsNormalTableSdb, - .pObj = pTable, - }; - sdbDeleteRow(&oper); - pNode = pLastNode; - numOfTables++; - continue; - } - } - - mTrace("db:%s, all normal tables:%d is dropped from sdb", pDropDb->name, numOfTables); -} diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 3ff78b985e..aa7a494a9f 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -138,13 +138,19 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { return; } + if (mgmtCheckExpired()) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); + return; + } + if (tsMgmtProcessShellMsgFp[rpcMsg->msgType] == NULL) { mgmtProcessUnSupportMsg(rpcMsg); rpcFreeCont(rpcMsg->pCont); return; } - SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle); + bool usePublicIp = false; + SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle, &usePublicIp); if (pUser == NULL) { mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_USER); rpcFreeCont(rpcMsg->pCont); @@ -158,6 +164,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { queuedMsg.contLen = rpcMsg->contLen; queuedMsg.pCont = rpcMsg->pCont; queuedMsg.pUser = pUser; + queuedMsg.usePublicIp = usePublicIp; (*tsMgmtProcessShellMsgFp[rpcMsg->msgType])(&queuedMsg); rpcFreeCont(rpcMsg->pCont); } else { @@ -167,6 +174,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { queuedMsg->contLen = rpcMsg->contLen; queuedMsg->pCont = rpcMsg->pCont; queuedMsg->pUser = pUser; + queuedMsg.usePublicIp = usePublicIp; mgmtAddToShellQueue(queuedMsg); } } diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index e8161deb4d..4b8a4aba76 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -15,7 +15,8 @@ #define _DEFAULT_SOURCE #include "os.h" - +#include "name.h" +#include "tsqlfunction.h" #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" @@ -27,14 +28,11 @@ #include "mgmtTable.h" #include "mgmtUser.h" #include "mgmtVgroup.h" -#include "mnode.h" - -#include "name.h" -#include "tsqlfunction.h" static void *tsSuperTableSdb; static int32_t tsSuperTableUpdateSize; - +static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *queueMsg); +static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); static int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mgmtGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -132,8 +130,10 @@ int32_t mgmtInitSuperTables() { return -1; } + mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_VGROUP, mgmtProcessSuperTableVgroupMsg); mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mgmtGetShowSuperTableMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mgmtRetrieveShowSuperTables); + mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); mTrace("stables is initialized"); return 0; @@ -143,10 +143,12 @@ void mgmtCleanUpSuperTables() { sdbCloseTable(tsSuperTableSdb); } -int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { +void mgmtCreateSuperTable(SQueuedMsg *pMsg) { + SCMCreateTableMsg *pCreate = pMsg->pCont; SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; } strcpy(pStable->tableId, pCreate->tableId); @@ -165,7 +167,8 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { if (pStable->schema == NULL) { free(pStable); mError("stable:%s, no schema input", pCreate->tableId); - return TSDB_CODE_INVALID_TABLE; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; } memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); @@ -186,17 +189,17 @@ int32_t mgmtCreateSuperTable(SCMCreateTableMsg *pCreate) { int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { mgmtDestroySuperTable(pStable); - return TSDB_CODE_SDB_ERROR; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); } else { mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); - return TSDB_CODE_SUCCESS; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); } } -int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pStable) { +void mgmtDropSuperTable(SQueuedMsg *newMsg, SSuperTableObj *pTable) { if (pStable->numOfTables != 0) { mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables); - return TSDB_CODE_OTHERS; + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { SSdbOperDesc oper = { .type = SDB_OPER_TYPE_GLOBAL, @@ -205,7 +208,7 @@ int32_t mgmtDropSuperTable(SQueuedMsg *newMsg, SDbObj *pDb, SSuperTableObj *pSta }; int32_t code = sdbDeleteRow(&oper); mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->tableId, tstrerror(code)); - return code; + mgmtSendSimpleResp(pMsg->thandle, code); } } @@ -213,14 +216,14 @@ void* mgmtGetSuperTable(char *tableId) { return sdbGetRow(tsSuperTableSdb, tableId); } -void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { +static void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); rsp->numOfDnodes = htonl(1); rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; } -int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) { +static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) { for (int32_t i = 0; i < pStable->numOfTags; i++) { SSchema *schema = (SSchema *)(pStable->schema + (pStable->numOfColumns + i) * sizeof(SSchema)); if (strcasecmp(tagName, schema->name) == 0) { @@ -231,7 +234,7 @@ int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) return -1; } -int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { +static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { return TSDB_CODE_APP_ERROR; } @@ -279,7 +282,7 @@ int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t return TSDB_CODE_SUCCESS; } -int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { +static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, tagName); if (col <= 0 || col >= pStable->numOfTags) { return TSDB_CODE_APP_ERROR; @@ -311,7 +314,7 @@ int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { return TSDB_CODE_SUCCESS; } -int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { +static int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { // Tag name does not exist @@ -362,7 +365,7 @@ static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colN return -1; } -int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { +static int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { return TSDB_CODE_APP_ERROR; } @@ -406,7 +409,7 @@ int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[], int32 return TSDB_CODE_SUCCESS; } -int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { +static int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *colName) { int32_t col = mgmtFindSuperTableColumnIndex(pStable, colName); if (col < 0) { return TSDB_CODE_APP_ERROR; @@ -519,12 +522,12 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v while (numOfRows < rows) { pShow->pNode = sdbFetchRow(tsSuperTableSdb, pShow->pNode, (void **) &pTable); if (pTable == NULL) break; - if (strncmp(pTable->tableId, prefix, prefixLen)) { + if (strncmp(pTable->info.tableId, prefix, prefixLen)) { continue; } memset(stableName, 0, tListLen(stableName)); - mgmtExtractTableName(pTable->tableId, stableName); + mgmtExtractTableName(pTable->info.tableId, stableName); if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) @@ -572,7 +575,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { break; } - if (strncmp(pDropDb->name, pTable->tableId, dbNameLen) == 0) { + if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOperDesc oper = { .type = SDB_OPER_TYPE_LOCAL, .table = tsSuperTableSdb, @@ -588,14 +591,6 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } -void mgmtAddTableIntoSuperTable(SSuperTableObj *pStable) { - pStable->numOfTables++; -} - -void mgmtRemoveTableFromSuperTable(SSuperTableObj *pStable) { - pStable->numOfTables--; -} - int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; for (int32_t i = 0; i < numOfCols; ++i) { @@ -609,32 +604,67 @@ int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { return (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); } -int32_t mgmtGetSuperTableMeta(SDbObj *pDb, SSuperTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { +void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) { + SCMTableInfoMsg *pInfo = pMsg->pCont; + SDbObj *pDb = pMsg->pDb; + + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); pMeta->uid = htobe64(pTable->uid); - pMeta->sid = htonl(pTable->sid); - pMeta->vgId = htonl(pTable->vgId); pMeta->sversion = htons(pTable->sversion); pMeta->precision = pDb->cfg.precision; pMeta->numOfTags = pTable->numOfTags; pMeta->numOfColumns = htons(pTable->numOfColumns); - pMeta->tableType = pTable->type; + pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); - strcpy(pMeta->tableId, pTable->tableId); + strcpy(pMeta->tableId, pTable->info.tableId); - return TSDB_CODE_SUCCESS; + SRpcMsg rpcRsp = { + .handle = pMsg->thandle, + .pCont = pMeta, + .contLen = pMeta->contLen, + }; + pMeta->contLen = htons(pMeta->contLen); + rpcSendResponse(&rpcRsp); + + mTrace("stable:%%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid); } -int32_t mgmtExtractTableName(const char* tableId, char* name) { - int pos = -1; - int num = 0; - for (pos = 0; tableId[pos] != 0; ++pos) { - if (tableId[pos] == '.') num++; - if (num == 2) break; +static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { + SCMSuperTableInfoMsg *pInfo = pMsg->pCont; + STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId); + if (pTable == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; } - if (num == 2) { - strcpy(name, tableId + pos + 1); + SCMSuperTableInfoRsp *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); + if (pRsp != NULL) { + int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t); + SRpcMsg rpcRsp = {0}; + rpcRsp.handle = pMsg->thandle; + rpcRsp.pCont = pRsp; + rpcRsp.contLen = msgLen; + rpcSendResponse(&rpcRsp); + } else { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); } - return 0; } +void mgmtAlterSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable) { + int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; + SCMAlterTableMsg *pAlter = pMsg->pCont; + + if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { + code = mgmtAddSuperTableTag((SSuperTableObj *) pTable, pAlter->schema, 1); + } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { + code = mgmtDropSuperTableTag((SSuperTableObj *) pTable, pAlter->schema[0].name); + } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { + code = mgmtModifySuperTableTagNameByName((SSuperTableObj *) pTable, pAlter->schema[0].name, pAlter->schema[1].name); + } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { + code = mgmtAddSuperTableColumn((SSuperTableObj *) pTable, pAlter->schema, 1); + } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { + code = mgmtDropSuperTableColumnByName((SSuperTableObj *) pTable, pAlter->schema[0].name); + } else {} + + mgmtSendSimpleResp(pMsg->thandle, code); +} \ No newline at end of file diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index c9a0333b17..0d02e5f8e1 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -15,8 +15,6 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "mgmtTable.h" #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDClient.h" @@ -25,44 +23,17 @@ #include "mgmtDServer.h" #include "mgmtGrant.h" #include "mgmtMnode.h" -#include "mgmtNormalTable.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" #include "mgmtSuperTable.h" +#include "mgmtTable.h" #include "mgmtUser.h" -#include "mgmtVgroup.h" -#include "mnode.h" - -#include "qast.h" -#include "qextbuffer.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tscompression.h" -#include "tskiplist.h" -#include "tsqlfunction.h" -#include "tstatus.h" -#include "ttime.h" -#include "name.h" - -extern void *tsNormalTableSdb; -extern void *tsChildTableSdb; static void mgmtProcessCreateTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessDropTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessTableMetaMsg(SQueuedMsg *queueMsg); -static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *queueMsg); -static void mgmtProcessSuperTableMetaMsg(SQueuedMsg *queueMsg); -static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg); -static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg); -static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); -static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg); -static int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle); -static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); int32_t mgmtInitTables() { int32_t code = mgmtInitSuperTables(); @@ -70,11 +41,6 @@ int32_t mgmtInitTables() { return code; } - code = mgmtInitNormalTables(); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - code = mgmtInitChildTables(); if (code != TSDB_CODE_SUCCESS) { return code; @@ -84,31 +50,17 @@ int32_t mgmtInitTables() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_TABLE, mgmtProcessDropTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mgmtProcessAlterTableMsg); mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLE_META, mgmtProcessTableMetaMsg); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_TABLES_META, mgmtProcessMultiTableMetaMsg); - mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_STABLE_META, mgmtProcessSuperTableMetaMsg); - mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mgmtGetShowTableMeta); - mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mgmtRetrieveShowTables); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mgmtProcessCreateTableRsp); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mgmtProcessDropTableRsp); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP, mgmtProcessAlterTableRsp); - mgmtAddDClientRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mgmtProcessDropStableRsp); - mgmtAddDServerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_TABLE, mgmtProcessTableCfgMsg); return TSDB_CODE_SUCCESS; } -STableInfo* mgmtGetTable(char *tableId) { - STableInfo *tableInfo = (STableInfo *) mgmtGetSuperTable(tableId); +STableInfo *mgmtGetTable(char *tableId) { + STableInfo *tableInfo = mgmtGetSuperTable(tableId); if (tableInfo != NULL) { return tableInfo; } - tableInfo = (STableInfo *) mgmtGetNormalTable(tableId); - if (tableInfo != NULL) { - return tableInfo; - } - - tableInfo = (STableInfo *) mgmtGetChildTable(tableId); + tableInfo = mgmtGetChildTable(tableId); if (tableInfo != NULL) { return tableInfo; } @@ -116,250 +68,29 @@ STableInfo* mgmtGetTable(char *tableId) { return NULL; } -STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { - SDnodeObj *pObj = mgmtGetDnode(dnodeId); - SVgObj *pVgroup = mgmtGetVgroup(vnode); - - if (pObj == NULL || pVgroup == NULL) { - return NULL; - } - - return pVgroup->tableList[sid]; -} - -int32_t mgmtGetTableMeta(SDbObj *pDb, STableInfo *pTable, STableMetaMsg *pMeta, bool usePublicIp) { - if (pTable->type == TSDB_CHILD_TABLE) { - mgmtGetChildTableMeta(pDb, (SChildTableObj *) pTable, pMeta, usePublicIp); - } else if (pTable->type == TSDB_NORMAL_TABLE) { - mgmtGetNormalTableMeta(pDb, (SNormalTableObj *) pTable, pMeta, usePublicIp); - } else if (pTable->type == TSDB_SUPER_TABLE) { - mgmtGetSuperTableMeta(pDb, (SSuperTableObj *) pTable, pMeta, usePublicIp); - } else { - mTrace("%s, uid:%" PRIu64 " table meta retrieve failed, invalid type", pTable->tableId, pTable->uid); - return TSDB_CODE_INVALID_TABLE; - } - - mTrace("%s, uid:%" PRIu64 " table meta is retrieved", pTable->tableId, pTable->uid); - return TSDB_CODE_SUCCESS; -} - -int32_t mgmtAlterTable(SDbObj *pDb, SCMAlterTableMsg *pAlter) { - STableInfo *pTable = mgmtGetTable(pAlter->tableId); - if (pTable == NULL) { - return TSDB_CODE_INVALID_TABLE; - } - - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { - return TSDB_CODE_MONITOR_DB_FORBIDDEN; - } - - if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { - if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtAddSuperTableTag((SSuperTableObj *) pTable, pAlter->schema, 1); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { - if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtDropSuperTableTag((SSuperTableObj *) pTable, pAlter->schema[0].name); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtModifySuperTableTagNameByName((SSuperTableObj *) pTable, pAlter->schema[0].name, pAlter->schema[1].name); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - if (pTable->type == TSDB_CHILD_TABLE) { - return mgmtModifyChildTableTagValueByName((SChildTableObj *) pTable, pAlter->schema[0].name, pAlter->tagVal); - } - } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - if (pTable->type == TSDB_NORMAL_TABLE) { - return mgmtAddNormalTableColumn((SNormalTableObj *) pTable, pAlter->schema, 1); - } else if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtAddSuperTableColumn((SSuperTableObj *) pTable, pAlter->schema, 1); - } else {} - } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - if (pTable->type == TSDB_NORMAL_TABLE) { - return mgmtDropNormalTableColumnByName((SNormalTableObj *) pTable, pAlter->schema[0].name); - } else if (pTable->type == TSDB_SUPER_TABLE) { - return mgmtDropSuperTableColumnByName((SSuperTableObj *) pTable, pAlter->schema[0].name); - } else {} - } else {} - - return TSDB_CODE_OPS_NOT_SUPPORT; -} - void mgmtCleanUpTables() { - mgmtCleanUpNormalTables(); mgmtCleanUpChildTables(); mgmtCleanUpSuperTables(); } -int32_t mgmtGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - SDbObj *pDb = mgmtGetDb(pShow->db); - if (pDb == NULL) { - return TSDB_CODE_DB_NOT_SELECTED; +void mgmtExtractTableName(const char* tableId, char* name) { + int pos = -1; + int num = 0; + for (pos = 0; tableId[pos] != 0; ++pos) { + if (tableId[pos] == '.') num++; + if (num == 2) break; } - int32_t cols = 0; - SSchema *pSchema = pMeta->schema; - - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "table name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "stable name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htons(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - } - - pShow->numOfRows = pDb->numOfTables; - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - - return 0; -} - -static void mgmtVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { - if (rows < capacity) { - for (int32_t i = 0; i < numOfCols; ++i) { - memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); - } + if (num == 2) { + strcpy(name, tableId + pos + 1); } } -int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { - SDbObj *pDb = mgmtGetDb(pShow->db); - if (pDb == NULL) return 0; - - SUserObj *pUser = mgmtGetUserFromConn(pConn); - if (pUser == NULL) return 0; - - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { - if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 && - strcmp(pUser->user, "monitor") != 0) { - return 0; - } - } - - int32_t numOfRows = 0; - int32_t numOfRead = 0; - int32_t cols = 0; - void *pTable = NULL; - char *pWrite = NULL; - char prefix[20] = {0}; - SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - strcpy(prefix, pDb->name); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); - - while (numOfRows < rows) { - int16_t numOfColumns = 0; - int64_t createdTime = 0; - char *tableId = NULL; - char *superTableId = NULL; - void *pNormalTableNode = sdbFetchRow(tsNormalTableSdb, pShow->pNode, (void **) &pTable); - if (pTable != NULL) { - SNormalTableObj *pNormalTable = (SNormalTableObj *) pTable; - pShow->pNode = pNormalTableNode; - tableId = pNormalTable->tableId; - superTableId = NULL; - createdTime = pNormalTable->createdTime; - numOfColumns = pNormalTable->numOfColumns; - } else { - pShow->pNode = NULL; - void *pChildTableNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable); - if (pTable != NULL) { - SChildTableObj *pChildTable = (SChildTableObj *) pTable; - pShow->pNode = pChildTableNode; - tableId = pChildTable->tableId; - superTableId = pChildTable->superTableId; - createdTime = pChildTable->createdTime; - numOfColumns = pChildTable->superTable->numOfColumns; - } else { - break; - } - } - - // not belong to current db - if (strncmp(tableId, prefix, prefixLen)) { - continue; - } - - char tableName[TSDB_TABLE_NAME_LEN] = {0}; - memset(tableName, 0, tListLen(tableName)); - numOfRead++; - - // pattern compare for meter name - mgmtExtractTableName(tableId, tableName); - - if (pShow->payloadLen > 0 && - patternMatch(pShow->payload, tableName, TSDB_TABLE_NAME_LEN, &info) != TSDB_PATTERN_MATCH) { - continue; - } - - cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, tableName, TSDB_TABLE_NAME_LEN); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *) pWrite = createdTime; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *) pWrite = numOfColumns; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (superTableId != NULL) { - mgmtExtractTableName(superTableId, pWrite); - } - cols++; - - numOfRows++; - } - - pShow->numOfReads += numOfRead; - const int32_t NUM_OF_COLUMNS = 4; - - mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); - - return numOfRows; -} - -void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - +static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->pCont; mTrace("table:%s, create msg is received from thandle:%p", pCreate->tableId, pMsg->thandle); - if (mgmtCheckExpired()) { - mError("table:%s, failed to create, grant expired", pCreate->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); - return; - } + if (mgmtCheckRedirect(pMsg->thandle)) return; if (!pMsg->pUser->writeAuth) { mError("table:%s, failed to create, no rights", pCreate->tableId); @@ -367,16 +98,8 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { return; } - SAcctObj *pAcct = pMsg->pUser->pAcct; - int32_t code = mgmtCheckTableLimit(pAcct, htons(pCreate->numOfColumns)); - if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to create, exceed the limit", pCreate->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); - return; - } - pMsg->pDb = mgmtGetDb(pCreate->db); - if (pMsg->pDb == NULL) { + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { mError("table:%s, failed to create, db not selected", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; @@ -396,92 +119,19 @@ void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { } if (pCreate->numOfTags != 0) { - mTrace("table:%s, is a super table", pCreate->tableId); - code = mgmtCreateSuperTable(pCreate); - mgmtSendSimpleResp(pMsg->thandle, code); - return; - } - - code = mgmtCheckTimeSeries(pCreate->numOfColumns); - if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to create, timeseries exceed the limit", pCreate->tableId); - mgmtSendSimpleResp(pMsg->thandle, code); - return; - } - - SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); - memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); - pMsg->pCont = NULL; - - SVgObj *pVgroup = mgmtGetAvailableVgroup(pMsg->pDb); - if (pVgroup == NULL) { - mTrace("table:%s, start to create a new vgroup", pCreate->tableId); - mgmtCreateVgroup(newMsg); - return; - } - - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid < 0) { - mTrace("tables:%s, no enough sid in vgroup:%d", pVgroup->vgId); - mgmtCreateVgroup(newMsg); - return; - } - - SMDCreateTableMsg *pMDCreate = NULL; - if (pCreate->numOfColumns == 0) { - mTrace("table:%s, is a child table, vgroup:%d sid:%d ahandle:%p", pCreate->tableId, pVgroup->vgId, sid, pMsg); - pTable = mgmtCreateChildTable(pCreate, pVgroup, sid); - if (pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - mgmtFreeQueuedMsg(newMsg); - return; - } - pMDCreate = mgmtBuildCreateChildTableMsg(pCreate, (SChildTableObj *) pTable); - if (pMDCreate == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - mgmtFreeQueuedMsg(newMsg); - return; - } + mTrace("table:%s, is a stable", pCreate->tableId); + mgmtCreateSuperTable(pMsg); } else { - mTrace("table:%s, is a normal table, vgroup:%d sid:%d ahandle:%p", pCreate->tableId, pVgroup->vgId, sid, pMsg); - pTable = mgmtCreateNormalTable(pCreate, pVgroup, sid); - if (pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - mgmtFreeQueuedMsg(newMsg); - return; - } - pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); - if (pMDCreate == NULL) { - mgmtSendSimpleResp(pMsg->thandle, terrno); - mgmtFreeQueuedMsg(newMsg); - return; - } + mTrace("table:%s, is a ctable", pCreate->tableId); + mgmtCreateChildTable(pMsg); } - - SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); - SRpcMsg rpcMsg = { - .handle = newMsg, - .pCont = pMDCreate, - .contLen = htonl(pMDCreate->contLen), - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE - }; - - newMsg->ahandle = pTable; - mgmtSendMsgToDnode(&ipSet, &rpcMsg); } -void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) return; - +static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->pCont; mTrace("table:%s, drop table msg is received from thandle:%p", pDrop->tableId, pMsg->thandle); - if (mgmtCheckExpired()) { - mError("table:%s, failed to drop, grant expired", pDrop->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); - return; - } + if (mgmtCheckRedirect(pMsg->thandle)) return; if (!pMsg->pUser->writeAuth) { mError("table:%s, failed to drop, no rights", pDrop->tableId); @@ -489,13 +139,19 @@ void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { return; } - SDbObj *pDb = mgmtGetDbByTableId(pDrop->tableId); - if (pDb == NULL) { + pMsg->pDb = mgmtGetDb(pCreate->db); + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { mError("table:%s, failed to drop table, db not selected", pDrop->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + mError("table:%s, failed to drop table, in monitor database", pDrop->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN); + return; + } + STableInfo *pTable = mgmtGetTable(pDrop->tableId); if (pTable == NULL) { if (pDrop->igNotExists) { @@ -509,409 +165,75 @@ void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { } } + if (pTable->type == TSDB_SUPER_TABLE) { + mTrace("table:%s, start to drop stable", pDrop->tableId); + mgmtDropSuperTable(pMsg, (SSuperTableObj *)pTable); + } else { + mTrace("table:%s, start to drop ctable", pDrop->tableId); + mgmtDropChildTable(pMsg, (SNormalTableObj *)pTable); + } +} + +static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { + SCMAlterTableMsg *pAlter = pMsg->pCont; + mTrace("table:%s, alter table msg is received from thandle:%p", pAlter->tableId, pMsg->thandle); + + if (mgmtCheckRedirect(pMsg->thandle)) return; + + if (!pMsg->pUser->writeAuth) { + mTrace("table:%s, failed to alter table, no rights", pAlter->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); + return; + } + + pMsg->pDb = mgmtGetDbByTableId(pTable->tableId); + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + mError("table:%s, failed to alter table, db not selected", pTable->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); + return; + } + if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { - mError("table:%s, failed to drop table, in monitor database", pDrop->tableId); + mError("table:%s, failed to alter table, its log db", pTable->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN); return; } - SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); - memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); - pMsg->pCont = NULL; - int32_t code; - - switch (pTable->type) { - case TSDB_SUPER_TABLE: - mTrace("table:%s, start to drop super table", pDrop->tableId); - code = mgmtDropSuperTable(newMsg, pDb, (SSuperTableObj *) pTable); - break; - case TSDB_CHILD_TABLE: - mTrace("table:%s, start to drop child table", pDrop->tableId); - code = mgmtDropChildTable(newMsg, (SChildTableObj *) pTable); - break; - case TSDB_NORMAL_TABLE: - mTrace("table:%s, start to drop normal table", pDrop->tableId); - code = mgmtDropNormalTable(newMsg, (SNormalTableObj *) pTable); - break; - case TSDB_STREAM_TABLE: - mTrace("table:%s, start to drop stream table", pDrop->tableId); - code = mgmtDropNormalTable(newMsg, (SNormalTableObj *) pTable); - break; - default: - code = TSDB_CODE_INVALID_TABLE_TYPE; - mError("table:%s, invalid table type:%d", pDrop->tableId, pTable->type); - } - - if (code != TSDB_CODE_SUCCESS) { - free(newMsg); - mgmtSendSimpleResp(pMsg->thandle, code); - } -} - -void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { - if (mgmtCheckRedirect(pMsg->thandle)) { + STableInfo *pTable = mgmtGetTable(pAlter->tableId); + if (pTable == NULL) { + mError("table:%s, failed to alter table, table not exist", pTable->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); return; } - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); - if (pUser == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); + pAlter->numOfCols = htons(pAlter->numOfCols); + if (pAlter->numOfCols > 2) { + mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_APP_ERROR); return; } - SCMAlterTableMsg *pAlter = pMsg->pCont; + for (int32_t i = 0; i < pAlter->numOfCols; ++i) { + pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes); + } - int32_t code; - if (!pUser->writeAuth) { - code = TSDB_CODE_NO_RIGHTS; + if (pTable->type == TSDB_SUPER_TABLE) { + mTrace("table:%s, start to alter stable", pDrop->tableId); + mgmtAlterSuperTable(pMsg, (SSuperTableObj *)pTable); } else { - pAlter->type = htons(pAlter->type); - pAlter->numOfCols = htons(pAlter->numOfCols); - - if (pAlter->numOfCols > 2) { - mError("table:%s error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); - code = TSDB_CODE_APP_ERROR; - } else { - SDbObj *pDb = mgmtGetDb(pAlter->db); - if (pDb) { - for (int32_t i = 0; i < pAlter->numOfCols; ++i) { - pAlter->schema[i].bytes = htons(pAlter->schema[i].bytes); - } - - code = mgmtAlterTable(pDb, pAlter); - if (code == 0) { - mLPrint("table:%s is altered by %s", pAlter->tableId, pUser->user); - } - } else { - code = TSDB_CODE_DB_NOT_SELECTED; - } - } + mTrace("table:%s, start to alter ctable", pDrop->tableId); + mgmtAlterChildTable(pMsg, (SNormalTableObj *)pTable); } - - mgmtSendSimpleResp(pMsg->thandle, code); } -void mgmtProcessGetTableMeta(STableInfo *pTable, void *thandle) { - SRpcMsg rpcRsp = {.handle = thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; - SDbObj* pDb = mgmtGetDbByTableId(pTable->tableId); - if (pDb == NULL || pDb->dirty) { - mError("table:%s, failed to get table meta, db not selected", pTable->tableId); - rpcRsp.code = TSDB_CODE_DB_NOT_SELECTED; - rpcSendResponse(&rpcRsp); - return; - } - - SRpcConnInfo connInfo; - if (rpcGetConnInfo(thandle, &connInfo) != 0) { - mError("conn:%p is already released while get table meta", thandle); - return; - } - - bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); - - STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); - rpcRsp.code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); - - if (rpcRsp.code != TSDB_CODE_SUCCESS) { - rpcFreeCont(pMeta); - } else { - rpcRsp.pCont = pMeta; - rpcRsp.contLen = pMeta->contLen; - - pMeta->contLen = htons(pMeta->contLen); - } - - rpcSendResponse(&rpcRsp); -} - -void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { - SCMTableInfoMsg *pInfo = pMsg->pCont; - pInfo->createFlag = htons(pInfo->createFlag); - - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); - if (pUser == NULL) { - mError("table:%s, failed to get table meta, invalid user", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); - return; - } +static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { + SCMTableInfoMsg *pInfo = mgmtGetTable(pInfo->tableId); + mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle); STableInfo *pTable = mgmtGetTable(pInfo->tableId); - if (pTable == NULL) { - if (pInfo->createFlag != 1) { - mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; - } else { - // on demand create table from super table if table does not exists - if (mgmtCheckRedirect(pMsg->thandle)) { - mError("table:%s, failed to create table while get meta info, need redirect message", pInfo->tableId); - return; - } - - int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData); - SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); - if (pCreateMsg == NULL) { - mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); - return; - } - - memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData)); - strcpy(pCreateMsg->tableId, pInfo->tableId); - - mError("table:%s, start to create table while get meta info", pInfo->tableId); -// mgmtCreateTable(pCreateMsg, contLen, pMsg->thandle, true); - } + if (pTable == NULL || pTable->type != TSDB_SUPER_TABLE) { + mgmtGetChildTableMeta(pMsg, (SSuperTableObj *)pTable); } else { - mgmtProcessGetTableMeta(pTable, pMsg->thandle); + mgmtGetSuperTableMeta(pMsg, (SNormalTableObj *)pTable); } -} - -void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { - SRpcConnInfo connInfo; - if (rpcGetConnInfo(pMsg->thandle, &connInfo) != 0) { - mError("conn:%p is already released while get mulit table meta", pMsg->thandle); - return; - } - - bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); - SUserObj *pUser = mgmtGetUser(connInfo.user); - if (pUser == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_USER); - return; - } - - SCMMultiTableInfoMsg *pInfo = pMsg->pCont; - pInfo->numOfTables = htonl(pInfo->numOfTables); - - int32_t totalMallocLen = 4*1024*1024; // first malloc 4 MB, subsequent reallocation as twice - SMultiTableMeta *pMultiMeta = rpcMallocCont(totalMallocLen); - if (pMultiMeta == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); - return; - } - - pMultiMeta->contLen = sizeof(SMultiTableMeta); - pMultiMeta->numOfTables = 0; - - for (int t = 0; t < pInfo->numOfTables; ++t) { - char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN); - STableInfo *pTable = mgmtGetTable(tableId); - if (pTable == NULL) continue; - - SDbObj *pDb = mgmtGetDbByTableId(tableId); - if (pDb == NULL) continue; - - int availLen = totalMallocLen - pMultiMeta->contLen; - if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) { - //TODO realloc - //totalMallocLen *= 2; - //pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen); - //if (pMultiMeta == NULL) { - /// rpcSendResponse(ahandle, TSDB_CODE_SERV_OUT_OF_MEMORY, NULL, 0); - // return TSDB_CODE_SERV_OUT_OF_MEMORY; - //} else { - // t--; - // continue; - //} - } - - STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen); - int32_t code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); - if (code == TSDB_CODE_SUCCESS) { - pMultiMeta->numOfTables ++; - pMultiMeta->contLen += pMeta->contLen; - } - } - - SRpcMsg rpcRsp = {0}; - rpcRsp.handle = pMsg->thandle; - rpcRsp.pCont = pMultiMeta; - rpcRsp.contLen = pMultiMeta->contLen; - rpcSendResponse(&rpcRsp); -} - -void mgmtProcessSuperTableMetaMsg(SQueuedMsg *pMsg) { - SCMSuperTableInfoMsg *pInfo = pMsg->pCont; - STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId); - if (pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; - } - - SCMSuperTableInfoRsp *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); - if (pRsp != NULL) { - int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t); - SRpcMsg rpcRsp = {0}; - rpcRsp.handle = pMsg->thandle; - rpcRsp.pCont = pRsp; - rpcRsp.contLen = msgLen; - rpcSendResponse(&rpcRsp); - } else { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - } -} - -static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; - - SQueuedMsg *queueMsg = rpcMsg->handle; - queueMsg->received++; - - STableInfo *pTable = queueMsg->ahandle; - mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->tableId, queueMsg->thandle, - rpcMsg->handle, tstrerror(rpcMsg->code)); - - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - if (pTable->type == TSDB_CHILD_TABLE) { - SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable - }; - sdbDeleteRow(&oper); - } else if (pTable->type == TSDB_NORMAL_TABLE){ - SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, - .table = tsNormalTableSdb, - .pObj = pTable - }; - sdbDeleteRow(&oper); - } else {} - mError("table:%s, failed to create in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); - } else { - mTrace("table:%s, created in dnode", pTable->tableId); - if (queueMsg->msgType != TSDB_MSG_TYPE_CM_CREATE_TABLE) { - SQueuedMsg *newMsg = calloc(1, sizeof(SQueuedMsg)); - newMsg->msgType = queueMsg->msgType; - newMsg->thandle = queueMsg->thandle; - newMsg->pDb = queueMsg->pDb; - newMsg->pUser = queueMsg->pUser; - newMsg->contLen = queueMsg->contLen; - newMsg->pCont = rpcMallocCont(newMsg->contLen); - memcpy(newMsg->pCont, queueMsg->pCont, newMsg->contLen); - mTrace("table:%s, start to get meta", pTable->tableId); - mgmtAddToShellQueue(newMsg); - } else { - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); - } - } - - free(queueMsg); -} - -static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg) { - mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); -} - -static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { - if (rpcMsg->handle == NULL) return; - - SQueuedMsg *queueMsg = rpcMsg->handle; - queueMsg->received++; - - STableInfo *pTable = queueMsg->ahandle; - mTrace("table:%s, drop table rsp received, thandle:%p result:%s", pTable->tableId, queueMsg->thandle, tstrerror(rpcMsg->code)); - - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - mError("table:%s, failed to drop in dnode, reason:%s", pTable->tableId, tstrerror(rpcMsg->code)); - mgmtSendSimpleResp(queueMsg->thandle, rpcMsg->code); - free(queueMsg); - return; - } - - SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); - if (pVgroup == NULL) { - mError("table:%s, failed to get vgroup", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID); - free(queueMsg); - return; - } - - if (pTable->type == TSDB_CHILD_TABLE) { - SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable - }; - int32_t code = sdbDeleteRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, update ctables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } - } else if (pTable->type == TSDB_NORMAL_TABLE){ - SSdbOperDesc oper = { - .type = SDB_OPER_TYPE_GLOBAL, - .table = tsNormalTableSdb, - .pObj = pTable - }; - int32_t code = sdbDeleteRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - mError("table:%s, update ntables sdb error", pTable->tableId); - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SDB_ERROR); - free(queueMsg); - return; - } - } - - if (pVgroup->numOfTables <= 0) { - mPrint("vgroup:%d, all tables is dropped, drop vgroup", pVgroup->vgId); - mgmtDropVgroup(pVgroup, NULL); - } - - mgmtSendSimpleResp(queueMsg->thandle, TSDB_CODE_SUCCESS); - free(queueMsg); -} - -static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { - mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); -} - -static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { - if (mgmtCheckRedirect(rpcMsg->handle)) return; - - SDMConfigTableMsg *pCfg = (SDMConfigTableMsg *) rpcMsg->pCont; - pCfg->dnode = htonl(pCfg->dnode); - pCfg->vnode = htonl(pCfg->vnode); - pCfg->sid = htonl(pCfg->sid); - mTrace("dnode:%s, vnode:%d, sid:%d, receive table config msg", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); - - STableInfo *pTable = mgmtGetTableByPos(pCfg->dnode, pCfg->vnode, pCfg->sid); - if (pTable == NULL) { - mError("dnode:%s, vnode:%d, sid:%d, table not found", taosIpStr(pCfg->dnode), pCfg->vnode, pCfg->sid); - mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_TABLE); - return; - } - - mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); - - SMDCreateTableMsg *pMDCreate = NULL; - if (pTable->type == TSDB_CHILD_TABLE) { - mTrace("table:%s, is a child table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); - pMDCreate = mgmtBuildCreateChildTableMsg(NULL, (SChildTableObj *) pTable); - if (pMDCreate == NULL) { - return; - } - } else if (pTable->type == TSDB_NORMAL_TABLE) { - mTrace("table:%s, is a normal table, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); - pMDCreate = mgmtBuildCreateNormalTableMsg((SNormalTableObj *) pTable); - if (pMDCreate == NULL) { - return; - } - } else { - mError("table:%s, invalid msg type, vgroup:%d sid:%d", pTable->tableId, pCfg->vnode, pCfg->sid); - } - - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); - SRpcMsg rpcRsp = { - .handle = NULL, - .pCont = pMDCreate, - .contLen = htonl(pMDCreate->contLen), - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE - }; - mgmtSendMsgToDnode(&ipSet, &rpcRsp); -} +} \ No newline at end of file diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 576b326669..8c72520563 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -310,9 +310,10 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void return numOfRows; } -SUserObj *mgmtGetUserFromConn(void *pConn) { +SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) { SRpcConnInfo connInfo; if (rpcGetConnInfo(pConn, &connInfo) == 0) { + *usePublicIp = (connInfo.serverIp == tsPublicIpInt); return mgmtGetUser(connInfo.user); } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 9e52574b8c..f0649dab81 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -428,7 +428,7 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo return numOfRows; } -void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { +void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) { pVgroup->tableList[pTable->sid] = pTable; taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); @@ -439,7 +439,7 @@ void mgmtAddTableIntoVgroup(SVgObj *pVgroup, STableInfo *pTable) { mgmtAddVgroupIntoDbTail(pVgroup); } -void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, STableInfo *pTable) { +void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) { pVgroup->tableList[pTable->sid] = NULL; taosFreeId(pVgroup->idPool, pTable->sid); From b6313b346fd414d8ec7b52548125ebded4693570 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 26 Mar 2020 01:01:52 +0800 Subject: [PATCH 46/85] [TD-9] fix compile errors --- src/inc/mnode.h | 3 +- src/mnode/inc/mgmtUser.h | 2 +- src/mnode/src/mgmtChildTable.c | 122 ++++++++++++++++++--------------- src/mnode/src/mgmtDb.c | 6 +- src/mnode/src/mgmtDnode.c | 6 +- src/mnode/src/mgmtMnode.c | 2 +- src/mnode/src/mgmtProfile.c | 6 +- src/mnode/src/mgmtShell.c | 7 +- src/mnode/src/mgmtSuperTable.c | 56 +++++++-------- src/mnode/src/mgmtTable.c | 28 ++++---- src/mnode/src/mgmtUser.c | 2 +- src/mnode/src/mgmtVgroup.c | 5 +- 12 files changed, 127 insertions(+), 118 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 80a5e2b36c..e6cc0f1367 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -154,7 +154,7 @@ typedef struct _vg_obj { struct _db_obj *pDb; int32_t numOfTables; void * idPool; - STableInfo ** tableList; + SChildTableObj ** tableList; } SVgObj; typedef struct _db_obj { @@ -241,6 +241,7 @@ typedef struct { int8_t usePublicIp; int8_t received; int8_t successed; + int8_t expected; int32_t contLen; int32_t code; void *ahandle; diff --git a/src/mnode/inc/mgmtUser.h b/src/mnode/inc/mgmtUser.h index 5001bc2770..465b8e4f96 100644 --- a/src/mnode/inc/mgmtUser.h +++ b/src/mnode/inc/mgmtUser.h @@ -24,7 +24,7 @@ extern "C" { int32_t mgmtInitUsers(); void mgmtCleanUpUsers(); SUserObj *mgmtGetUser(char *name); -SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) +SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 9b4be2fdc2..a2605cefd5 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -21,11 +21,23 @@ #include "ttime.h" #include "tstatus.h" #include "tutil.h" +#include "qast.h" +#include "qextbuffer.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tscompression.h" +#include "tskiplist.h" +#include "tsqlfunction.h" +#include "tstatus.h" +#include "ttime.h" +#include "name.h" #include "mnode.h" #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" #include "mgmtDClient.h" +#include "mgmtDnode.h" +#include "mgmtDServer.h" #include "mgmtGrant.h" #include "mgmtMnode.h" #include "mgmtProfile.h" @@ -34,6 +46,7 @@ #include "mgmtSuperTable.h" #include "mgmtTable.h" #include "mgmtVgroup.h" +#include "mgmtUser.h" static void *tsChildTableSdb; static int32_t tsChildTableUpdateSize; @@ -79,7 +92,7 @@ static int32_t mgmtChildTableActionInsert(SSdbOperDesc *pOper) { if (pTable->info.type == TSDB_CHILD_TABLE) { pTable->superTable = mgmtGetSuperTable(pTable->superTableId); - pStable->numOfTables++; + pTable->superTable->numOfTables++; mgmtAddTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); } else { mgmtAddTimeSeries(pAcct, pTable->numOfColumns - 1); @@ -115,12 +128,12 @@ static int32_t mgmtChildTableActionDelete(SSdbOperDesc *pOper) { if (pTable->info.type == TSDB_CHILD_TABLE) { mgmtRestoreTimeSeries(pAcct, pTable->superTable->numOfColumns - 1); - pStable->numOfTables--; + pTable->superTable->numOfTables--; } else { mgmtRestoreTimeSeries(pAcct, pTable->numOfColumns - 1); } mgmtRemoveTableFromDb(pDb); - mgmtRemoveTableFromVgroup(pVgroup, (STableInfo *) pTable); + mgmtRemoveTableFromVgroup(pVgroup, pTable); return TSDB_CODE_SUCCESS; } @@ -163,14 +176,14 @@ static int32_t mgmtChildTableActionDecode(SSdbOperDesc *pOper) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); if (pTable->schema == NULL) { - mgmtDestroyNormalTable(pTable); + mgmtDestroyChildTable(pTable); return TSDB_CODE_SERV_OUT_OF_MEMORY; } memcpy(pTable->schema, pOper->rowData + tsChildTableUpdateSize, schemaSize); pTable->sql = (char *)malloc(pTable->sqlLen); if (pTable->sql == NULL) { - mgmtDestroyNormalTable(pTable); + mgmtDestroyChildTable(pTable); return TSDB_CODE_SERV_OUT_OF_MEMORY; } memcpy(pTable->sql, pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sqlLen); @@ -191,7 +204,7 @@ int32_t mgmtInitChildTables() { SSdbTableDesc tableDesc = { .tableName = "ctables", .hashSessions = tsMaxTables, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS,, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, .keyType = SDB_KEY_TYPE_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, @@ -319,7 +332,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb } memcpy(pCreate->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN + 1); - memcpy(pCreate->superTableId, pTable->superTable->tableId, TSDB_TABLE_ID_LEN + 1); + memcpy(pCreate->superTableId, pTable->superTable->info.tableId, TSDB_TABLE_ID_LEN + 1); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); pCreate->tableType = pTable->info.type; @@ -381,7 +394,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj pTable->sid = tid; pTable->vgId = pVgroup->vgId; - if (Table->info.type == TSDB_CHILD_TABLE) { + if (pTable->info.type == TSDB_CHILD_TABLE) { char *pTagData = (char *) pCreate->schema; // it is a tag key SSuperTableObj *pSuperTable = mgmtGetSuperTable(pTagData); if (pSuperTable == NULL) { @@ -391,7 +404,7 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj return NULL; } - strcpy(pTable->superTableId, pSuperTable->tableId); + strcpy(pTable->superTableId, pSuperTable->info.tableId); pTable->uid = (((uint64_t) pTable->vgId) << 40) + ((((uint64_t) pTable->sid) & ((1ul << 24) - 1ul)) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->superTable = pSuperTable; @@ -503,7 +516,7 @@ void mgmtCreateChildTable(SQueuedMsg *pMsg) { mgmtSendMsgToDnode(&ipSet, &rpcMsg); } -static int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable) { +void mgmtDropChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable) { SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("ctable:%s, failed to drop child table, vgroup not exist", pTable->info.tableId); @@ -528,14 +541,14 @@ static int32_t mgmtDropChildTable(SQueuedMsg *newMsg, SChildTableObj *pTable) { mTrace("ctable:%s, send drop table msg", pDrop->tableId); SRpcMsg rpcMsg = { - .handle = newMsg, + .handle = pMsg, .pCont = pDrop, .contLen = sizeof(SMDDropTableMsg), .code = 0, .msgType = TSDB_MSG_TYPE_MD_DROP_TABLE }; - newMsg->ahandle = pTable; + pMsg->ahandle = pTable; mgmtSendMsgToDnode(&ipSet, &rpcMsg); } @@ -593,7 +606,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName } -static int32_t mgmtFindNormalTableColumnIndex(SNormalTableObj *pTable, char *colName) { +static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; for (int32_t i = 0; i < pTable->numOfColumns; i++) { if (strcasecmp(schema[i].name, colName) == 0) { @@ -604,7 +617,7 @@ static int32_t mgmtFindNormalTableColumnIndex(SNormalTableObj *pTable, char *col return -1; } -static int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[], int32_t ncols) { +static int32_t mgmtAddNormalTableColumn(SChildTableObj *pTable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { return TSDB_CODE_APP_ERROR; } @@ -644,7 +657,7 @@ static int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[ SSdbOperDesc desc = {0}; desc.type = SDB_OPER_TYPE_GLOBAL; desc.pObj = pTable; - desc.table = tsNormalTableSdb; + desc.table = tsChildTableSdb; desc.rowData = pTable; desc.rowSize = tsChildTableUpdateSize; sdbUpdateRow(&desc); @@ -652,7 +665,7 @@ static int32_t mgmtAddNormalTableColumn(SNormalTableObj *pTable, SSchema schema[ return TSDB_CODE_SUCCESS; } -static int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *colName) { +static int32_t mgmtDropNormalTableColumnByName(SChildTableObj *pTable, char *colName) { int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); if (col < 0) { return TSDB_CODE_APP_ERROR; @@ -681,7 +694,7 @@ static int32_t mgmtDropNormalTableColumnByName(SNormalTableObj *pTable, char *co SSdbOperDesc desc = {0}; desc.type = SDB_OPER_TYPE_GLOBAL; desc.pObj = pTable; - desc.table = tsNormalTableSdb; + desc.table = tsChildTableSdb; desc.rowData = pTable; desc.rowSize = tsChildTableUpdateSize; sdbUpdateRow(&desc); @@ -702,14 +715,7 @@ static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pT return numOfCols * sizeof(SSchema); } -static void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SDbObj *pDb, SChildTableObj *pTable) { - STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); - if (pMeta == NULL) { - mError("table:%s, failed to get table meta, no enough memory", pTable->info.tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); - return; - } - +static int32_t mgmtDoGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STableMetaMsg *pMeta, bool usePublicIp) { pMeta->uid = htobe64(pTable->uid); pMeta->sid = htonl(pTable->sid); pMeta->vgId = htonl(pTable->vgId); @@ -732,12 +738,11 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SDbObj *pDb, SChildTableObj SVgObj *pVgroup = mgmtGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("table:%s, failed to get table meta, db not selected", pTable->info.tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_VGROUP_ID); - return; + return TSDB_CODE_INVALID_VGROUP_ID; } for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - if (pMsg->usePublicIp) { + if (usePublicIp) { pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; } else { pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].privateIp; @@ -746,18 +751,12 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SDbObj *pDb, SChildTableObj } pMeta->numOfVpeers = pVgroup->numOfVnodes; - SRpcMsg rpcRsp = { - .handle = pMsg->thandle, - .pCont = pMeta, - .contLen = pMeta->contLen, - }; - pMeta->contLen = htons(pMeta->contLen); - rpcSendResponse(&rpcRsp); - mTrace("table:%s, uid:%" PRIu64 " table meta is retrieved", pTable->info.tableId, pTable->uid); + + return TSDB_CODE_SUCCESS; } -void mgmtProcessChildTableMetaMsg(SQueuedMsg *pMsg) { +void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SChildTableObj *pTable) { SCMTableInfoMsg *pInfo = pMsg->pCont; SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); if (pDb == NULL || pDb->dirty) { @@ -766,7 +765,6 @@ void mgmtProcessChildTableMetaMsg(SQueuedMsg *pMsg) { return; } - STableInfo *pTable = mgmtGetTable(pInfo->tableId); if (pTable == NULL) { if (htons(pInfo->createFlag) != 1) { mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); @@ -796,7 +794,22 @@ void mgmtProcessChildTableMetaMsg(SQueuedMsg *pMsg) { } } - mgmtGetChildTableMeta(pMsg, pDb, pTable); + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); + if (pMeta == NULL) { + mError("table:%s, failed to get table meta, no enough memory", pTable->info.tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; + } + + mgmtDoGetChildTableMeta(pDb, pTable, pMeta, pMsg->usePublicIp); + + SRpcMsg rpcRsp = { + .handle = pMsg->thandle, + .pCont = pMeta, + .contLen = pMeta->contLen, + }; + pMeta->contLen = htons(pMeta->contLen); + rpcSendResponse(&rpcRsp); } void mgmtDropAllChildTables(SDbObj *pDropDb) { @@ -853,7 +866,7 @@ void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { } } - mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->tableId, numOfTables); + mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } static STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { @@ -908,7 +921,7 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { SQueuedMsg *queueMsg = rpcMsg->handle; queueMsg->received++; - STableInfo *pTable = queueMsg->ahandle; + SChildTableObj *pTable = queueMsg->ahandle; mTrace("table:%s, drop table rsp received, thandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, tstrerror(rpcMsg->code)); if (rpcMsg->code != TSDB_CODE_SUCCESS) { @@ -948,17 +961,13 @@ static void mgmtProcessDropTableRsp(SRpcMsg *rpcMsg) { free(queueMsg); } -static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { - mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); -} - static void mgmtProcessCreateTableRsp(SRpcMsg *rpcMsg) { if (rpcMsg->handle == NULL) return; SQueuedMsg *queueMsg = rpcMsg->handle; queueMsg->received++; - STableInfo *pTable = queueMsg->ahandle; + SChildTableObj *pTable = queueMsg->ahandle; mTrace("table:%s, create table rsp received, thandle:%p ahandle:%p result:%s", pTable->info.tableId, queueMsg->thandle, rpcMsg->handle, tstrerror(rpcMsg->code)); @@ -1026,7 +1035,7 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { for (int t = 0; t < pInfo->numOfTables; ++t) { char *tableId = (char*)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN); - STableInfo *pTable = mgmtGetTable(tableId); + SChildTableObj *pTable = mgmtGetChildTable(tableId); if (pTable == NULL) continue; SDbObj *pDb = mgmtGetDbByTableId(tableId); @@ -1047,7 +1056,7 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { } STableMetaMsg *pMeta = (STableMetaMsg *)(pMultiMeta->metas + pMultiMeta->contLen); - int32_t code = mgmtGetTableMeta(pDb, pTable, pMeta, usePublicIp); + int32_t code = mgmtDoGetChildTableMeta(pDb, pTable, pMeta, usePublicIp); if (code == TSDB_CODE_SUCCESS) { pMultiMeta->numOfTables ++; pMultiMeta->contLen += pMeta->contLen; @@ -1120,7 +1129,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, SDbObj *pDb = mgmtGetDb(pShow->db); if (pDb == NULL) return 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { @@ -1140,18 +1149,17 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, int32_t prefixLen = strlen(prefix); while (numOfRows < rows) { - pShow->pNode = sdbFetchRow(tsNormalTableSdb, pShow->pNode, (void **) &pTable); + pShow->pNode = sdbFetchRow(tsChildTableSdb, pShow->pNode, (void **) &pTable); if (pTable == NULL) break; // not belong to current db - if (strncmp(tableId, prefix, prefixLen)) { + if (strncmp(pTable->info.tableId, prefix, prefixLen)) { continue; } char tableName[TSDB_TABLE_NAME_LEN] = {0}; memset(tableName, 0, tListLen(tableName)); - numOfRead++; - + // pattern compare for meter name mgmtExtractTableName(pTable->info.tableId, tableName); @@ -1188,7 +1196,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, numOfRows++; } - pShow->numOfReads += numOfRead; + pShow->numOfReads += numOfRows; const int32_t NUM_OF_COLUMNS = 4; mgmtVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow); @@ -1201,11 +1209,11 @@ void mgmtAlterChildTable(SQueuedMsg *pMsg, SChildTableObj *pTable) { SCMAlterTableMsg *pAlter = pMsg->pCont;; if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { - code = mgmtModifyChildTableTagValueByName((SChildTableObj *)pTable, pAlter->schema[0].name, pAlter->tagVal); + code = mgmtModifyChildTableTagValueByName(pTable, pAlter->schema[0].name, pAlter->tagVal); } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { - code = mgmtAddNormalTableColumn((SNormalTableObj *)pTable, pAlter->schema, 1); + code = mgmtAddNormalTableColumn(pTable, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { - code = mgmtDropNormalTableColumnByName((SNormalTableObj *)pTable, pAlter->schema[0].name); + code = mgmtDropNormalTableColumnByName(pTable, pAlter->schema[0].name); } else { } diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 0cc793f2c0..16f9751e20 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -28,7 +28,6 @@ #include "mgmtGrant.h" #include "mgmtShell.h" #include "mgmtMnode.h" -#include "mgmtNormalTable.h" #include "mgmtChildTable.h" #include "mgmtSdb.h" #include "mgmtSuperTable.h" @@ -82,7 +81,6 @@ static int32_t mgmtDbActionDelete(SSdbOperDesc *pOper) { SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); mgmtRemoveDbFromAcct(pAcct, pDb); - mgmtDropAllNormalTables(pDb); mgmtDropAllChildTables(pDb); mgmtDropAllSuperTables(pDb); mgmtDropAllVgroups(pDb); @@ -382,7 +380,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; pShow->bytes[cols] = TSDB_DB_NAME_LEN; @@ -532,7 +530,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * SDbObj *pDb = NULL; char * pWrite; int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; while (numOfRows < rows) { diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 0aab083471..291cbe8a48 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -76,7 +76,7 @@ bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; @@ -169,7 +169,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; @@ -256,7 +256,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index 3dfce5e6af..71ce50eccb 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -118,7 +118,7 @@ static void *mgmtGetNextMnode(SShowObj *pShow, SMnodeObj **pMnode) { static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t cols = 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) return 0; if (strcmp(pUser->user, "root") != 0) return TSDB_CODE_NO_RIGHTS; diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 1b98ac9596..44bd45faea 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -678,7 +678,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; if (mgmtCheckRedirect(pMsg->thandle)) return; - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; rpcSendResponse(&rpcRsp); @@ -702,7 +702,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; if (mgmtCheckRedirect(pMsg->thandle)) return; - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; rpcSendResponse(&rpcRsp); @@ -726,7 +726,7 @@ void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; if (mgmtCheckRedirect(pMsg->thandle)) return; - SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle); + SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL); if (pUser == NULL) { rpcRsp.code = TSDB_CODE_INVALID_USER; rpcSendResponse(&rpcRsp); diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index aa7a494a9f..b092445e72 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -30,7 +30,6 @@ #include "mgmtDnode.h" #include "mgmtGrant.h" #include "mgmtMnode.h" -#include "mgmtNormalTable.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -139,7 +138,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { } if (mgmtCheckExpired()) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_GRANT_EXPIRED); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_GRANT_EXPIRED); return; } @@ -174,7 +173,7 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { queuedMsg->contLen = rpcMsg->contLen; queuedMsg->pCont = rpcMsg->pCont; queuedMsg->pUser = pUser; - queuedMsg.usePublicIp = usePublicIp; + queuedMsg->usePublicIp = usePublicIp; mgmtAddToShellQueue(queuedMsg); } } @@ -448,7 +447,7 @@ static bool mgmtCheckMeterMetaMsgType(void *pMsg) { static bool mgmtCheckMsgReadOnly(int8_t type, void *pCont) { if ((type == TSDB_MSG_TYPE_CM_TABLE_META && (!mgmtCheckMeterMetaMsgType(pCont))) || - type == TSDB_MSG_TYPE_CM_STABLE_META || type == TSDB_MSG_TYPE_RETRIEVE || + type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_RETRIEVE || type == TSDB_MSG_TYPE_CM_SHOW || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_CONNECT) { return true; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index 4b8a4aba76..e340643998 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -20,6 +20,7 @@ #include "mgmtAcct.h" #include "mgmtChildTable.h" #include "mgmtDb.h" +#include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtGrant.h" #include "mgmtShell.h" @@ -47,8 +48,8 @@ static int32_t mgmtSuperTableActionDestroy(SSdbOperDesc *pOper) { } static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { - STableInfo *pStable = pOper->pObj; - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SSuperTableObj *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { mgmtAddSuperTableIntoDb(pDb); } @@ -56,8 +57,8 @@ static int32_t mgmtSuperTableActionInsert(SSdbOperDesc *pOper) { } static int32_t mgmtSuperTableActionDelete(SSdbOperDesc *pOper) { - STableInfo *pStable = pOper->pObj; - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SSuperTableObj *pStable = pOper->pObj; + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { mgmtRemoveSuperTableFromDb(pDb); mgmtDropAllChildTablesInStable((SSuperTableObj *)pStable); @@ -151,11 +152,9 @@ void mgmtCreateSuperTable(SQueuedMsg *pMsg) { return; } - strcpy(pStable->tableId, pCreate->tableId); - pStable->type = TSDB_SUPER_TABLE; + strcpy(pStable->info.tableId, pCreate->tableId); + pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); - pStable->vgId = 0; - pStable->sid = 0; pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pStable->sversion = 0; pStable->numOfColumns = htons(pCreate->numOfColumns); @@ -191,14 +190,14 @@ void mgmtCreateSuperTable(SQueuedMsg *pMsg) { mgmtDestroySuperTable(pStable); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SDB_ERROR); } else { - mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->tableId, pStable->numOfTags, pStable->numOfColumns); + mLPrint("stable:%s, is created, tags:%d cols:%d", pStable->info.tableId, pStable->numOfTags, pStable->numOfColumns); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); } } -void mgmtDropSuperTable(SQueuedMsg *newMsg, SSuperTableObj *pTable) { +void mgmtDropSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pStable) { if (pStable->numOfTables != 0) { - mError("stable:%s, numOfTables:%d not 0", pStable->tableId, pStable->numOfTables); + mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { SSdbOperDesc oper = { @@ -207,7 +206,7 @@ void mgmtDropSuperTable(SQueuedMsg *newMsg, SSuperTableObj *pTable) { .pObj = pStable }; int32_t code = sdbDeleteRow(&oper); - mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->tableId, tstrerror(code)); + mLPrint("stable:%s, is dropped from sdb, result:%s", pStable->info.tableId, tstrerror(code)); mgmtSendSimpleResp(pMsg->thandle, code); } } @@ -248,9 +247,9 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i } } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("meter: %s not belongs to any database", pStable->tableId); + mError("meter: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -278,7 +277,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i pAcct->acctInfo.numOfTimeSeries += (ntags * pStable->numOfTables); // sdbUpdateRow(tsSuperTableSdb, pStable, tsSuperTableUpdateSize, SDB_OPER_GLOBAL); - mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->tableId); + mTrace("Succeed to add tag column %s to table %s", schema[0].name, pStable->info.tableId); return TSDB_CODE_SUCCESS; } @@ -288,9 +287,9 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { return TSDB_CODE_APP_ERROR; } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("table: %s not belongs to any database", pStable->tableId); + mError("table: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -318,11 +317,11 @@ static int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char * int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { // Tag name does not exist - mError("Failed to modify table %s tag column, oname: %s, nname: %s", pStable->tableId, oldTagName, newTagName); + mError("Failed to modify table %s tag column, oname: %s, nname: %s", pStable->info.tableId, oldTagName, newTagName); return TSDB_CODE_INVALID_MSG_TYPE; } - int32_t rowSize = 0; + // int32_t rowSize = 0; uint32_t len = strlen(newTagName); if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { @@ -346,11 +345,11 @@ static int32_t mgmtModifySuperTableTagNameByName(SSuperTableObj *pStable, char * tfree(msg); if (ret < 0) { - mError("Failed to modify table %s tag column", pStable->tableId); + mError("Failed to modify table %s tag column", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } - mTrace("Succeed to modify table %s tag column", pStable->tableId); + mTrace("Succeed to modify table %s tag column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } @@ -376,9 +375,9 @@ static int32_t mgmtAddSuperTableColumn(SSuperTableObj *pStable, SSchema schema[] } } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("meter: %s not belongs to any database", pStable->tableId); + mError("meter: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -415,9 +414,9 @@ static int32_t mgmtDropSuperTableColumnByName(SSuperTableObj *pStable, char *col return TSDB_CODE_APP_ERROR; } - SDbObj *pDb = mgmtGetDbByTableId(pStable->tableId); + SDbObj *pDb = mgmtGetDbByTableId(pStable->info.tableId); if (pDb == NULL) { - mError("table: %s not belongs to any database", pStable->tableId); + mError("table: %s not belongs to any database", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } @@ -504,7 +503,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v SDbObj *pDb = mgmtGetDb(pShow->db); if (pDb == NULL) return 0; - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { if (strcmp(pUser->user, "root") != 0 && strcmp(pUser->user, "_root") != 0 && strcmp(pUser->user, "monitor") != 0 ) { @@ -605,7 +604,6 @@ int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { } void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) { - SCMTableInfoMsg *pInfo = pMsg->pCont; SDbObj *pDb = pMsg->pDb; STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); @@ -667,4 +665,8 @@ void mgmtAlterSuperTable(SQueuedMsg *pMsg, SSuperTableObj *pTable) { } else {} mgmtSendSimpleResp(pMsg->thandle, code); +} + +static void mgmtProcessDropStableRsp(SRpcMsg *rpcMsg) { + mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); } \ No newline at end of file diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 0d02e5f8e1..3e47a7c277 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -73,7 +73,7 @@ void mgmtCleanUpTables() { mgmtCleanUpSuperTables(); } -void mgmtExtractTableName(const char* tableId, char* name) { +void mgmtExtractTableName(char* tableId, char* name) { int pos = -1; int num = 0; for (pos = 0; tableId[pos] != 0; ++pos) { @@ -139,14 +139,14 @@ static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { return; } - pMsg->pDb = mgmtGetDb(pCreate->db); + pMsg->pDb = mgmtGetDbByTableId(pDrop->tableId); if (pMsg->pDb == NULL || pMsg->pDb->dirty) { mError("table:%s, failed to drop table, db not selected", pDrop->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { + if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("table:%s, failed to drop table, in monitor database", pDrop->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN); return; @@ -170,7 +170,7 @@ static void mgmtProcessDropTableMsg(SQueuedMsg *pMsg) { mgmtDropSuperTable(pMsg, (SSuperTableObj *)pTable); } else { mTrace("table:%s, start to drop ctable", pDrop->tableId); - mgmtDropChildTable(pMsg, (SNormalTableObj *)pTable); + mgmtDropChildTable(pMsg, (SChildTableObj *)pTable); } } @@ -186,15 +186,15 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { return; } - pMsg->pDb = mgmtGetDbByTableId(pTable->tableId); + pMsg->pDb = mgmtGetDbByTableId(pAlter->tableId); if (pMsg->pDb == NULL || pMsg->pDb->dirty) { - mError("table:%s, failed to alter table, db not selected", pTable->tableId); + mError("table:%s, failed to alter table, db not selected", pAlter->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } - if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) { - mError("table:%s, failed to alter table, its log db", pTable->tableId); + if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { + mError("table:%s, failed to alter table, its log db", pAlter->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN); return; } @@ -218,22 +218,22 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { } if (pTable->type == TSDB_SUPER_TABLE) { - mTrace("table:%s, start to alter stable", pDrop->tableId); + mTrace("table:%s, start to alter stable", pAlter->tableId); mgmtAlterSuperTable(pMsg, (SSuperTableObj *)pTable); } else { - mTrace("table:%s, start to alter ctable", pDrop->tableId); - mgmtAlterChildTable(pMsg, (SNormalTableObj *)pTable); + mTrace("table:%s, start to alter ctable", pAlter->tableId); + mgmtAlterChildTable(pMsg, (SChildTableObj *)pTable); } } static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { - SCMTableInfoMsg *pInfo = mgmtGetTable(pInfo->tableId); + SCMTableInfoMsg *pInfo = pMsg->pCont; mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle); STableInfo *pTable = mgmtGetTable(pInfo->tableId); if (pTable == NULL || pTable->type != TSDB_SUPER_TABLE) { - mgmtGetChildTableMeta(pMsg, (SSuperTableObj *)pTable); + mgmtGetChildTableMeta(pMsg, (SChildTableObj *)pTable); } else { - mgmtGetSuperTableMeta(pMsg, (SNormalTableObj *)pTable); + mgmtGetSuperTableMeta(pMsg, (SSuperTableObj *)pTable); } } \ No newline at end of file diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 8c72520563..c69035c304 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -231,7 +231,7 @@ static int32_t mgmtDropUser(SAcctObj *pAcct, char *name) { } static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - SUserObj *pUser = mgmtGetUserFromConn(pConn); + SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL); if (pUser == NULL) { return TSDB_CODE_INVALID_USER; } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index f0649dab81..deb1da0025 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -20,6 +20,7 @@ #include "tstatus.h" #include "mnode.h" #include "mgmtBalance.h" +#include "mgmtChildTable.h" #include "mgmtDb.h" #include "mgmtDClient.h" #include "mgmtDnode.h" @@ -276,9 +277,9 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t maxReplica = 0; SVgObj *pVgroup = NULL; - STableInfo *pTable = NULL; + SChildTableObj *pTable = NULL; if (pShow->payloadLen > 0 ) { - pTable = mgmtGetTable(pShow->payload); + pTable = mgmtGetChildTable(pShow->payload); if (NULL == pTable) { return TSDB_CODE_INVALID_TABLE_ID; } From 66de5c58c9497581a02a1e54a0278856aaf10eac Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 26 Mar 2020 10:09:19 +0800 Subject: [PATCH 47/85] [TD-9] fix crash while create table --- src/client/src/tscServer.c | 2 +- src/inc/taosmsg.h | 4 ++-- src/mnode/inc/mgmtDb.h | 13 +++++++------ src/mnode/inc/mgmtMnode.h | 12 ++++++------ src/mnode/inc/mgmtProfile.h | 8 ++++---- src/mnode/inc/mgmtSdb.h | 10 +++++----- src/mnode/inc/mgmtShell.h | 4 ++-- src/mnode/inc/mgmtVgroup.h | 2 -- src/mnode/src/mgmtChildTable.c | 2 +- src/mnode/src/mgmtDb.c | 1 - src/mnode/src/mgmtUser.c | 16 +++++++++------- src/mnode/src/mgmtVgroup.c | 23 ++++++++++------------- src/rpc/src/rpcMain.c | 2 +- 13 files changed, 48 insertions(+), 51 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 348c0709f1..7856856606 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1752,7 +1752,7 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { msgLen = pMsg - pStart; pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_META; + pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; assert(msgLen + minMsgSize() <= size); return TSDB_CODE_SUCCESS; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f1e07c98a8..49e498d22f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -738,8 +738,8 @@ typedef struct { } SDMConfigTableMsg; typedef struct { - uint32_t dnode; - int32_t vnode; + uint32_t dnodeId; + int32_t vgId; } SDMConfigVnodeMsg; typedef struct { diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index f2049f836c..27918b9cf9 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -22,22 +22,23 @@ extern "C" { #include "mnode.h" -void mgmtAddVgroupIntoDb(SVgObj *pVgroup); -void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup); -void mgmtRemoveVgroupFromDb(SVgObj *pVgroup); -void mgmtMoveVgroupToTail(SVgObj *pVgroup); -void mgmtMoveVgroupToHead(SVgObj *pVgroup); - +// api int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); SDbObj *mgmtGetDb(char *db); SDbObj *mgmtGetDbByTableId(char *db); bool mgmtCheckIsMonitorDB(char *db, char *monitordb); +// util func void mgmtAddSuperTableIntoDb(SDbObj *pDb); void mgmtRemoveSuperTableFromDb(SDbObj *pDb); void mgmtAddTableIntoDb(SDbObj *pDb); void mgmtRemoveTableFromDb(SDbObj *pDb); +void mgmtAddVgroupIntoDb(SVgObj *pVgroup); +void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup); +void mgmtRemoveVgroupFromDb(SVgObj *pVgroup); +void mgmtMoveVgroupToTail(SVgObj *pVgroup); +void mgmtMoveVgroupToHead(SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index 729d31544f..8cfe3c7e23 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -21,14 +21,14 @@ extern "C" { #endif int32_t mgmtInitMnodes(); -void mgmtCleanupMnodes(); +void mgmtCleanupMnodes(); -bool mgmtInServerStatus(); -bool mgmtIsMaster(); +bool mgmtInServerStatus(); +bool mgmtIsMaster(); -bool mgmtCheckRedirect(void *handle); -void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); -void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); +bool mgmtCheckRedirect(void *handle); +void mgmtGetMnodePrivateIpList(SRpcIpSet *ipSet); +void mgmtGetMnodePublicIpList(SRpcIpSet *ipSet); int32_t mgmtAddMnode(uint32_t privateIp, uint32_t publicIp); int32_t mgmtRemoveMnode(uint32_t privateIp); diff --git a/src/mnode/inc/mgmtProfile.h b/src/mnode/inc/mgmtProfile.h index 8c2e073c2d..a40f572f8b 100644 --- a/src/mnode/inc/mgmtProfile.h +++ b/src/mnode/inc/mgmtProfile.h @@ -24,11 +24,11 @@ extern "C" { int32_t mgmtInitProfile(); void mgmtCleanUpProfile(); -bool mgmtCheckQhandle(uint64_t qhandle); -void mgmtSaveQhandle(void *qhandle); -void mgmtFreeQhandle(void *qhandle); +bool mgmtCheckQhandle(uint64_t qhandle); +void mgmtSaveQhandle(void *qhandle); +void mgmtFreeQhandle(void *qhandle); -void mgmtFreeQueuedMsg(SQueuedMsg *pMsg); +void mgmtFreeQueuedMsg(SQueuedMsg *pMsg); #ifdef __cplusplus } diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index ccddc11e04..f291e93418 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -53,16 +53,16 @@ typedef struct { int32_t (*destroyFp)(SSdbOperDesc *pDesc); } SSdbTableDesc; -void *sdbOpenTable(SSdbTableDesc *desc); -void sdbCloseTable(void *handle); +void * sdbOpenTable(SSdbTableDesc *desc); +void sdbCloseTable(void *handle); int32_t sdbInsertRow(SSdbOperDesc *pOper); int32_t sdbDeleteRow(SSdbOperDesc *pOper); int32_t sdbUpdateRow(SSdbOperDesc *pOper); -void *sdbGetRow(void *handle, void *key); -void *sdbFetchRow(void *handle, void *pNode, void **ppRow); -int64_t sdbGetNumOfRows(void *handle); +void *sdbGetRow(void *handle, void *key); +void *sdbFetchRow(void *handle, void *pNode, void **ppRow); +int64_t sdbGetNumOfRows(void *handle); uint64_t sdbGetVersion(); #ifdef __cplusplus diff --git a/src/mnode/inc/mgmtShell.h b/src/mnode/inc/mgmtShell.h index 56625b982a..b92e9de1f4 100644 --- a/src/mnode/inc/mgmtShell.h +++ b/src/mnode/inc/mgmtShell.h @@ -22,8 +22,8 @@ extern "C" { #include "mnode.h" int32_t mgmtInitShell(); -void mgmtCleanUpShell(); -void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg)); +void mgmtCleanUpShell(); +void mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg)); typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index aba95fb478..de0648da44 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -27,12 +27,10 @@ extern "C" { int32_t mgmtInitVgroups(); void mgmtCleanUpVgroups(); SVgObj *mgmtGetVgroup(int32_t vgId); -SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode); void mgmtDropAllVgroups(SDbObj *pDropDb); void mgmtCreateVgroup(SQueuedMsg *pMsg); void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle); -void mgmtUpdateVgroup(SVgObj *pVgroup); void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle); SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb); diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index a2605cefd5..2d4f37581d 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -332,7 +332,6 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb } memcpy(pCreate->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN + 1); - memcpy(pCreate->superTableId, pTable->superTable->info.tableId, TSDB_TABLE_ID_LEN + 1); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); pCreate->tableType = pTable->info.type; @@ -342,6 +341,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb pCreate->uid = htobe64(pTable->uid); if (pTable->info.type == TSDB_CHILD_TABLE) { + memcpy(pCreate->superTableId, pTable->superTable->info.tableId, TSDB_TABLE_ID_LEN + 1); pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); pCreate->numOfTags = htons(pTable->superTable->numOfTags); pCreate->sversion = htonl(pTable->superTable->sversion); diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 16f9751e20..8c8173f781 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -41,7 +41,6 @@ static int32_t tsDbUpdateSize; static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate); static void mgmtDropDb(void *handle, void *tmrId); static int32_t mgmtSetDbDirty(SDbObj *pDb); - static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg); diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index c69035c304..dc2418efd9 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -25,14 +25,14 @@ #include "mgmtShell.h" #include "mgmtUser.h" -void *tsUserSdb = NULL; +static void *tsUserSdb = NULL; static int32_t tsUserUpdateSize = 0; -static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass); -static int32_t mgmtDropUser(SAcctObj *pAcct, char *name); -static int32_t mgmtUpdateUser(SUserObj *pUser); -static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static int32_t mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass); +static int32_t mgmtDropUser(SAcctObj *pAcct, char *name); +static int32_t mgmtUpdateUser(SUserObj *pUser); +static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); @@ -313,7 +313,9 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) { SRpcConnInfo connInfo; if (rpcGetConnInfo(pConn, &connInfo) == 0) { - *usePublicIp = (connInfo.serverIp == tsPublicIpInt); + if (usePublicIp) { + *usePublicIp = (connInfo.serverIp == tsPublicIpInt); + } return mgmtGetUser(connInfo.user); } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index deb1da0025..a8e7219e2e 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -67,8 +67,8 @@ static int32_t mgmtVgroupActionInsert(SSdbOperDesc *pOper) { pVgroup->prev = NULL; pVgroup->next = NULL; - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)calloc(pDb->cfg.maxSessions, sizeof(STableInfo *)); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; + pVgroup->tableList = calloc(pDb->cfg.maxSessions, sizeof(SChildTableObj *)); if (pVgroup->tableList == NULL) { mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); return -1; @@ -112,8 +112,8 @@ static int32_t mgmtVgroupActionUpdate(SSdbOperDesc *pOper) { if (pDb->cfg.maxSessions != oldTables) { mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(STableInfo *) * pDb->cfg.maxSessions; - pVgroup->tableList = (STableInfo **)realloc(pVgroup->tableList, size); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; + pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size); } } @@ -483,17 +483,14 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { return pVnode; } -SVgObj *mgmtGetVgroupByVnode(uint32_t dnode, int32_t vnode) { - if (vnode < 0 || vnode >= TSDB_MAX_VNODES) { - return NULL; - } +static SVgObj *mgmtGetVgroupInDnode(uint32_t dnodeId, int32_t vgId) { + if (vnovgId < 1 || dnodeId < 1) return NULL; - SDnodeObj *pDnode = mgmtGetDnode(dnode); + SDnodeObj *pDnode = mgmtGetDnode(dnodeId); if (pDnode == NULL) { return NULL; } - int32_t vgId = pDnode->vload[vnode].vgId; return mgmtGetVgroup(vgId); } @@ -660,10 +657,10 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { if (mgmtCheckRedirect(rpcMsg->handle)) return; SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; - pCfg->dnode = htonl(pCfg->dnode); - pCfg->vnode = htonl(pCfg->vnode); + pCfg->dnodeId = htonl(pCfg->dnode); + pCfg->vgId = htonl(pCfg->vnode); - SVgObj *pVgroup = mgmtGetVgroupByVnode(pCfg->dnode, pCfg->vnode); + SVgObj *pVgroup = mgmtGetVgroupInDnode(pCfg->dnodeId, pCfg->vgId); if (pVgroup == NULL) { mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index c6f527a7d2..93334333ef 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -360,7 +360,7 @@ void rpcSendRequest(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg) { // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_RETRIEVE || - type == TSDB_MSG_TYPE_CM_STABLE_META || type == TSDB_MSG_TYPE_CM_TABLES_META || + type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; From c0e31d6c235f4cb8cf3c1edeeb228f016b3fc6de Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 26 Mar 2020 10:34:56 +0800 Subject: [PATCH 48/85] [td-32] fix bugs in projection query --- src/dnode/src/dnodeRead.c | 58 ++++--- src/query/inc/queryExecutor.h | 28 ++-- src/query/src/queryExecutor.c | 275 +++++++++++++++------------------- src/vnode/tsdb/src/tsdbRead.c | 10 +- 4 files changed, 181 insertions(+), 190 deletions(-) diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index 3419128c72..0ae093728f 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -226,47 +226,59 @@ static void dnodeProcessReadResult(SReadMsg *pRead) { rpcFreeCont(pRead->rpcMsg.pCont); // free the received message } +static void dnodeContinueExecuteQuery(void* qhandle, SReadMsg *pMsg) { + SReadMsg readMsg = { + .rpcMsg = {.msgType = TSDB_MSG_TYPE_QUERY}, + .pCont = qhandle, + .contLen = 0, + .pRpcContext = pMsg->pRpcContext, + .pVnode = pMsg->pVnode, + }; + + taos_queue queue = dnodeGetVnodeRworker(pMsg->pVnode); + taosWriteQitem(queue, &readMsg); +} + static void dnodeProcessQueryMsg(SReadMsg *pMsg) { SQueryTableMsg* pQueryTableMsg = (SQueryTableMsg*) pMsg->pCont; SQInfo* pQInfo = NULL; - void* tsdb = dnodeGetVnodeTsdb(pMsg->pVnode); - int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); + if (pMsg->rpcMsg.contLen != 0) { + void* tsdb = dnodeGetVnodeTsdb(pMsg->pVnode); + int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, pMsg, &pQInfo); - SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); - pRsp->code = code; - pRsp->qhandle = htobe64((uint64_t) (pQInfo)); - - SRpcMsg rpcRsp = { - .handle = pMsg->rpcMsg.handle, - .pCont = pRsp, - .contLen = sizeof(SQueryTableRsp), - .code = code, - .msgType = 0 - }; + SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); + pRsp->code = code; + pRsp->qhandle = htobe64((uint64_t) (pQInfo)); - rpcSendResponse(&rpcRsp); + SRpcMsg rpcRsp = { + .handle = pMsg->rpcMsg.handle, + .pCont = pRsp, + .contLen = sizeof(SQueryTableRsp), + .code = code, + .msgType = 0 + }; + + rpcSendResponse(&rpcRsp); + } else { + pQInfo = pMsg->pCont; + } // do execute query qTableQuery(pQInfo); } -static int32_t c = 0; static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { SRetrieveTableMsg *pRetrieve = pMsg->pCont; void *pQInfo = (void*) htobe64(pRetrieve->qhandle); dTrace("QInfo:%p vgId:%d, retrieve msg is received", pQInfo, pRetrieve->header.vgId); - if ((++c)%2 == 0) { - int32_t k = 1; - } - int32_t rowSize = 0; - int32_t numOfRows = 0; + int32_t contLen = 0; SRetrieveTableRsp *pRsp = NULL; - int32_t code = qRetrieveQueryResultInfo(pQInfo, &numOfRows, &rowSize); + int32_t code = qRetrieveQueryResultInfo(pQInfo); if (code != TSDB_CODE_SUCCESS) { contLen = sizeof(SRetrieveTableRsp); @@ -275,6 +287,10 @@ static void dnodeProcessRetrieveMsg(SReadMsg *pMsg) { } else { // todo check code and handle error in build result set code = qDumpRetrieveResult(pQInfo, &pRsp, &contLen); + + if (qNeedFurtherExec(pQInfo)) { + dnodeContinueExecuteQuery(pQInfo, pMsg); + } } SRpcMsg rpcRsp = (SRpcMsg) { diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 4ce606f599..c30d47e261 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -68,8 +68,10 @@ typedef struct SWindowResult { } SWindowResult; typedef struct SResultRec { - int64_t pointsTotal; - int64_t pointsRead; + int64_t total; + int64_t size; + int64_t capacity; + int32_t threshold; // the threshold size, when the number of rows in result buffer, return to client } SResultRec; typedef struct SWindowResInfo { @@ -112,7 +114,7 @@ typedef struct STableQueryInfo { typedef struct STableDataInfo { int32_t numOfBlocks; - int32_t start; // start block index + int32_t start; // start block index int32_t tableIndex; void* pMeterObj; int32_t groupIdx; // group id in table list @@ -143,7 +145,6 @@ typedef struct SQuery { int32_t pos; int64_t pointsOffset; // the number of points offset to save read data SData** sdata; - int32_t capacity; SSingleColumnFilterInfo* pFilterInfo; } SQuery; @@ -171,15 +172,13 @@ typedef struct SQueryRuntimeEnv { typedef struct SQInfo { void* signature; - void* pVnode; + void* param; // pointer to the RpcReadMsg TSKEY startTime; TSKEY elapsedTime; - SResultRec rec; int32_t pointsInterpo; - int32_t code; // error code to returned to client -// int32_t killed; // denotes if current query is killed + int32_t code; // error code to returned to client sem_t dataReady; - SArray* pTableIdList; // table list + SArray* pTableIdList; // table id list SQueryRuntimeEnv runtimeEnv; int32_t subgroupIdx; int32_t offset; /* offset in group result set of subgroup */ @@ -204,7 +203,7 @@ typedef struct SQInfo { * @param pQInfo * @return */ -int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); +int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, void* param, SQInfo** pQInfo); /** * query on single table @@ -222,7 +221,7 @@ void qSuperTableQuery(void* pReadMsg); * wait for the query completed, and retrieve final results to client * @param pQInfo */ -int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* rowsize); +int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo); /** * @@ -232,4 +231,11 @@ int32_t qRetrieveQueryResultInfo(SQInfo* pQInfo, int32_t *numOfRows, int32_t* ro */ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen); +/** + * + * @param pQInfo + * @return + */ +bool qNeedFurtherExec(SQInfo* pQInfo); + #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index a996c8467f..a4da2073ea 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -364,8 +364,8 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functio bool doRevisedResultsByLimit(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - if ((pQuery->limit.limit > 0) && (pQuery->rec.pointsRead + pQInfo->rec.pointsRead > pQuery->limit.limit)) { - pQuery->rec.pointsRead = pQuery->limit.limit - pQInfo->rec.pointsRead; + if ((pQuery->limit.limit > 0) && (pQuery->rec.size + pQuery->rec.size > pQuery->limit.limit)) { + pQuery->rec.size = pQuery->limit.limit - pQuery->rec.size; // query completed setQueryStatus(pQuery, QUERY_COMPLETED); @@ -1344,17 +1344,16 @@ static int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forward static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, int32_t *numOfRes, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { - SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + SQuery *pQuery = pRuntimeEnv->pQuery; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { *numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { *numOfRes = blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pQuery->lastKey = lastKey + step; + pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1368,12 +1367,8 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl assert(*numOfRes >= 0); // check if buffer is large enough for accommodating all qualified points - if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1) { - pQuery->pointsOffset -= *numOfRes; - if (pQuery->pointsOffset <= 0) { // todo return correct numOfRes for ts_comp function - pQuery->pointsOffset = 0; - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - } + if (*numOfRes > 0 && pQuery->checkBufferInLoop == 1 && ((*numOfRes) >= pQuery->rec.threshold)) { + setQueryStatus(pQuery, QUERY_RESBUF_FULL); } return 0; @@ -2302,7 +2297,7 @@ int32_t vnodeSTableQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param) { pQuery->status = 0; - pQInfo->rec = (SResultRec){0}; + pQuery->rec = (SResultRec){0}; pQuery->rec = (SResultRec){0}; changeExecuteScanOrder(pQuery, true); @@ -2668,9 +2663,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } int32_t numOfRes = 0; - SDataStatis *pStatis = NULL; - SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + + SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); @@ -3035,9 +3030,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { offset += pData->numOfElems; } - assert(pQuery->rec.pointsRead == 0); + assert(pQuery->rec.size == 0); - pQuery->rec.pointsRead += rows; + pQuery->rec.size += rows; pQInfo->offset += 1; } @@ -3367,7 +3362,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3414,14 +3409,14 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQuery->rec.pointsRead == 0 || pQuery->limit.offset == 0) { + if (pQuery->rec.size == 0 || pQuery->limit.offset == 0) { return; } - if (pQuery->rec.pointsRead <= pQuery->limit.offset) { - pQuery->limit.offset -= pQuery->rec.pointsRead; + if (pQuery->rec.size <= pQuery->limit.offset) { + pQuery->limit.offset -= pQuery->rec.size; - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; // pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer resetCtxOutputBuf(pRuntimeEnv); @@ -3430,13 +3425,13 @@ void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->status &= (~QUERY_RESBUF_FULL); } else { int32_t numOfSkip = (int32_t)pQuery->limit.offset; - pQuery->rec.pointsRead -= numOfSkip; + pQuery->rec.size -= numOfSkip; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; assert(0); - // memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->pointsRead * bytes); + // memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->size * bytes); pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -3999,8 +3994,9 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSQL_SO_ASC; int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); - pQuery->rec.pointsRead += numOfResult; - // assert(pQuery->rec.pointsRead <= pQuery->pointsToRead); + pQuery->rec.size += numOfResult; + + assert(pQuery->rec.size <= pQuery->rec.capacity); } static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo) { @@ -4038,31 +4034,6 @@ void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo updatelastkey(pQuery, pTableQueryInfo); } -// we need to split the refstatsult into different packages. -int32_t vnodeGetResultSize(void *thandle, int32_t *numOfRows) { - SQInfo *pQInfo = (SQInfo *)thandle; - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; - - /* - * get the file size and set the numOfRows to be the file size, since for tsComp query, - * the returned row size is equalled to 1 - * - * TODO handle the case that the file is too large to send back one time - */ - if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fstat; - if (stat(pQuery->sdata[0]->data, &fstat) == 0) { - *numOfRows = fstat.st_size; - return fstat.st_size; - } else { - dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); - return 0; - } - } else { - return pQuery->rowSize * (*numOfRows); - } -} - bool vnodeHasRemainResults(void *handle) { SQInfo *pQInfo = (SQInfo *)handle; @@ -4074,7 +4045,7 @@ bool vnodeHasRemainResults(void *handle) { SQuery * pQuery = pRuntimeEnv->pQuery; SInterpolationInfo *pInterpoInfo = &pRuntimeEnv->interpoInfo; - if (pQuery->limit.limit > 0 && pQInfo->rec.pointsRead >= pQuery->limit.limit) { + if (pQuery->limit.limit > 0 && pQuery->rec.size >= pQuery->limit.limit) { return false; } @@ -4147,6 +4118,11 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; } + + // all data returned, set query over + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + setQueryStatus(pQuery, QUERY_OVER); + } } int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage **pDataSrc, int32_t numOfRows, @@ -4255,8 +4231,6 @@ int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); - pQInfo->rec = (SResultRec){0}; - // dataInCache requires lastKey value pQuery->lastKey = pQuery->window.skey; @@ -4535,7 +4509,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start // accumulate the point interpolation result if (numOfRes > 0) { - pQuery->rec.pointsRead += numOfRes; + pQuery->rec.size += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } @@ -4623,7 +4597,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pSupporter->subgroupIdx++; // output buffer is full, return to client - if (pQuery->pointsRead >= pQuery->pointsToRead) { + if (pQuery->size >= pQuery->pointsToRead) { break; } } @@ -4639,9 +4613,9 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { */ if (pSupporter->subgroupIdx > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); - pQInfo->pointsRead += pQuery->pointsRead; + pQInfo->size += pQuery->size; - if (pQuery->pointsRead > 0) { + if (pQuery->size > 0) { return; } } @@ -4707,7 +4681,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { vnodeScanAllData(pRuntimeEnv); - pQuery->pointsRead = getNumOfResult(pRuntimeEnv); + pQuery->size = getNumOfResult(pRuntimeEnv); doSkipResults(pRuntimeEnv); // the limitation of output result is reached, set the query completed @@ -4742,7 +4716,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { pQuery->skey = pQuery->lastKey; // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter - if (pQuery->pointsRead == 0) { + if (pQuery->size == 0) { assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); continue; } else { @@ -4789,17 +4763,17 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { } pQInfo->pTableQuerySupporter->subgroupIdx = 0; - pQuery->pointsRead = 0; + pQuery->size = 0; copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); } - pQInfo->pointsRead += pQuery->pointsRead; + pQInfo->size += pQuery->size; pQuery->pointsOffset = pQuery->pointsToRead; dTrace( "QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d," "next skey:%" PRId64 ", offset:%" PRId64, - pQInfo, vid, pTableIdList->numOfTables, pSupporter->meterIdx, pTableIdList->numOfSubSet, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo, vid, pTableIdList->numOfTables, pSupporter->meterIdx, pTableIdList->numOfSubSet, pQuery->size, pQInfo->size, pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset); #endif } @@ -4911,13 +4885,13 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; - if (pQuery->rec.pointsRead == 0) { + if (pQuery->rec.size == 0) { // vnodePrintQueryStatistics(pSupporter); } - dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.pointsRead, pQInfo->rec.pointsTotal); + dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.size, pQuery->rec.total); return; } #if 0 @@ -4970,8 +4944,8 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { } // handle the limitation of output buffer - pQInfo->pointsRead += pQuery->pointsRead; - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, + pQInfo->size += pQuery->size; + dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->size, pQInfo->size, pQInfo->pointsReturned); #endif } @@ -4994,8 +4968,8 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { } // since the numOfOutputElems must be identical for all sql functions that are allowed to be executed simutanelously. - pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); - // assert(pQuery->pointsRead <= pQuery->pointsToRead && + pQuery->rec.size = getNumOfResult(pRuntimeEnv); + // assert(pQuery->size <= pQuery->pointsToRead && // Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED)); // must be top/bottom query if offset > 0 @@ -5006,7 +4980,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { doSkipResults(pRuntimeEnv); doRevisedResultsByLimit(pQInfo); - pQInfo->rec.pointsRead = pQuery->rec.pointsRead; + pQuery->rec.size = pQuery->rec.size; } static void tableMultiOutputProcessor(SQInfo *pQInfo) { @@ -5026,16 +5000,16 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { return; } - pQuery->rec.pointsRead = getNumOfResult(pRuntimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.pointsRead > 0) { + pQuery->rec.size = getNumOfResult(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.size > 0) { doSkipResults(pRuntimeEnv); } /* - * 1. if pQuery->pointsRead == 0, pQuery->limit.offset >= 0, still need to check data - * 2. if pQuery->pointsRead > 0, pQuery->limit.offset must be 0 + * 1. if pQuery->size == 0, pQuery->limit.offset >= 0, still need to check data + * 2. if pQuery->size > 0, pQuery->limit.offset must be 0 */ - if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { break; } @@ -5046,23 +5020,21 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { } doRevisedResultsByLimit(pQInfo); - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { -// dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, -// pQInfo, pQuery->lastKey, pQuery->ekey); + dTrace("QInfo:%p query paused due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, + pQInfo, pQuery->lastKey, pQuery->window.ekey); } // dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, -// pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); +// pMeterObj->sid, pMeterObj->meterId, pQuery->size, pQInfo->size, pQInfo->pointsReturned); // pQuery->pointsOffset = pQuery->pointsToRead; //restore the available buffer // if (!isTSCompQuery(pQuery)) { -// assert(pQuery->pointsRead <= pQuery->pointsToRead); +// assert(pQuery->size <= pQuery->pointsToRead); // } } -static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { +static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { @@ -5088,13 +5060,7 @@ static void vnodeSingleMeterIntervalMainLooper(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= c; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - break; - } - - // load the data block for the next retrieve - // loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED|QUERY_RESBUF_FULL)) { break; } } @@ -5108,12 +5074,11 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { int32_t numOfInterpo = 0; while (1) { - resetCtxOutputBuf(pRuntimeEnv); - vnodeSingleMeterIntervalMainLooper(pRuntimeEnv); + tableIntervalProcessImpl(pRuntimeEnv); if (pQuery->intervalTime > 0) { pQInfo->subgroupIdx = 0; // always start from 0 - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); @@ -5124,43 +5089,43 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { doRevisedResultsByLimit(pQInfo); break; } else { - taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.pointsRead, pQuery->interpoType); + taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.size, pQuery->interpoType); SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.pointsRead * pQuery->pSelectExpr[i].resBytes); + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.size * pQuery->pSelectExpr[i].resBytes); } numOfInterpo = 0; - pQuery->rec.pointsRead = vnodeQueryResultInterpolate( - pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.pointsRead, &numOfInterpo); + pQuery->rec.size = vnodeQueryResultInterpolate( + pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.size, &numOfInterpo); - dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.pointsRead); - if (pQuery->rec.pointsRead > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + dTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.size); + if (pQuery->rec.size > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { doRevisedResultsByLimit(pQInfo); break; } // no result generated yet, continue retrieve data - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; } } // all data scanned, the group by normal column can return if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // todo refactor with merge interval time result pQInfo->subgroupIdx = 0; - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); } - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; pQInfo->pointsInterpo += numOfInterpo; // dTrace("%p vid:%d sid:%d id:%s, %d points returned %d points interpo, totalRead:%d totalInterpo:%d // totalReturn:%d", - // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, - // pQInfo->pointsRead - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); + // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->size, numOfInterpo, + // pQInfo->size - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } void qTableQuery(SQInfo *pQInfo) { @@ -5187,16 +5152,16 @@ void qTableQuery(SQInfo *pQInfo) { int32_t numOfInterpo = 0; int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); - pQuery->rec.pointsRead = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, + pQuery->rec.size = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); doRevisedResultsByLimit(pQInfo); pQInfo->pointsInterpo += numOfInterpo; - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; // dTrace("QInfo:%p %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", - // pQInfo, pQuery->pointsRead, numOfInterpo, pQInfo->pointsRead, pQInfo->pointsInterpo, + // pQInfo, pQuery->size, numOfInterpo, pQInfo->size, pQInfo->pointsInterpo, // pQInfo->pointsReturned); sem_post(&pQInfo->dataReady); return; @@ -5206,22 +5171,22 @@ void qTableQuery(SQInfo *pQInfo) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - (pQuery->intervalTime > 0 && pQInfo->rec.pointsTotal < pQuery->limit.limit)) { + (pQuery->intervalTime > 0 && pQuery->rec.total < pQuery->limit.limit)) { // todo limit the output for interval query? - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; pQInfo->subgroupIdx = 0; // always start from 0 if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); - pQInfo->rec.pointsRead += pQuery->rec.pointsRead; + pQuery->rec.size += pQuery->rec.size; clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); - if (pQuery->rec.pointsRead > 0) { + if (pQuery->rec.size > 0) { // dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d // totalReturn:%d", - // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, - // pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); + // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->size, + // pQInfo->size, pQInfo->pointsInterpo, pQInfo->pointsReturned); sem_post(&pQInfo->dataReady); return; @@ -5231,7 +5196,7 @@ void qTableQuery(SQInfo *pQInfo) { // dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, // pMeterObj->sid, - // pMeterObj->meterId, pQInfo->pointsRead); + // pMeterObj->meterId, pQInfo->size); // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); @@ -5239,7 +5204,7 @@ void qTableQuery(SQInfo *pQInfo) { } // number of points returned during this query - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); @@ -5265,7 +5230,7 @@ void qTableQuery(SQInfo *pQInfo) { if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); } else { - dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.pointsRead); + dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.size); } sem_post(&pQInfo->dataReady); @@ -5288,7 +5253,7 @@ void qSuperTableQuery(void *pReadMsg) { // assert(pQInfo->refCount >= 1); #if 0 SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; - pQuery->rec.pointsRead = 0; + pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); if (pQuery->intervalTime > 0 || @@ -5306,13 +5271,13 @@ void qSuperTableQuery(void *pReadMsg) { pQInfo->elapsedTime += (taosGetTimestampUs() - st); pQuery->status = isQueryKilled(pQInfo) ? 1 : 0; -// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->pointsRead, +// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, // pQInfo->query.interpoType); - if (pQuery->rec.pointsRead == 0) { + if (pQuery->rec.size == 0) { // pQInfo->over = 1; // dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, -// pQInfo->pointsRead); +// pQInfo->size); // vnodePrintQueryStatistics(pSupporter); } @@ -5916,12 +5881,14 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // set the output buffer capacity - pQuery->capacity = 4096; + pQuery->rec.capacity = 4096; + pQuery->rec.threshold = 2; + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { assert(pExprs[col].interResBytes >= pExprs[col].resBytes); // allocate additional memory for interResults that are usually larger then final results - size_t size = (pQuery->capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); + size_t size = (pQuery->rec.capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); pQuery->sdata[col] = (SData *)calloc(1, size); if (pQuery->sdata[col] == NULL) { goto _clean_memory; @@ -5943,9 +5910,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->pTableIdList = pTableIdList; pQuery->pos = -1; - // dTrace("vid:%d sid:%d meterId:%s, QInfo is allocated:%p", pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, - // pQInfo); - + + dTrace("QInfo %p is allocated", pQInfo); return pQInfo; _clean_memory: @@ -6098,7 +6064,7 @@ _error: return code; } -int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, void* param, SQInfo **pQInfo) { assert(pQueryTableMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -6136,6 +6102,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ // pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryTableMsg, &code); } else { code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); + (*pQInfo)->param = param; } _query_over: @@ -6161,7 +6128,7 @@ _query_over: return TSDB_CODE_SUCCESS; } -int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *rowsize) { +int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { if (pQInfo == NULL || !isQInfoValid(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } @@ -6177,11 +6144,8 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo, int32_t *numOfRows, int32_t *ro } sem_wait(&pQInfo->dataReady); - - *numOfRows = pQInfo->rec.pointsRead; - *rowsize = pQuery->rowSize; - - dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, *rowsize, *numOfRows, pQInfo->code); + dTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.size, + pQInfo->code); return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } @@ -6208,7 +6172,7 @@ static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { } } -static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int32_t *size) { +static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // the remained number of retrieved rows, not the interpolated result SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -6231,28 +6195,31 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int32_t *size) { pQuery->sdata[0]->data, strerror(errno)); } } else { - doCopyQueryResultToMsg(pQInfo, pQInfo->rec.pointsRead, data); + doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); } - pQInfo->rec.pointsTotal += pQInfo->rec.pointsRead; - dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQInfo->rec.pointsRead, pQInfo->rec.pointsTotal); + pQuery->rec.total += pQuery->rec.size; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); - setQueryStatus(pQuery, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; // todo if interpolation exists, the result may be dump to client by several rounds } -static void addToTaskQueue(SQInfo* pQInfo) { - // no error occurred, continue retrieving data - if (pQInfo->code == TSDB_CODE_SUCCESS) { -#ifdef _TD_ARM_ - dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:doDumpQueryResult", pQInfo, pQInfo->signature); -#else - dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:%s", pQInfo, pQInfo->signature, __FUNCTION__); -#endif - - // todo add to task queue +bool qNeedFurtherExec(SQInfo* pQInfo) { + if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { + return false; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { + return false; + } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { + return true; + } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + return true; + } else { + assert(0); } } @@ -6262,13 +6229,12 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c } SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQInfo->rec.pointsRead); + size_t size = getResultSize(pQInfo, &pQuery->rec.size); *contLen = size + sizeof(SRetrieveTableRsp); // todo handle failed to allocate memory *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); - - (*pRsp)->numOfRows = htonl(pQInfo->rec.pointsRead); + (*pRsp)->numOfRows = htonl(pQuery->rec.size); int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { @@ -6279,16 +6245,13 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c (*pRsp)->useconds = 0; } - if (pQInfo->rec.pointsRead > 0 && code == TSDB_CODE_SUCCESS) { - code = doDumpQueryResult(pQInfo, (*pRsp)->data, NULL); - - // has more data to return or need next round to execute - addToTaskQueue(pQInfo); + if (pQuery->rec.size > 0 && code == TSDB_CODE_SUCCESS) { + code = doDumpQueryResult(pQInfo, (*pRsp)->data); } else { code = pQInfo->code; } - if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { (*pRsp)->completed = 1; // notify no more result to client vnodeFreeQInfo(pQInfo); } diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 36472857fe..2dc61f5107 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -124,6 +124,7 @@ typedef struct STsdbQueryHandle { int32_t tableIndex; bool isFirstSlot; void * qinfo; // query info handle, for debug purpose + SSkipListIterator* memIter; } STsdbQueryHandle; int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { @@ -367,8 +368,13 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { int32_t rows = 0; if (pTable->mem != NULL) { - SSkipListIterator* iter = tSkipListCreateIter(pTable->mem->pData); - rows = tsdbReadRowsFromCache(iter, INT64_MAX, 4000, &skey, &ekey, pHandle); + + // create mem table iterator if it is not created yet + if (pHandle->memIter == NULL) { + pHandle->memIter = tSkipListCreateIter(pTable->mem->pData); + } + + rows = tsdbReadRowsFromCache(pHandle->memIter, INT64_MAX, 2, &skey, &ekey, pHandle); } SDataBlockInfo blockInfo = { From 6f65c5a9616a67431be62a7864a7d92a0fb9c5f0 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 26 Mar 2020 11:22:06 +0800 Subject: [PATCH 49/85] [TD-9] error while get metadata --- src/mnode/src/mgmtChildTable.c | 12 ++++++------ src/mnode/src/mgmtSuperTable.c | 4 ++-- src/mnode/src/mgmtTable.c | 7 +++++++ src/mnode/src/mgmtVgroup.c | 29 +++++++++++++---------------- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 2d4f37581d..58cee331b3 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -725,13 +725,13 @@ static int32_t mgmtDoGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STab if (pTable->info.type == TSDB_CHILD_TABLE) { pMeta->sversion = htons(pTable->superTable->sversion); - pMeta->numOfTags = htons(pTable->superTable->numOfTags); - pMeta->numOfColumns = htons(pTable->superTable->numOfColumns); + pMeta->numOfTags = 0; + pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns); pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); } else { pMeta->sversion = htons(pTable->sversion); pMeta->numOfTags = 0; - pMeta->numOfColumns = htons(pTable->numOfColumns); + pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromNormalTable(pMeta->schema, pTable); } @@ -758,9 +758,9 @@ static int32_t mgmtDoGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STab void mgmtGetChildTableMeta(SQueuedMsg *pMsg, SChildTableObj *pTable) { SCMTableInfoMsg *pInfo = pMsg->pCont; - SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); + SDbObj *pDb = mgmtGetDbByTableId(pInfo->tableId); if (pDb == NULL || pDb->dirty) { - mError("table:%s, failed to get table meta, db not selected", pTable->info.tableId); + mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } @@ -877,7 +877,7 @@ static STableInfo* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t si return NULL; } - return pVgroup->tableList[sid]; + return (STableInfo *)pVgroup->tableList[sid]; } static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index e340643998..ba7be9ba59 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -610,8 +610,8 @@ void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) { pMeta->uid = htobe64(pTable->uid); pMeta->sversion = htons(pTable->sversion); pMeta->precision = pDb->cfg.precision; - pMeta->numOfTags = pTable->numOfTags; - pMeta->numOfColumns = htons(pTable->numOfColumns); + pMeta->numOfTags = htons((int16_t)pTable->numOfTags); + pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); strcpy(pMeta->tableId, pTable->info.tableId); diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 3e47a7c277..5169f4a817 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -230,6 +230,13 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { SCMTableInfoMsg *pInfo = pMsg->pCont; mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle); + pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId); + if (pMsg->pDb == NULL || pMsg->pDb->dirty) { + mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); + return; + } + STableInfo *pTable = mgmtGetTable(pInfo->tableId); if (pTable == NULL || pTable->type != TSDB_SUPER_TABLE) { mgmtGetChildTableMeta(pMsg, (SChildTableObj *)pTable); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index a8e7219e2e..f6f0e9b76a 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -25,6 +25,7 @@ #include "mgmtDClient.h" #include "mgmtDnode.h" #include "mgmtDServer.h" +#include "mgmtMnode.h" #include "mgmtProfile.h" #include "mgmtSdb.h" #include "mgmtShell.h" @@ -483,17 +484,6 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { return pVnode; } -static SVgObj *mgmtGetVgroupInDnode(uint32_t dnodeId, int32_t vgId) { - if (vnovgId < 1 || dnodeId < 1) return NULL; - - SDnodeObj *pDnode = mgmtGetDnode(dnodeId); - if (pDnode == NULL) { - return NULL; - } - - return mgmtGetVgroup(vgId); -} - SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) { SRpcIpSet ipSet = { .numOfIps = pVgroup->numOfVnodes, @@ -657,19 +647,26 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { if (mgmtCheckRedirect(rpcMsg->handle)) return; SDMConfigVnodeMsg *pCfg = (SDMConfigVnodeMsg *) rpcMsg->pCont; - pCfg->dnodeId = htonl(pCfg->dnode); - pCfg->vgId = htonl(pCfg->vnode); + pCfg->dnodeId = htonl(pCfg->dnodeId); + pCfg->vgId = htonl(pCfg->vgId); - SVgObj *pVgroup = mgmtGetVgroupInDnode(pCfg->dnodeId, pCfg->vgId); + SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnodeId); + if (pDnode == NULL) { + mTrace("dnode:%s, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); + return; + } + + SVgObj *pVgroup = mgmtGetVgroup(pCfg->vgId); if (pVgroup == NULL) { - mTrace("dnode:%s, vnode:%d, no vgroup info", taosIpStr(pCfg->dnode), pCfg->vnode); + mTrace("dnode:%s, vgId:%d, no vgroup info", taosIpStr(pCfg->dnodeId), pCfg->vgId); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_NOT_ACTIVE_VNODE); return; } mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); } From 021d4b424012470d69554b572d36a9ca4212364d Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 26 Mar 2020 12:09:22 +0800 Subject: [PATCH 50/85] [TD-9] error while create child table --- src/mnode/src/mgmtSuperTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index ba7be9ba59..44e8014dad 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -610,7 +610,7 @@ void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) { pMeta->uid = htobe64(pTable->uid); pMeta->sversion = htons(pTable->sversion); pMeta->precision = pDb->cfg.precision; - pMeta->numOfTags = htons((int16_t)pTable->numOfTags); + pMeta->numOfTags = (uint8_t)pTable->numOfTags; pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); From f629aee1c72013f6128bed4b9a376ce7a9107deb Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 26 Mar 2020 14:15:15 +0800 Subject: [PATCH 51/85] [TD-32] fix error in retrieve data from table created according to stable --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscAsync.c | 5 ---- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscSchemaUtil.c | 23 +++++++++------- src/client/src/tscServer.c | 50 +++++++++++++++------------------- src/client/src/tscSql.c | 3 +- src/inc/taosmsg.h | 25 ++++++----------- src/mnode/src/mgmtChildTable.c | 1 + 8 files changed, 48 insertions(+), 63 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index e9d2311715..25a5db6118 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -32,7 +32,7 @@ extern "C" { #define UTIL_TABLE_IS_SUPERTABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) #define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo) (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo))) -#define UTIL_TABLE_CREATE_FROM_STABLE(metaInfo) \ +#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 2bab6e03fe..1b9b0a365f 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -406,11 +406,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pSql->fp == NULL) { - tscError("%p callBack is NULL!!!", pSql); - return; - } - if (pSql->fp == (void *)1) { pSql->fp = NULL; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 6427578e82..34f043a15b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4240,7 +4240,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo))) { return invalidSqlErrMsg(pQueryInfo->msg, msg4); } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) && - UTIL_TABLE_CREATE_FROM_STABLE(pTableMetaInfo)) { + UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 78d29be1af..66776e36f7 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -51,12 +51,12 @@ int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) { SSchema *tscGetTableSchema(const STableMeta *pTableMeta) { assert(pTableMeta != NULL); - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - STableMeta* pSTableMeta = pTableMeta->pSTable; - assert (pSTableMeta != NULL); - - return pSTableMeta->schema; - } +// if (pTableMeta->tableType == TSDB_CHILD_TABLE) { +// STableMeta* pSTableMeta = pTableMeta->pSTable; +// assert (pSTableMeta != NULL); +// +// return pSTableMeta->schema; +// } return (SSchema*) pTableMeta->schema; } @@ -72,12 +72,14 @@ SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) { STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); - + +#if 0 if (pTableMeta->tableType == TSDB_CHILD_TABLE) { assert (pTableMeta->pSTable != NULL); return pTableMeta->pSTable->tableInfo; } - +#endif + return pTableMeta->tableInfo; } @@ -130,12 +132,13 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t startCol) assert(pTableMeta != NULL); SSchema* pSchema = pTableMeta->schema; - +#if 0 if (pTableMeta->tableType == TSDB_CHILD_TABLE) { assert (pTableMeta->pSTable != NULL); pSchema = pTableMeta->pSTable->schema; } - +#endif + return &pSchema[startCol]; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index decf9875d2..5826d02cf2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -325,8 +325,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); if (rpcMsg->code != TSDB_CODE_ACTION_IN_PROGRESS) { - int command = pCmd->command; - void *taosres = tscKeepConn[command] ? pSql : NULL; + void *taosres = tscKeepConn[pCmd->command] ? pSql : NULL; + rpcMsg->code = pRes->code ? -pRes->code : pRes->numOfRows; + tscTrace("%p Async SQL result:%s res:%p", pSql, tstrerror(pRes->code), taosres); /* @@ -1817,14 +1818,13 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->vgId = htonl(pMetaMsg->vgId); pMetaMsg->uid = htobe64(pMetaMsg->uid); pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if (pMetaMsg->sid < 0 || pMetaMsg->vgId < 0) { tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgId, pMetaMsg->sid); return TSDB_CODE_INVALID_VALUE; } - pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->numOfTags > TSDB_MAX_TAGS || pMetaMsg->numOfTags < 0) { tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); return TSDB_CODE_INVALID_VALUE; @@ -1848,23 +1848,20 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pSchema++; } -// rsp += numOfTotalCols * sizeof(SSchema); -// -// int32_t tagLen = 0; -// SSchema *pTagsSchema = tscGetTableTagSchema(pMetaMsg); -// -// if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { -// for (int32_t i = 0; i < pMetaMsg->numOfTags; ++i) { -// tagLen += pTagsSchema[i].bytes; -// } -// } -// -// rsp += tagLen; -// int32_t size = (int32_t)(rsp - (char *)pMetaMsg); - size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); +#if 0 + // if current table is created according to super table, get the table meta of super table + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + char id[TSDB_TABLE_ID_LEN + 1] = {0}; + strncpy(id, pMetaMsg->stableId, TSDB_TABLE_ID_LEN); + + // NOTE: if the table meta of super table is not cached at client side yet, the pSTable is NULL + pTableMeta->pSTable = taosCacheAcquireByName(tscCacheHandle, id); + } +#endif + // todo add one more function: taosAddDataIfNotExists(); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); @@ -1878,6 +1875,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } free(pTableMeta); + return TSDB_CODE_SUCCESS; } @@ -2343,7 +2341,7 @@ int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) { void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { +static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("%p malloc failed for new sqlobj to get meter meta", pSql); @@ -2370,7 +2368,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - strcpy(pNewMeterMetaInfo->name, pTableMetaInfo->name); + strncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, tListLen(pNewMeterMetaInfo->name)); memcpy(pNew->cmd.payload, pSql->cmd.payload, TSDB_DEFAULT_PAYLOAD_SIZE); // tag information if table does not exists. tscTrace("%p new pSqlObj:%p to get tableMeta", pSql, pNew); @@ -2388,7 +2386,7 @@ static int32_t doGetMeterMetaFromServer(SSqlObj *pSql, STableMetaInfo *pTableMet int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { assert(strlen(pTableMetaInfo->name) != 0); - // If this STableMetaInfo owns a metermeta, release it first + // If this STableMetaInfo owns a table meta, release it first if (pTableMetaInfo->pTableMeta != NULL) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false); } @@ -2401,12 +2399,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { return TSDB_CODE_SUCCESS; } - - /* - * for async insert operation, release data block buffer before issue new object to get metermeta - * because in table meta callback function, the tscParse function will generate the submit data blocks - */ - return doGetMeterMetaFromServer(pSql, pTableMetaInfo); + + return getTableMetaFromMgmt(pSql, pTableMetaInfo); } int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { @@ -2455,7 +2449,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { tscWaitingForCreateTable(pCmd); taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - code = doGetMeterMetaFromServer(pSql, pTableMetaInfo); // todo ?? + code = getTableMetaFromMgmt(pSql, pTableMetaInfo); // todo ?? } else { tscTrace("%p metric query not update metric meta, numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, tscGetNumOfTags(pTableMetaInfo->pTableMeta), pCmd->numOfCols, pTableMetaInfo->pTableMeta->uid, diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d62dac088b..eb46d6101c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -651,8 +651,7 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) { static void asyncFetchCallback(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; - if (numOfRows < 0) { - // set the error code + if (numOfRows < 0) { // set the error code pSql->res.code = -numOfRows; } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 49e498d22f..838b43da1e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -206,14 +206,6 @@ typedef struct SMsgHead { int32_t vgId; } SMsgHead; -// typedef struct { -// SMsgDesc desc; -// SMsgHead header; -// int16_t import; -// int32_t numOfTables; // total number of sid -// char blks[]; // number of data blocks, each table has at least one data block -//} SShellSubmitMsg; - // Submit message for one table typedef struct SSubmitBlk { int64_t uid; // table unique id @@ -683,14 +675,15 @@ typedef struct { } SSuperTableMeta; typedef struct STableMetaMsg { - char tableId[TSDB_TABLE_ID_LEN]; // note: This field must be at the front - int32_t contLen; - uint8_t numOfTags; - uint8_t precision; - uint8_t tableType; - int16_t numOfColumns; - int16_t sversion; - + int32_t contLen; + + char tableId[TSDB_TABLE_ID_LEN]; // table id + char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table + uint8_t numOfTags; + uint8_t precision; + uint8_t tableType; + int16_t numOfColumns; + int16_t sversion; int8_t numOfVpeers; SVnodeDesc vpeerDesc[TSDB_VNODES_SUPPORT]; int32_t sid; diff --git a/src/mnode/src/mgmtChildTable.c b/src/mnode/src/mgmtChildTable.c index 58cee331b3..4936c941ca 100644 --- a/src/mnode/src/mgmtChildTable.c +++ b/src/mnode/src/mgmtChildTable.c @@ -728,6 +728,7 @@ static int32_t mgmtDoGetChildTableMeta(SDbObj *pDb, SChildTableObj *pTable, STab pMeta->numOfTags = 0; pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns); pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); + strncpy(pMeta->stableId, pTable->superTable->info.tableId, tListLen(pMeta->stableId)); } else { pMeta->sversion = htons(pTable->sversion); pMeta->numOfTags = 0; From 240a201b9a7b822d98556ccef75a3f1aa258bd88 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 26 Mar 2020 23:26:48 +0800 Subject: [PATCH 52/85] TD-34 --- src/common/inc/dataformat.h | 17 +++--- src/common/src/dataformat.c | 1 + src/util/inc/tutil.h | 8 +++ src/util/src/tutil.c | 69 +++++++++++++++++++++ src/vnode/tsdb/inc/tsdbFile.h | 30 ++++++---- src/vnode/tsdb/src/tsdbFile.c | 107 +++++++++++++++++++++++++++++++-- src/vnode/tsdb/src/tsdbMain.c | 110 ++++++++++++++++++++++++++++++++-- 7 files changed, 314 insertions(+), 28 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index e123efd11e..52b2d1e156 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -105,11 +105,12 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int8_t type; - int bytes; - int len; - int offset; - void * pData; + int8_t type; + int16_t colId; + int bytes; + int len; + int offset; + void * pData; } SDataCol; typedef struct { @@ -122,9 +123,9 @@ typedef struct { SDataCol cols[]; } SDataCols; -#define keyCol(cols) (&((cols)->cols[0])) // Key column -#define dataColsKeyFirst(cols) ((int64_t *)(keyCol(cols)->pData))[0] -#define dataColsKeyLast(cols) ((int64_t *)(keyCol(cols)->pData))[(cols)->numOfPoints - 1] +#define keyCol(pCols) (&((pCols)->cols[0])) // Key column +#define dataColsKeyFirst(pCols) ((int64_t *)(keyCol(pCols)->pData))[0] +#define dataColsKeyLast(pCols) ((int64_t *)(keyCol(pCols)->pData))[(pCols)->numOfPoints - 1] SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index da55663d0b..8f6a40805f 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -324,6 +324,7 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { pCols->cols[i].type = colType(schemaColAt(pSchema, i)); pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i)); pCols->cols[i].offset = colOffset(schemaColAt(pSchema, i)); + pCols->cols[i].colId = colColId(schemaColAt(pSchema, i)); } return pCols; diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index df97dde5ac..b80aad1ceb 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -176,6 +176,14 @@ uint32_t ip2uint(const char *const ip_addr); void taosSetAllocMode(int mode, const char* path, bool autoDump); void taosDumpMemoryLeak(); +#define TD_EQ 0x1 +#define TD_GT 0x2 +#define TD_LT 0x4 +#define TD_GE (TD_EQ | TD_GT) +#define TD_LE (TD_EQ | TD_LT) +void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, + int (*compar)(const void *, const void *), int flags); + #ifdef TAOS_MEM_CHECK void * taos_malloc(size_t size, const char *file, uint32_t line); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 9c384b25ba..cbd08954cc 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -617,3 +617,72 @@ char *taosCharsetReplace(char *charsetstr) { return strdup(charsetstr); } + +#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx)) +void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *), int flags) { + // TODO: need to check the correctness of this function + int l = 0; + int r = nmemb; + int idx = 0; + int comparison; + + if (flags == TD_EQ) { + return bsearch(key, base, nmemb, size, compar); + } else if (flags == TD_GE) { + if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0); + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx) < 0)) { + return elePtrAt(base, size, idx); + } else { + if (idx + 1 > nmemb - 1) { + return NULL; + } else { + return elePtrAt(base, size, idx + 1); + } + } + } else if (flags == TD_LE) { + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1); + if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx)) > 0) { + return elePtrAt(base, size, idx); + } else { + if (idx == 0) { + return NULL; + } else { + return elePtrAt(base, size, idx - 1); + } + } + + } else { + assert(0); + return NULL; + } + + return NULL; +} diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 9a4d94c58f..22563275cd 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -17,6 +17,7 @@ #include +#include "dataformat.h" #include "taosdef.h" #include "tglobalcfg.h" @@ -69,20 +70,26 @@ typedef struct { STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles); void tsdbCloseFileH(STsdbFileH *pFileH); int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); +int tsdbOpenFile(SFile *pFile, int oflag); +SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); typedef struct { int32_t len; - int32_t padding; // For padding purpose - int64_t offset; -} SCompIdx; + int32_t offset; + int32_t hasLast : 1; + int32_t numOfSuperBlocks : 31; + int32_t checksum; + TSKEY maxKey; +} SCompIdx; /* sizeof(SCompIdx) = 24 */ /** - * if numOfSubBlocks == -1, then the SCompBlock is a sub-block - * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to - * the data block offset and length - * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the - * binary + * if numOfSubBlocks == 0, then the SCompBlock is a sub-block + * if numOfSubBlocks >= 1, then the SCompBlock is a super-block + * - if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to + * the data block offset and length + * - if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the + * binary */ typedef struct { int64_t last : 1; // If the block in data file or last file @@ -101,11 +108,12 @@ typedef struct { int32_t delimiter; // For recovery usage int32_t checksum; // TODO: decide if checksum logic in this file or make it one API int64_t uid; - int32_t padding; // For padding purpose - int32_t numOfBlocks; // TODO: make the struct padding SCompBlock blocks[]; } SCompInfo; +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); + // TODO: take pre-calculation into account typedef struct { int16_t colId; // Column ID @@ -122,6 +130,8 @@ typedef struct { SCompCol cols[]; } SCompData; +int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols); + void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index f622c38b5f..7d0bdbd845 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -39,6 +39,7 @@ static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname); static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile); static int tsdbWriteFileHead(SFile *pFile); static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); +static SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * maxFiles); @@ -70,9 +71,7 @@ int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) SFileGroup fGroup; SFileGroup *pFGroup = &fGroup; - if (fid < TSDB_MIN_FILE_ID(pFileH) || fid > TSDB_MAX_FILE_ID(pFileH) || - bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey) == - NULL) { + if (tsdbSearchFGroup(pFileH, fid) == NULL) { pFGroup->fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(dataDir, fid, type, maxTables, &(pFGroup->files[type])) < 0) { @@ -107,6 +106,86 @@ int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { return 0; } +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); + if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + + if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1; + // TODO: need to check the correctness + return 0; +} + +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); + + if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1; + + if (read(pFile->fd, buf, pIdx->len) < 0) return -1; + + // TODO: need to check the correctness + + return 0; +} + +static int tsdbWriteBlockToFileImpl(SFile * pFile, // File to write + SDataCols * pCols, // Data column buffer + int numOfPointsToWrie, // Number of points to write to the file + SCompBlock *pBlock // SCompBlock to hold block information to return + ) { + // pBlock->last = 0; + // pBlock->offset = lseek(pFile->fd, 0, SEEK_END); + // // pBlock->algorithm = ; + // pBlock->numOfPoints = pCols->numOfPoints; + // // pBlock->sversion = ; + // // pBlock->len = ; + // pBlock->numOfSubBlocks = 1; + // pBlock->keyFirst = dataColsKeyFirst(pCols); + // pBlock->keyLast = dataColsKeyLast(pCols); + // for (int i = 0; i < pCols->numOfCols; i++) { + // // TODO: if all col value is NULL, do not save it + // pBlock->numOfCols++; + // pCompData->numOfCols++; + // SCompCol *pCompCol = pCompData->cols + i; + // pCompCol->colId = pCols->cols[i].colId; + // pCompCol->type = pCols->cols[i].type; + + // // pCompCol->len = ; + // // pCompCol->offset = ; + // } + + return 0; +} + +int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols) { + memset((void *)pBlock, 0, sizeof(SCompBlock)); + SFile *pFile = NULL; + SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols); + if (pCompData == NULL) return -1; + pCompData->delimiter = TSDB_FILE_DELIMITER; + // pCompData->uid = ; + + if (isMerge) { + TSKEY keyFirst = dataColsKeyFirst(pCols); + // 1. Binary search the block the data can merged into + + if (1/* the data should only merged into last file */) { + } else { + } + } else { + // Write directly to the file without merge + if (1/*pCols->numOfPoints < pCfg->minRowsPerFileBlock*/) { + // TODO: write the data to the last file + } else { + // TODO: wirte the data to the data file + } + } + + // TODO: need to update pIdx + + if (pCompData) free(pCompData); + return 0; +} + static int compFGroupKey(const void *key, const void *fgroup) { int fid = *(int *)key; SFileGroup *pFGroup = (SFileGroup *)fgroup; @@ -158,7 +237,7 @@ static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname) return 0; } -static int tsdbOpenFileForWrite(SFile *pFile, int oflag) { // TODO: change the function +int tsdbOpenFile(SFile *pFile, int oflag) { // TODO: change the function if (TSDB_IS_FILE_OPENED(pFile)) return -1; pFile->fd = open(pFile->fname, oflag, 0755); @@ -167,6 +246,16 @@ static int tsdbOpenFileForWrite(SFile *pFile, int oflag) { // TODO: change the f return 0; } +SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { + SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); + if (pGroup == NULL) return NULL; + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbOpenFile(&(pGroup->files[type]), O_RDWR); + } + return pGroup; +} + static int tsdbCloseFile(SFile *pFile) { if (!TSDB_IS_FILE_OPENED(pFile)) return -1; int ret = close(pFile->fd); @@ -186,7 +275,7 @@ static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, return -1; } - if (tsdbOpenFileForWrite(pFile, O_WRONLY | O_CREAT) < 0) { + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) { // TODO: deal with the ERROR here return -1; } @@ -212,4 +301,12 @@ void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t file TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; +} + +static SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { + if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) + return NULL; + void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); + if (ptr == NULL) return NULL; + return (SFileGroup *)ptr; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index a8e04e216a..8e433ecb5b 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -764,6 +764,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { int numOfRows = 0; + do { SSkipListNode *node = tSkipListIterGet(pIter); if (node == NULL) break; @@ -776,6 +777,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max numOfRows++; if (numOfRows > maxRowsToRead) break; } while (tSkipListIterNext(pIter)); + return numOfRows; } @@ -865,24 +867,122 @@ static void *tsdbCommitData(void *arg) { } static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { + int flag = 0; + STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; + SFile tFile, lFile; + SFileGroup *pGroup = NULL; + SCompIdx * pIndices = NULL; + SCompInfo * pCompInfo = NULL; + size_t compInfoSize = 0; + SCompBlock compBlock; + SCompBlock *pBlock = &compBlock; + TSKEY minKey = 0, maxKey = 0; tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; + STable * pTable = pMeta->tables[tid]; SSkipListIterator *pIter = iters[tid]; + int isLoadCompBlocks = 0; if (pIter == NULL) continue; tdInitDataCols(pCols, pTable->schema); - while (tsdbReadRowsFromCache(pIter, maxKey, pCfg->maxRowsPerFileBlock, pCols)) { - // TODO - int k = 0; - } + int numOfWrites = 0; + // while (tsdbReadRowsFromCache(pIter, maxKey, pCfg->maxRowsPerFileBlock, pCols)) { + // break; + // if (!flag) { + // // There are data to commit to this file, we need to create/open it for read/write. + // // At the meantime, we set the flag to prevent further create/open operations + // if (tsdbCreateFGroup(pFileH, pRepo->rootDir, fid, pCfg->maxTables) < 0) { + // // TODO: deal with the ERROR here + // } + // // Open files for commit + // pGroup = tsdbOpenFilesForCommit(pFileH, fid); + // if (pGroup == NULL) { + // // TODO: deal with the ERROR here + // } + // // TODO: open .h file and if neccessary, open .l file + // {} + // pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); + // if (pIndices == NULL) { + // // TODO: deal with the ERROR + // } + // // load the SCompIdx part + // if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { + // // TODO: deal with the ERROR here + // } + + // // TODO: sendfile those not need changed table content + // for (int ttid = 0; ttid < tid; ttid++) { + // // SCompIdx *pIdx = &pIndices[ttid]; + // // if (pIdx->len > 0) { + // // lseek(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, pIdx->offset, 0, SEEK_CUR); + // // sendfile(fd, pGroup->files[TSDB_FILE_TYPE_HEAD].fd, NULL, pIdx->len); + // // } + // } + // flag = 1; + // } + + // SCompIdx *pIdx = &pIndices[tid]; + + // /* The first time to write to the table, need to decide + // * if it is neccessary to load the SComplock part. If it + // * is needed, just load it, or, just use sendfile and + // * append it. + // */ + // if (numOfWrites == 0 && pIdx->offset > 0) { + // if (dataColsKeyFirst(pCols) <= pIdx->maxKey || pIdx->hasLast) { + // pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len); + // if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { + // // TODO: deal with the ERROR here + // } + // if (pCompInfo->uid == pTable->tableId.uid) isLoadCompBlocks = 1; + // } else { + // // TODO: sendfile the prefix part + // } + // } + + // // if (tsdbWriteBlockToFile(pGroup, pCompInfo, pIdx, isLoadCompBlocks, pBlock, pCols) < 0) { + // // // TODO: deal with the ERROR here + // // } + + // // pCompInfo = tsdbMergeBlock(pCompInfo, pBlock); + + + // // if (1 /* the SCompBlock part is not loaded*/) { + // // // Append to .data file generate a SCompBlock and record it + // // } else { + // // } + + // // // TODO: need to reset the pCols + + // numOfWrites++; + // } + + // if (pCols->numOfPoints > 0) { + // // TODO: still has data to commit, commit it + // } + + // if (1/* SCompBlock part is loaded, write it to .head file*/) { + // // TODO + // } else { + // // TODO: use sendfile send the old part and append the newly added part + // } } + // Write the SCompIdx part + + // Close all files and return + if (flag) { + // TODO + } + + if (pIndices) free(pIndices); + if (pCompInfo) free(pCompInfo); + return 0; } \ No newline at end of file From c269b6d474cdc4d2d1dc1cd4ca2785b3dfcf471a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 27 Mar 2020 01:59:26 +0800 Subject: [PATCH 53/85] [TD-32] fix error in retrieve data --- src/client/inc/tscUtil.h | 6 +- src/client/inc/tsclient.h | 5 +- src/client/src/tscAsync.c | 4 +- src/client/src/tscLocal.c | 7 +- src/client/src/tscParseInsert.c | 8 +- src/client/src/tscSQLParser.c | 32 +++-- src/client/src/tscSchemaUtil.c | 2 +- src/client/src/tscSecondaryMerge.c | 10 +- src/client/src/tscServer.c | 214 +++++++++++++---------------- src/client/src/tscSql.c | 10 +- src/client/src/tscStream.c | 2 +- src/client/src/tscSub.c | 32 ++--- src/client/src/tscSubquery.c | 50 +++---- src/client/src/tscUtil.c | 46 ++++--- src/dnode/src/dnodeRead.c | 2 +- src/inc/taosmsg.h | 136 +++++++++--------- src/mnode/src/mgmtSuperTable.c | 6 +- src/query/inc/qsqltype.h | 2 +- src/query/inc/queryExecutor.h | 8 +- src/query/src/queryExecutor.c | 136 ++++++++---------- src/vnode/tsdb/src/tsdbMeta.c | 23 +++- src/vnode/tsdb/src/tsdbRead.c | 5 +- 22 files changed, 364 insertions(+), 382 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 25a5db6118..402b6b3706 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -106,7 +106,7 @@ bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryOnMetric(SSqlCmd* pCmd); -bool tscQueryMetricTags(SQueryInfo* pQueryInfo); +bool tscQueryTags(SQueryInfo* pQueryInfo); bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, @@ -176,7 +176,7 @@ void tscIncStreamExecutionCount(void* pStream); bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload -SCond* tsGetMetricQueryCondPos(STagCond* pCond, uint64_t tableIndex); +SCond* tsGetSTableQueryCondPos(STagCond* pCond, uint64_t tableIndex); void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); void tscTagCondCopy(STagCond* dest, const STagCond* src); @@ -207,7 +207,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid); -int tscGetMetricMeta(SSqlObj* pSql, int32_t clauseIndex); +int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index fdc0ae9095..9cb43a9f36 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -70,8 +70,9 @@ typedef struct STableMeta { typedef struct STableMetaInfo { STableMeta * pTableMeta; // table meta, cached in client side and acquried by name - SSuperTableMeta *pMetricMeta; // metricmeta - +// SSuperTableMeta *pMetricMeta; // metricmeta + SArray* vgroupIdList; + /* * 1. keep the vnode index during the multi-vnode super table projection query * 2. keep the vnode index for multi-vnode insertion diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 1b9b0a365f..366faa15dc 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -459,7 +459,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - code = tscGetMetricMeta(pSql, 0); + code = tscGetSTableVgroupInfo(pSql, 0); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; @@ -489,7 +489,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - code = tscGetMetricMeta(pSql, pCmd->clauseIndex); + code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); pRes->code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 46f3ab6687..42b04d7a16 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -286,6 +286,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { // todo add order support static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { +#if 0 // the result structure has been completed in sql parse, so we // only need to reorganize the results in the column format SSqlCmd * pCmd = &pSql->cmd; @@ -337,6 +338,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { } } +#endif return 0; } @@ -345,7 +347,7 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - +#if 0 SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result int32_t rowLen = tscGetResRowLength(pQueryInfo); @@ -369,7 +371,8 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { } rowIdx++; } - +#endif + return 0; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 5651e5aa38..9c7a3e0bf0 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1295,11 +1295,11 @@ int tsParseInsertSql(SSqlObj *pSql) { int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) { int32_t ret = TSDB_CODE_SUCCESS; - if (NULL == pSql->asyncTblPos) { - tscCleanSqlCmd(&pSql->cmd); - } else { +// if (NULL == pSql->asyncTblPos) { +// tscCleanSqlCmd(&pSql->cmd); +// } else { tscTrace("continue parse sql: %s", pSql->asyncTblPos); - } +// } if (tscIsInsertOrImportData(pSql->sqlstr)) { /* diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 34f043a15b..6d5d08292d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -209,9 +209,14 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); - assert(pQueryInfo->numOfTables == 0); +// assert(pQueryInfo->numOfTables == 0); - STableMetaInfo* pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); + STableMetaInfo* pTableMetaInfo = NULL; + if (pQueryInfo->numOfTables == 0) { + pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); + } else { + pTableMetaInfo = &pQueryInfo->pTableMetaInfo[0]; + } pCmd->command = pInfo->type; @@ -639,7 +644,7 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { * check invalid SQL: * select tbname, tags_fields from super_table_name interval(1s) */ - if (tscQueryMetricTags(pQueryInfo) && pQueryInfo->intervalTime > 0) { + if (tscQueryTags(pQueryInfo) && pQueryInfo->intervalTime > 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -746,7 +751,7 @@ int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlO tscClearMeterMetaInfo(pTableMetaInfo, false); } } else { - assert(pTableMetaInfo->pTableMeta == NULL && pTableMetaInfo->pMetricMeta == NULL); + assert(pTableMetaInfo->pTableMeta == NULL); } tfree(oldName); @@ -1252,7 +1257,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (tscQueryMetricTags(pQueryInfo)) { // local handle the metric tag query + if (tscQueryTags(pQueryInfo)) { // local handle the metric tag query pCmd->count = numOfCols; // the number of meter schema, tricky. pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; } @@ -1293,7 +1298,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c int16_t functionId = (int16_t)((colIdx >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); if (functionId == TSDB_FUNC_TAGPRJ) { - addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); +// addRequiredTagColumn(pQueryInfo, colIdx - numOfCols, tableIndex); pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; } else { pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; @@ -1396,8 +1401,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum SColumnList ids = {0}; ids.ids[0] = *pIndex; - // tag columns do not add to source list - ids.num = (j >= tscGetNumOfColumns(pTableMeta)) ? 0 : 1; + ids.num = 1; insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); } @@ -4666,14 +4670,13 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* * And then launching multiple async-queries against all qualified virtual nodes, during the first-stage * query operation. */ - int32_t code = tscGetMetricMeta(pSql, clauseIndex); + int32_t code = tscGetSTableVgroupInfo(pSql, clauseIndex); if (code != TSDB_CODE_SUCCESS) { return code; } // No tables included. No results generated. Query results are empty. - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - if (pTableMetaInfo->pTableMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfTables == 0) { + if (pTableMetaInfo->pTableMeta == NULL) { tscTrace("%p no table in metricmeta, no output result", pSql); pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; } @@ -5687,6 +5690,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr); + + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + int32_t code = tscGetSTableVgroupInfo(pSql, index); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } // parse the group by clause in the first place if (parseGroupbyClause(pQueryInfo, pQuerySql->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 66776e36f7..2492352284 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -32,7 +32,7 @@ int32_t tscGetNumOfTags(const STableMeta* pTableMeta) { } if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) { - assert(tinfo.numOfTags > 0); + assert(tinfo.numOfTags >= 0); return tinfo.numOfTags; } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 0ee8afd53c..d69f6d295f 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes); +// (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes); if (*pMemBuffer == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -636,10 +636,10 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); - for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) { - (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); - (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; - } +// for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) { +// (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); +// (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; +// } if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5826d02cf2..b2bd9110e7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -35,7 +35,6 @@ SRpcIpSet tscDnodeIpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); -void (*tscUpdateVnodeMsg[TSDB_SQL_MAX])(SSqlObj *pSql, char *buf); void tscProcessActivityTimer(void *handle, void *tmrId); int tscKeepConn[TSDB_SQL_MAX] = {0}; TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid); @@ -188,7 +187,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .pCont = pMsg, .contLen = pSql->cmd.payloadLen, .handle = pSql, - .code = 0 + .code = 0 }; rpcSendRequest(pVnodeConn, pSql->ipList, &rpcMsg); } else { @@ -315,7 +314,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { pRes->numOfRows += pMsg->affectedRows; tscTrace("%p cmd:%d code:%d, inserted rows:%d, rsp len:%d", pSql, pCmd->command, pRes->code, - *(int32_t *)pRes->pRsp, pRes->rspLen); + pMsg->affectedRows, pRes->rspLen); } else { tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); } @@ -361,7 +360,7 @@ int doProcessSql(SSqlObj *pSql) { pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || pCmd->command == TSDB_SQL_META || - pCmd->command == TSDB_SQL_METRIC) { + pCmd->command == TSDB_SQL_STABLEVGROUP) { tscBuildMsg[pCmd->command](pSql, NULL); } @@ -509,22 +508,6 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_SUCCESS; } -void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) { - //SSubmitMsg *pShellMsg; - //char * pMsg; - //STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - - //STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - - //pMsg = buf + tsRpcHeadSize; - - //TODO set iplist - //pShellMsg = (SSubmitMsg *)pMsg; - //pShellMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); - //tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pSql->index].ip), - // htons(pShellMsg->vnode)); -} - int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSubmitMsg *pShellMsg; char * pMsg, *pStart; @@ -554,24 +537,6 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_SUCCESS; } -void tscUpdateVnodeInQueryMsg(SSqlObj *pSql, char *buf) { - //TODO -// SSqlCmd * pCmd = &pSql->cmd; -// STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); -// -// char * pStart = buf + tsRpcHeadSize; -// SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; -// -// if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // pColumnModel == NULL, query on meter -// STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; -// pQueryMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode); -// } else { // query on metric -// SSuperTableMeta * pMetricMeta = pTableMetaInfo->pMetricMeta; -// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); -// pQueryMsg->vnode = htons(pVnodeSidList->vpeerDesc[pSql->index].vnode); -// } -} - /* * for meter query, simply return the size <= 1k * for metric query, estimate size according to meter tags @@ -589,7 +554,10 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize; } - + + int32_t size = 4096; + +#if 0 SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); @@ -600,45 +568,23 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { if (pQueryInfo->tsBuf != NULL) { size += pQueryInfo->tsBuf->fileSize; } - +#endif + return size; } -static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfTables, int32_t vgId, char *pMsg) { +static char *doSerializeTableInfo(SSqlObj *pSql, int32_t vgId, char *pMsg) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; - - tscTrace("%p vgId:%d, query on %d tables", pSql, vgId, numOfTables); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { -#ifdef _DEBUG_VIEW - tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pTableMetaInfo->pTableMeta->sid, pTableMetaInfo->pTableMeta->uid); -#endif - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->sid = htonl(pTableMeta->sid); - pTableIdInfo->uid = htobe64(pTableMeta->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); - pMsg += sizeof(STableIdInfo); - } else { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); - - for (int32_t i = 0; i < numOfTables; ++i) { - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - STableIdInfo *pQueryMeterInfo = tscGetMeterSidInfo(pVnodeSidList, i); - - pTableIdInfo->sid = htonl(pQueryMeterInfo->sid); - pTableIdInfo->uid = htobe64(pQueryMeterInfo->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pQueryMeterInfo->uid)); - - pMsg += sizeof(STableIdInfo); - -#ifdef _DEBUG_VIEW - tscTrace("%p sid:%d, uid:%" PRId64, pSql, pQueryMeterInfo->sid, pQueryMeterInfo->uid); -#endif - } - } - + tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, vgId, pTableMetaInfo->name, pTableMeta->uid); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->sid = htonl(pTableMeta->sid); + pTableIdInfo->uid = htobe64(pTableMeta->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); + + pMsg += sizeof(STableIdInfo); return pMsg; } @@ -655,7 +601,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; +// SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; if (pQueryInfo->colList.numOfCols <= 0) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); @@ -678,7 +624,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p error vnodeIdx:%d", pSql, pTableMetaInfo->vnodeIndex); return -1; } - + + uint32_t vnodeId = 1; + +#if 0 SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vnodeIndex); uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; @@ -687,9 +636,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscError("%p vid:%d,error numOfTables in query message:%d", pSql, vnodeId, numOfTables); return -1; // error } - +#endif + tscTrace("%p query on vid:%d, number of tables:%d", pSql, vnodeId, numOfTables); - pQueryMsg->head.vgId = htons(vnodeId); + pQueryMsg->head.vgId = htonl(vnodeId); + numOfTables = 1; } pQueryMsg->numOfTables = htonl(numOfTables); @@ -723,24 +674,23 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // query on meter pQueryMsg->tagLength = 0; } else { // query on super table - pQueryMsg->tagLength = htons(pMetricMeta->tagLen); + pQueryMsg->tagLength = htons(0); } pQueryMsg->queryType = htons(pQueryInfo->type); pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs); - if (pQueryInfo->fieldsInfo.numOfOutputCols < 0) { - tscError("%p illegal value of number of output columns in query msg: %d", pSql, - pQueryInfo->fieldsInfo.numOfOutputCols); + int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutputCols; + if (numOfOutput < 0) { + tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput); return -1; } // set column list ids - char * pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); + char *pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { @@ -848,7 +798,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->colNameLen = htonl(len); // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pSql, numOfTables, htons(pQueryMsg->head.vgId), pMsg); + pMsg = doSerializeTableInfo(pSql, htons(pQueryMsg->head.vgId), pMsg); SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr; if (pGroupbyExpr->numOfGroupCols != 0) { @@ -1443,11 +1393,15 @@ int tscProcessTagRetrieveRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int32_t numOfRes = 0; +#if 0 if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_TAGPRJ) { numOfRes = pTableMetaInfo->pMetricMeta->numOfTables; } else { numOfRes = 1; // for count function, there is only one output. } + +#endif + return tscLocalResultCommonBuilder(pSql, numOfRes); } @@ -1533,8 +1487,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(STagData); } - msgLen = pMsg - (char*)pInfoMsg; - pCmd->payloadLen = msgLen; + pCmd->payloadLen = pMsg - (char*)pInfoMsg;; pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META; tfree(tmpData); @@ -1608,7 +1561,9 @@ static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); } -int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + +#if 0 SSuperTableMetaMsg *pMetaMsg; char * pMsg, *pStart; int msgLen = 0; @@ -1671,13 +1626,13 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // convert to unicode before sending to mnode for metric query int32_t condLen = 0; if (pTagCond->numOfTagCond > 0) { - SCond *pCond = tsGetMetricQueryCondPos(pTagCond, uid); + SCond *pCond = tsGetSTableQueryCondPos(pTagCond, uid); if (pCond != NULL && pCond->cond != NULL) { condLen = strlen(pCond->cond) + 1; bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); if (!ret) { - tscError("%p mbs to ucs4 failed:%s", pSql, tsGetMetricQueryCondPos(pTagCond, uid)); + tscError("%p mbs to ucs4 failed:%s", pSql, tsGetSTableQueryCondPos(pTagCond, uid)); return 0; } } @@ -1749,7 +1704,18 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = msgLen; pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; assert(msgLen + minMsgSize() <= size); +#endif + SSqlCmd *pCmd = &pSql->cmd; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + + SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pCmd->payload; + strncpy(pStableVgroupMsg->tableId, pTableMetaInfo->name, tListLen(pStableVgroupMsg->tableId)); + + pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; + pCmd->payloadLen = sizeof(SCMSTableVgroupMsg); + return TSDB_CODE_SUCCESS; } @@ -1989,22 +1955,11 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } -int tscProcessMetricMetaRsp(SSqlObj *pSql) { - SSuperTableMeta *pMeta; - uint8_t ieType; +int tscProcessSTableVgroupRsp(SSqlObj *pSql) { +#if 0 void ** metricMetaList = NULL; int32_t * sizes = NULL; - - char *rsp = pSql->res.pRsp; - - ieType = *rsp; - if (ieType != TSDB_IE_TYPE_META) { - tscError("invalid ie type:%d", ieType); - return TSDB_CODE_INVALID_IE; - } - - rsp++; - + int32_t num = htons(*(int16_t *)rsp); rsp += sizeof(int16_t); @@ -2088,7 +2043,6 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) { // release the used metricmeta taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); - pTableMetaInfo->pMetricMeta = (SSuperTableMeta *)taosCachePut(tscCacheHandle, name, (char *)metricMetaList[i], sizes[i], tsMetricMetaKeepTimer); tfree(metricMetaList[i]); @@ -2108,7 +2062,23 @@ _error_clean: free(sizes); free(metricMetaList); - +#endif + + SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pSql->res.pRsp; + pStableVgroup->numOfDnodes = htonl(pStableVgroup->numOfDnodes); + + SSqlObj* pparent = pSql->param; + assert(pparent != NULL); + + SSqlCmd* pCmd = &pparent->cmd; + STableMetaInfo* pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + pInfo->vgroupIdList = taosArrayInit(pStableVgroup->numOfDnodes, sizeof(int32_t)); + + // todo opt performance + for(int32_t i = 0; i < pStableVgroup->numOfDnodes; ++i) { + taosArrayPush(pInfo->vgroupIdList, &pStableVgroup->dnodeIps[i]); + } + return pSql->res.code; } @@ -2234,7 +2204,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { if (pTableMetaInfo->pTableMeta) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); } return 0; @@ -2255,7 +2225,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { bool isSuperTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); if (isSuperTable) { // if it is a super table, reset whole query cache tscTrace("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); @@ -2459,30 +2429,34 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { return code; } -int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { +int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { int code = TSDB_CODE_NETWORK_UNAVAIL; SSqlCmd *pCmd = &pSql->cmd; - /* - * the query condition is serialized into pCmd->payload, we need to rebuild key for metricmeta info in cache. - */ - bool required = false; + //the query condition is serialized into pCmd->payload, we need to rebuild key for stable meta info in cache. + bool required = false; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + if (pQueryInfo->pTableMetaInfo[0]->vgroupIdList != NULL) { + return TSDB_CODE_SUCCESS; + } + +#if 0 + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid); - taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); +// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr); if (ppMeta == NULL) { required = true; break; } else { - pTableMetaInfo->pMetricMeta = ppMeta; +// pTableMetaInfo->pMetricMeta = ppMeta; } } @@ -2490,12 +2464,13 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { if (!required) { return TSDB_CODE_SUCCESS; } - +#endif + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_METRIC; + pNew->cmd.command = TSDB_SQL_STABLEVGROUP; SQueryInfo *pNewQueryInfo = NULL; if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { @@ -2532,7 +2507,7 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { // tscFreeSubqueryInfo(pCmd); // } - tscTrace("%p allocate new pSqlObj:%p to get metricMeta", pSql, pNew); + tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; code = tscProcessSql(pNew); @@ -2569,7 +2544,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg; - tscBuildMsg[TSDB_SQL_METRIC] = tscBuildMetricMetaMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; tscBuildMsg[TSDB_SQL_MULTI_META] = tscBuildMultiMeterMetaMsg; tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; @@ -2587,7 +2562,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; - tscProcessMsgRsp[TSDB_SQL_METRIC] = tscProcessMetricMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; @@ -2613,7 +2588,4 @@ void tscInitMsgsFp() { tscKeepConn[TSDB_SQL_SELECT] = 1; tscKeepConn[TSDB_SQL_FETCH] = 1; tscKeepConn[TSDB_SQL_HB] = 1; - - tscUpdateVnodeMsg[TSDB_SQL_SELECT] = tscUpdateVnodeInQueryMsg; - tscUpdateVnodeMsg[TSDB_SQL_INSERT] = tscUpdateVnodeInSubmitMsg; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index eb46d6101c..e4b07a0f64 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -546,11 +546,11 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { * if the global limitation is not reached, and current result has not exhausted, or next more vnodes are * available, goes on */ - if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows && - (!tscHasReachLimitation(pQueryInfo1, pRes1))) { - allSubqueryExhausted = false; - break; - } +// if (pMetaInfo->vnodeIndex < pMetaInfo->pMetricMeta->numOfVnodes && pRes1->row < pRes1->numOfRows && +// (!tscHasReachLimitation(pQueryInfo1, pRes1))) { +// allSubqueryExhausted = false; +// break; +// } } hasData = !allSubqueryExhausted; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 0b464c362b..4fadad5021 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -80,7 +80,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - code = tscGetMetricMeta(pSql, 0); + code = tscGetSTableVgroupInfo(pSql, 0); pSql->res.code = code; if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 014b0c5cb7..616bcbba50 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -178,11 +178,11 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); int numOfTables = 0; if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); - numOfTables += pVnodeSidList->numOfSids; - } +// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; +// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { +// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); +// numOfTables += pVnodeSidList->numOfSids; +// } } SSubscriptionProgress* progress = (SSubscriptionProgress*)calloc(numOfTables, sizeof(SSubscriptionProgress)); @@ -197,17 +197,17 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { progress[0].uid = uid; progress[0].key = tscGetSubscriptionProgress(pSub, uid); } else { - SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; - numOfTables = 0; - for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); - for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { - STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); - int64_t uid = pTableMetaInfo->uid; - progress[numOfTables].uid = uid; - progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); - } - } +// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; +// numOfTables = 0; +// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { +// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); +// for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { +// STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); +// int64_t uid = pTableMetaInfo->uid; +// progress[numOfTables].uid = uid; +// progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); +// } +// } qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1df977d996..21ce270466 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -476,7 +476,8 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; + int32_t totalVnode = 0; if ((++pTableMetaInfo->vnodeIndex) < totalVnode) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, pTableMetaInfo->vnodeIndex - 1, pTableMetaInfo->vnodeIndex, totalVnode, pRes->numOfTotal); @@ -541,16 +542,16 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode if current vnode is exhausted - if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { - pSupporter->pState->numOfCompleted = 0; - pSupporter->pState->numOfTotal = 1; - - pSql->cmd.command = TSDB_SQL_SELECT; - pSql->fp = tscJoinQueryCallback; - tscProcessSql(pSql); - - return; - } +// if ((++pTableMetaInfo->vnodeIndex) < pTableMetaInfo->pMetricMeta->numOfVnodes) { +// pSupporter->pState->numOfCompleted = 0; +// pSupporter->pState->numOfTotal = 1; +// +// pSql->cmd.command = TSDB_SQL_SELECT; +// pSql->fp = tscJoinQueryCallback; +// tscProcessSql(pSql); +// +// return; +// } } int32_t numOfTotal = pSupporter->pState->numOfTotal; @@ -608,10 +609,10 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && - (!tscHasReachLimitation(pQueryInfo, pRes))) { - numOfFetch++; - } +// if (pRes->row >= pRes->numOfRows && pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes && +// (!tscHasReachLimitation(pQueryInfo, pRes))) { +// numOfFetch++; +// } } else { if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pQueryInfo, pRes))) { numOfFetch++; @@ -788,7 +789,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { * data instead of returning to its invoker */ if (pTableMetaInfo->vnodeIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); +// assert(pTableMetaInfo->vnodeIndex < pTableMetaInfo->pMetricMeta->numOfVnodes); pSupporter->pState->numOfCompleted = 0; // reset the record value pSql->fp = joinRetrieveCallback; // continue retrieve data @@ -1009,7 +1010,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; + + int32_t numOfSubQueries = 0; +// int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; assert(numOfSubQueries > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); @@ -1260,7 +1263,8 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); +// SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); + SVnodeSidList *vnodeInfo = 0; SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; if (numOfRows > 0) { @@ -1409,10 +1413,10 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SVnodeSidList *vnodeInfo = NULL; SVnodeDesc * pSvd = NULL; - if (pTableMetaInfo->pMetricMeta != NULL) { - vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); - pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; - } +// if (pTableMetaInfo->pMetricMeta != NULL) { +// vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); +// pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; +// } SSubqueryState* pState = trsupport->pState; assert(pState->numOfCompleted < pState->numOfTotal && pState->numOfCompleted >= 0 && @@ -1453,7 +1457,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; } else { SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL); +// assert(pNewQueryInfo->pTableMetaInfo[0]->pTableMeta != NULL && pNewQueryInfo->pTableMetaInfo[0]->pMetricMeta != NULL); tscProcessSql(pNew); return; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e33fdc70a4..8df84744ce 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -53,7 +53,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - SCond* cond = tsGetMetricQueryCondPos(pTagCond, uid); + SCond* cond = tsGetSTableQueryCondPos(pTagCond, uid); char join[512] = {0}; if (pTagCond->joinInfo.hasJoin) { @@ -92,7 +92,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { free(tmp); } -SCond* tsGetMetricQueryCondPos(STagCond* pTagCond, uint64_t uid) { +SCond* tsGetSTableQueryCondPos(STagCond* pTagCond, uint64_t uid) { for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { if (uid == pTagCond->cond[i].uid) { return &pTagCond->cond[i]; @@ -122,7 +122,7 @@ bool tscQueryOnMetric(SSqlCmd* pCmd) { (pCmd->msgType == TSDB_MSG_TYPE_QUERY); } -bool tscQueryMetricTags(SQueryInfo* pQueryInfo) { +bool tscQueryTags(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) { return false; @@ -172,6 +172,7 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) { } SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx) { +#if 0 if (pMetricmeta == NULL) { tscError("illegal metricmeta"); return 0; @@ -189,6 +190,8 @@ SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx } return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta); +#endif + } STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { @@ -221,12 +224,12 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { // for select query super table, the metricmeta can not be null in any cases. if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - assert(pTableMetaInfo->pMetricMeta != NULL); +// assert(pTableMetaInfo->pMetricMeta != NULL); } - if (pTableMetaInfo->pMetricMeta == NULL) { - return false; - } +// if (pTableMetaInfo->pMetricMeta == NULL) { +// return false; +// } if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { return false; @@ -259,12 +262,12 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // only query on tag, not a projection query - if (tscQueryMetricTags(pQueryInfo)) { + if (tscQueryTags(pQueryInfo)) { return false; } // for project query, only the following two function is allowed - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { @@ -1925,7 +1928,7 @@ STableMetaInfo* tscAddMeterMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->pMetricMeta = pMetricMeta; +// pTableMetaInfo->pMetricMeta = pMetricMeta; pTableMetaInfo->numOfTags = numOfTags; if (tags != NULL) { @@ -1975,7 +1978,7 @@ void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) } taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); +// taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -2114,15 +2117,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMeta* pPrevMeterMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); - SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); +// SSuperTableMeta* pPrevMetricMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pMetricMeta); - pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, - pTableMetaInfo->tagColumnIndex); +// pFinalInfo = tscAddMeterMetaInfo(pNewQueryInfo, name, pPrevMeterMeta, pPrevMetricMeta, pTableMetaInfo->numOfTags, +// pTableMetaInfo->tagColumnIndex); } assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1); if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - assert(pFinalInfo->pMetricMeta != NULL); +// assert(pFinalInfo->pMetricMeta != NULL); } tscTrace( @@ -2222,13 +2225,13 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { +// if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { return false; - } +// } - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; - return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1); +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; +// return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && +// (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vnodeIndex < totalVnode - 1); } void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { @@ -2244,7 +2247,8 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; + int32_t totalVnode = 0; +// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; while (++pTableMetaInfo->vnodeIndex < totalVnode) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index a23336630e..965727c9d0 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -261,7 +261,7 @@ static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { SQInfo* pQInfo = NULL; if (pMsg->contLen != 0) { void* tsdb = dnodeGetVnodeTsdb(pVnode); - int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, NULL, &pQInfo); + int32_t code = qCreateQueryInfo(tsdb, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 838b43da1e..1345d80fe5 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -28,83 +28,83 @@ extern "C" { #include "trpc.h" // message type -#define TSDB_MSG_TYPE_REG 1 -#define TSDB_MSG_TYPE_REG_RSP 2 -#define TSDB_MSG_TYPE_SUBMIT 3 -#define TSDB_MSG_TYPE_SUBMIT_RSP 4 -#define TSDB_MSG_TYPE_QUERY 5 -#define TSDB_MSG_TYPE_QUERY_RSP 6 -#define TSDB_MSG_TYPE_RETRIEVE 7 -#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 +#define TSDB_MSG_TYPE_REG 1 +#define TSDB_MSG_TYPE_REG_RSP 2 +#define TSDB_MSG_TYPE_SUBMIT 3 +#define TSDB_MSG_TYPE_SUBMIT_RSP 4 +#define TSDB_MSG_TYPE_QUERY 5 +#define TSDB_MSG_TYPE_QUERY_RSP 6 +#define TSDB_MSG_TYPE_RETRIEVE 7 +#define TSDB_MSG_TYPE_RETRIEVE_RSP 8 // message from mnode to dnode -#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 +#define TSDB_MSG_TYPE_MD_CREATE_TABLE 9 #define TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP 10 -#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 -#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 -#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 -#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 +#define TSDB_MSG_TYPE_MD_DROP_TABLE 11 +#define TSDB_MSG_TYPE_MD_DROP_TABLE_RSP 12 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE 13 +#define TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP 14 +#define TSDB_MSG_TYPE_MD_CREATE_VNODE 15 #define TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP 16 -#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 -#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 -#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 -#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 -#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 -#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 +#define TSDB_MSG_TYPE_MD_DROP_VNODE 17 +#define TSDB_MSG_TYPE_MD_DROP_VNODE_RSP 18 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE 19 +#define TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP 20 +#define TSDB_MSG_TYPE_MD_DROP_STABLE 21 +#define TSDB_MSG_TYPE_MD_DROP_STABLE_RSP 22 +#define TSDB_MSG_TYPE_MD_ALTER_STREAM 23 #define TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP 24 -#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 +#define TSDB_MSG_TYPE_MD_CONFIG_DNODE 25 #define TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP 26 // message from client to mnode -#define TSDB_MSG_TYPE_CM_CONNECT 31 -#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 -#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 -#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 -#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 -#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 -#define TSDB_MSG_TYPE_CM_CREATE_USER 39 -#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 -#define TSDB_MSG_TYPE_CM_ALTER_USER 41 -#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 -#define TSDB_MSG_TYPE_CM_DROP_USER 43 -#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 -#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 +#define TSDB_MSG_TYPE_CM_CONNECT 31 +#define TSDB_MSG_TYPE_CM_CONNECT_RSP 32 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT 33 +#define TSDB_MSG_TYPE_CM_CREATE_ACCT_RSP 34 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT 35 +#define TSDB_MSG_TYPE_CM_ALTER_ACCT_RSP 36 +#define TSDB_MSG_TYPE_CM_DROP_ACCT 37 +#define TSDB_MSG_TYPE_CM_DROP_ACCT_RSP 38 +#define TSDB_MSG_TYPE_CM_CREATE_USER 39 +#define TSDB_MSG_TYPE_CM_CREATE_USER_RSP 40 +#define TSDB_MSG_TYPE_CM_ALTER_USER 41 +#define TSDB_MSG_TYPE_CM_ALTER_USER_RSP 42 +#define TSDB_MSG_TYPE_CM_DROP_USER 43 +#define TSDB_MSG_TYPE_CM_DROP_USER_RSP 44 +#define TSDB_MSG_TYPE_CM_CREATE_DNODE 45 #define TSDB_MSG_TYPE_CM_CREATE_DNODE_RSP 46 -#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 -#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 -#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE +#define TSDB_MSG_TYPE_CM_DROP_DNODE 47 +#define TSDB_MSG_TYPE_CM_DROP_DNODE_RSP 48 +#define TSDB_MSG_TYPE_CM_CONFIG_DNODE TSDB_MSG_TYPE_MD_CONFIG_DNODE #define TSDB_MSG_TYPE_CM_CONFIG_DNODE_RSP TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP -#define TSDB_MSG_TYPE_CM_CREATE_DB 49 -#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 -#define TSDB_MSG_TYPE_CM_DROP_DB 51 -#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 -#define TSDB_MSG_TYPE_CM_USE_DB 53 -#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 -#define TSDB_MSG_TYPE_CM_ALTER_DB 55 -#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 -#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 +#define TSDB_MSG_TYPE_CM_CREATE_DB 49 +#define TSDB_MSG_TYPE_CM_CREATE_DB_RSP 50 +#define TSDB_MSG_TYPE_CM_DROP_DB 51 +#define TSDB_MSG_TYPE_CM_DROP_DB_RSP 52 +#define TSDB_MSG_TYPE_CM_USE_DB 53 +#define TSDB_MSG_TYPE_CM_USE_DB_RSP 54 +#define TSDB_MSG_TYPE_CM_ALTER_DB 55 +#define TSDB_MSG_TYPE_CM_ALTER_DB_RSP 56 +#define TSDB_MSG_TYPE_CM_CREATE_TABLE 57 #define TSDB_MSG_TYPE_CM_CREATE_TABLE_RSP 58 -#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 -#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 -#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 -#define TSDB_MSG_TYPE_CM_TABLE_META 63 -#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 -#define TSDB_MSG_TYPE_CM_STABLE_VGROUP 65 +#define TSDB_MSG_TYPE_CM_DROP_TABLE 59 +#define TSDB_MSG_TYPE_CM_DROP_TABLE_RSP 60 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE 61 +#define TSDB_MSG_TYPE_CM_ALTER_TABLE_RSP 62 +#define TSDB_MSG_TYPE_CM_TABLE_META 63 +#define TSDB_MSG_TYPE_CM_TABLE_META_RSP 64 +#define TSDB_MSG_TYPE_CM_STABLE_VGROUP 65 #define TSDB_MSG_TYPE_CM_STABLE_VGROUP_RSP 66 -#define TSDB_MSG_TYPE_CM_TABLES_META 67 -#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 -#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 +#define TSDB_MSG_TYPE_CM_TABLES_META 67 +#define TSDB_MSG_TYPE_CM_TABLES_META_RSP 68 +#define TSDB_MSG_TYPE_CM_ALTER_STREAM 69 #define TSDB_MSG_TYPE_CM_ALTER_STREAM_RSP 70 -#define TSDB_MSG_TYPE_CM_SHOW 71 -#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 -#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 -#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 -#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 +#define TSDB_MSG_TYPE_CM_SHOW 71 +#define TSDB_MSG_TYPE_CM_SHOW_RSP 72 +#define TSDB_MSG_TYPE_CM_KILL_QUERY 73 +#define TSDB_MSG_TYPE_CM_KILL_QUERY_RSP 74 +#define TSDB_MSG_TYPE_CM_KILL_STREAM 75 #define TSDB_MSG_TYPE_CM_KILL_STREAM_RSP 76 #define TSDB_MSG_TYPE_CM_KILL_CONN 77 #define TSDB_MSG_TYPE_CM_KILL_CONN_RSP 78 @@ -622,14 +622,14 @@ typedef struct { char tableIds[]; } SCMMultiTableInfoMsg; -typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; -} SCMSuperTableInfoMsg; +typedef struct SCMSTableVgroupMsg { + char tableId[TSDB_TABLE_ID_LEN]; +} SCMSTableVgroupMsg; typedef struct { int32_t numOfDnodes; uint32_t dnodeIps[]; -} SCMSuperTableInfoRsp; +} SCMSTableVgroupRspMsg; typedef struct { int16_t elemLen; diff --git a/src/mnode/src/mgmtSuperTable.c b/src/mnode/src/mgmtSuperTable.c index 44e8014dad..306d414cbc 100644 --- a/src/mnode/src/mgmtSuperTable.c +++ b/src/mnode/src/mgmtSuperTable.c @@ -216,7 +216,7 @@ void* mgmtGetSuperTable(char *tableId) { } static void *mgmtGetSuperTableVgroup(SSuperTableObj *pStable) { - SCMSuperTableInfoRsp *rsp = rpcMallocCont(sizeof(SCMSuperTableInfoRsp) + sizeof(uint32_t) * mgmtGetDnodesNum()); + SCMSTableVgroupRspMsg *rsp = rpcMallocCont(sizeof(SCMSTableVgroupRspMsg) + sizeof(uint32_t) * mgmtGetDnodesNum()); rsp->numOfDnodes = htonl(1); rsp->dnodeIps[0] = htonl(inet_addr(tsPrivateIp)); return rsp; @@ -628,14 +628,14 @@ void mgmtGetSuperTableMeta(SQueuedMsg *pMsg, SSuperTableObj *pTable) { } static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { - SCMSuperTableInfoMsg *pInfo = pMsg->pCont; + SCMSTableVgroupMsg *pInfo = pMsg->pCont; STableInfo *pTable = mgmtGetSuperTable(pInfo->tableId); if (pTable == NULL) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); return; } - SCMSuperTableInfoRsp *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); + SCMSTableVgroupRspMsg *pRsp = mgmtGetSuperTableVgroup((SSuperTableObj *) pTable); if (pRsp != NULL) { int32_t msgLen = sizeof(SSuperTableObj) + htonl(pRsp->numOfDnodes) * sizeof(int32_t); SRpcMsg rpcRsp = {0}; diff --git a/src/query/inc/qsqltype.h b/src/query/inc/qsqltype.h index 5d85ab4b80..ad9affa1cc 100644 --- a/src/query/inc/qsqltype.h +++ b/src/query/inc/qsqltype.h @@ -54,7 +54,7 @@ enum _sql_type { TSDB_SQL_CONNECT, TSDB_SQL_USE_DB, // 30 TSDB_SQL_META, - TSDB_SQL_METRIC, + TSDB_SQL_STABLEVGROUP, TSDB_SQL_MULTI_META, TSDB_SQL_HB, diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 16f2006d6f..dc7aff105f 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -203,7 +203,7 @@ typedef struct SQInfo { * @param pQInfo * @return */ -int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, void* param, SQInfo** pQInfo); +int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, SQInfo** pQInfo); /** * query on single table @@ -211,12 +211,6 @@ int32_t qCreateQueryInfo(void* pVnode, SQueryTableMsg* pQueryTableMsg, void* par */ void qTableQuery(SQInfo* pQInfo); -/** - * query on super table - * @param pReadMsg - */ -void qSuperTableQuery(void* pReadMsg); - /** * wait for the query completed, and retrieve final results to client * @param pQInfo diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e9923c6fd8..a1167ff440 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5128,137 +5128,99 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { // pQInfo->size - pQInfo->pointsInterpo, pQInfo->pointsInterpo, pQInfo->pointsReturned); } -void qTableQuery(SQInfo *pQInfo) { - if (pQInfo == NULL || pQInfo->signature != pQInfo) { - dTrace("%p freed abort query", pQInfo); - return; - } +static void singleTableQueryImpl(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - SQuery *pQuery = pRuntimeEnv->pQuery; - if (isQueryKilled(pQInfo)) { - dTrace("QInfo:%p it is already killed, abort", pQInfo); - return; - } - - dTrace("QInfo:%p query task is launched", pQInfo); - if (vnodeHasRemainResults(pQInfo)) { /* * There are remain results that are not returned due to result interpolation * So, we do keep in this procedure instead of launching retrieve procedure for next results. */ int32_t numOfInterpo = 0; - int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); pQuery->rec.size = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, - (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); - + (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); + doRevisedResultsByLimit(pQInfo); - + pQInfo->pointsInterpo += numOfInterpo; pQuery->rec.size += pQuery->rec.size; - - // dTrace("QInfo:%p %d points returned %d points interpo, totalRead:%d totalInterpo:%d totalReturn:%d", - // pQInfo, pQuery->size, numOfInterpo, pQInfo->size, pQInfo->pointsInterpo, - // pQInfo->pointsReturned); + + dTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } - + // here we have scan all qualified data in both data file and cache if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - (pQuery->intervalTime > 0 && pQuery->rec.total < pQuery->limit.limit)) { + ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { + // todo limit the output for interval query? pQuery->rec.size = 0; pQInfo->subgroupIdx = 0; // always start from 0 - + if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.size += pQuery->rec.size; - + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->subgroupIdx); - + if (pQuery->rec.size > 0) { - // dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d - // totalReturn:%d", - // pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->size, - // pQInfo->size, pQInfo->pointsInterpo, pQInfo->pointsReturned); - + dTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } } } - - // dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, - // pMeterObj->sid, - // pMeterObj->meterId, pQInfo->size); - + + dTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); return; } - + // number of points returned during this query pQuery->rec.size = 0; - int64_t st = taosGetTimestampUs(); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) - // assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); tableIntervalProcessor(pQInfo); } else { if (isFixedOutputQuery(pQuery)) { assert(pQuery->checkBufferInLoop == 0); - + tableFixedOutputProcessor(pQInfo); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBufferInLoop == 1); tableMultiOutputProcessor(pQInfo); } } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); - + /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { dTrace("QInfo:%p query is killed", pQInfo); } else { dTrace("QInfo:%p query task completed, %d points are returned", pQInfo, pQuery->rec.size); } - + sem_post(&pQInfo->dataReady); - // vnodeDecRefCount(pQInfo); } -void qSuperTableQuery(void *pReadMsg) { -// SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; -// -// if (pQInfo == NULL) { -// return; -// } - -// if (pQInfo->killed) { -// vnodeDecRefCount(pQInfo); -// dTrace("QInfo:%p it is already killed, abort", pQInfo); -// return; -// } - -// assert(pQInfo->refCount >= 1); -#if 0 - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; +void multiTableQueryImpl(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); + if (pQuery->intervalTime > 0 || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { - assert(pQuery->checkBufferInLoop == 0); vnodeMultiMeterQueryProcessor(pQInfo); } else { assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || @@ -5267,23 +5229,40 @@ void qSuperTableQuery(void *pReadMsg) { vnodeSTableSeqProcessor(pQInfo); } - /* record the total elapsed time */ + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); - pQuery->status = isQueryKilled(pQInfo) ? 1 : 0; - -// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, -// pQInfo->query.interpoType); +// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); if (pQuery->rec.size == 0) { -// pQInfo->over = 1; -// dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, -// pQInfo->size); + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + dTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, numOfTables, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); } sem_post(&pQInfo->dataReady); -// vnodeDecRefCount(pQInfo); -#endif +} + +void qTableQuery(SQInfo *pQInfo) { + if (pQInfo == NULL || pQInfo->signature != pQInfo) { + dTrace("%p freed abort query", pQInfo); + return; + } + + if (isQueryKilled(pQInfo)) { + dTrace("QInfo:%p it is already killed, abort", pQInfo); + return; + } + + dTrace("QInfo:%p query task is launched", pQInfo); + + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + if (numOfTables == 1) { + singleTableQueryImpl(pQInfo); + } else { + multiTableQueryImpl(pQInfo); + } + + // vnodeDecRefCount(pQInfo); } static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { @@ -6064,7 +6043,7 @@ _error: return code; } -int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, void* param, SQInfo **pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { assert(pQueryTableMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -6098,12 +6077,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, void* param goto _query_over; } - if (QUERY_IS_STABLE_QUERY(pQueryTableMsg->queryType)) { - // pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryTableMsg, &code); - } else { - code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); -// (*pQInfo)->param = param; - } + code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); _query_over: if (code != TSDB_CODE_SUCCESS) { diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 72c1667fe1..397807f547 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -373,8 +373,27 @@ static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable) { return 0; } static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { - assert(pTable->type == TSDB_CHILD_TABLE); - // TODO + assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); + STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); + assert(pSTable != NULL); + + int32_t level = 0; + int32_t headSize = 0; + + // first tag column + STColumn* s = pSTable->tagSchema->columns[0]; //??? + + tSkipListRandNodeInfo(pSTable->pIndex, &level, &headSize); + SSkipListNode* pNode = calloc(1, headSize + s->bytes + POINTER_BYTES); + pNode->level = level; + + SSkipList* list = pSTable->pIndex; + + memcpy(SL_GET_NODE_KEY(list, pNode), dataRowTuple(pTable->tagVal), s->columns[0].bytes); + memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES); + + tSkipListPut(list, pNode); + return 0; } diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 2dc61f5107..fc196bb62f 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -275,8 +275,9 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond STableIdInfo* idInfo = taosArrayGet(pQueryHandle->pTableIdList, 0); - STableId tableId = {.uid = idInfo->uid, .tid = idInfo->sid}; - STable *pTable = tsdbIsValidTableToInsert(tsdbGetMeta(pQueryHandle->pTsdb), tableId); + STable *pTable = tsdbGetTableByUid(tsdbGetMeta(pQueryHandle->pTsdb), idInfo->uid); + assert(pTable != NULL); + pTableQRec->pTableObj = pTable; // malloc buffer in order to load data from file From b01e8b8d111991615ff5f587a04b4c8eea1619fb Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 25 Mar 2020 18:00:18 +0800 Subject: [PATCH 54/85] TD-34 --- src/common/inc/dataformat.h | 20 +++++++++++--- src/common/src/dataformat.c | 51 +++++++++++++++++++++++++++++++---- src/vnode/tsdb/src/tsdbMain.c | 28 ++++++++++++++++--- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 1f46c68abc..207ff4dbfd 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -105,11 +105,25 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int64_t len; - char data[]; + int32_t len; + void * pData; } SDataCol; -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter); +typedef struct { + TSKEY firstKey; + TSKEY lastKey; + int numOfPoints; + int numOfCols; + void * buf; + SDataCol cols[]; +} SDataCols; + +#define keyCol(cols) (&((cols)->cols[0])) // Key column + +SDataCols *tdNewDataCols(STSchema *pSchema, int nRows); +void tdFreeDataCols(SDataCols *pCols); +void tdResetDataCols(SDataCols *pCols); +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 04826e43ac..3c692f9eba 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -294,14 +294,55 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } -void tdConvertDataRowToCol(SDataCol *cols, STSchema *pSchema, int *iter) { - int row = *iter; +SDataCols *tdNewDataCols(STSchema *pSchema, int nRows) { + int nCols = schemaNCols(pSchema); - for (int i = 0; i < schemaNCols(pSchema); i++) { - // TODO + SDataCols *pInfo = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * nCols); + if (pInfo == NULL) return NULL; + + pInfo->numOfCols = nCols; + pInfo->firstKey = INT64_MIN; + pInfo->lastKey = INT64_MAX; + pInfo->buf = malloc(tdMaxRowBytesFromSchema(pSchema) * nRows); + if (pInfo->buf == NULL) { + free(pInfo); + return NULL; } - *iter = row + 1; + pInfo->cols[0].pData = pInfo->buf; + for (int i = 1; i < nCols; i++) { + pInfo->cols[i].pData = (char *)(pInfo->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * nRows; + } + + return pInfo; +} + +void tdFreeDataCols(SDataCols *pCols) { + if (pCols) { + if (pCols->buf) free(pCols->buf); + free(pCols); + } +} + +void tdResetDataCols(SDataCols *pCols) { + pCols->firstKey = INT64_MAX; + pCols->lastKey = INT64_MIN; + pCols->numOfPoints = 0; + for (int i = 0; i < pCols->numOfCols; i++) { + pCols->cols[i].len = 0; + } +} + +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema) { + TSKEY key = dataRowKey(row); + if (pCols->numOfPoints == 0) pCols->firstKey = key; + pCols->lastKey = key; + for (int i = 0; i < pCols->numOfCols; i++) { + SDataCol *pCol = pCols->cols + i; + memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, colOffset(schemaColAt(pSchema, i))), + colBytes(schemaColAt(pSchema, i))); + pCol->len += colBytes(schemaColAt(pSchema, i)); + } } /** diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index af3a923d90..7b1836378d 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -84,7 +84,7 @@ static int tsdbOpenMetaFile(char *tsdbDir); static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); -static void * tsdbCommitToFile(void *arg); +static void * tsdbCommitData(void *arg); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -327,7 +327,7 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { pRepo->tsdbCache->curBlock = NULL; // TODO: here should set as detached or use join for memory leak - pthread_create(&(pRepo->commitThread), NULL, tsdbCommitToFile, (void *)repo); + pthread_create(&(pRepo->commitThread), NULL, tsdbCommitData, (void *)repo); tsdbUnLockRepo(repo); return 0; @@ -816,7 +816,7 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) } // Commit to file -static void *tsdbCommitToFile(void *arg) { +static void *tsdbCommitData(void *arg) { // TODO printf("Starting to commit....\n"); STsdbRepo * pRepo = (STsdbRepo *)arg; @@ -894,4 +894,26 @@ static void *tsdbCommitToFile(void *arg) { tsdbUnLockRepo(arg); return NULL; +} + +static int tsdbCommitToFile(STsdbRepo *pRepo, SSkipListIterator **iters, int fid) { + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &pRepo->config; + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + + for (int tid = 0; tid < pCfg->maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + SSkipListIterator *pIter = iters[tid]; + + if (pIter == NULL) continue; + + // Read data + // while () { + + // } + } + + return 0; } \ No newline at end of file From 492ff11f5a1bf61b4092c1c9236c259830af7515 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 25 Mar 2020 22:49:54 +0800 Subject: [PATCH 55/85] TD-34 --- src/common/inc/dataformat.h | 20 +++++++++++------- src/common/src/dataformat.c | 41 ++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 207ff4dbfd..37a3ccea5a 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -105,24 +105,30 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int32_t len; - void * pData; + int8_t type; + int bytes; + int len; + void * pData; } SDataCol; typedef struct { - TSKEY firstKey; - TSKEY lastKey; + int maxRowSize; + int maxCols; // max number of columns + int maxPoints; // max number of points int numOfPoints; - int numOfCols; + int numOfCols; // Total number of cols void * buf; SDataCol cols[]; } SDataCols; #define keyCol(cols) (&((cols)->cols[0])) // Key column +#define dataColsKeyFirst(cols) ((int64_t *)(keyCol(cols)->pData))[0] +#define dataColsKeyLast(cols) ((int64_t *)(keyCol(cols)->pData))[(cols)->numOfPoints - 1] -SDataCols *tdNewDataCols(STSchema *pSchema, int nRows); -void tdFreeDataCols(SDataCols *pCols); +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema); #ifdef __cplusplus diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 3c692f9eba..84ac1efd71 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -294,27 +294,34 @@ SDataRow tdDataRowDup(SDataRow row) { return trow; } -SDataCols *tdNewDataCols(STSchema *pSchema, int nRows) { - int nCols = schemaNCols(pSchema); +SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { + SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * maxCols); + if (pCols == NULL) return NULL; - SDataCols *pInfo = (SDataCols *)calloc(1, sizeof(SDataCols) + sizeof(SDataCol) * nCols); - if (pInfo == NULL) return NULL; + pCols->maxRowSize = maxRowSize; + pCols->maxCols = maxCols; + pCols->maxPoints = maxRows; - pInfo->numOfCols = nCols; - pInfo->firstKey = INT64_MIN; - pInfo->lastKey = INT64_MAX; - pInfo->buf = malloc(tdMaxRowBytesFromSchema(pSchema) * nRows); - if (pInfo->buf == NULL) { - free(pInfo); + pCols->buf = malloc(maxRowSize * maxRows); + if (pCols->buf == NULL) { + free(pCols); return NULL; } - pInfo->cols[0].pData = pInfo->buf; - for (int i = 1; i < nCols; i++) { - pInfo->cols[i].pData = (char *)(pInfo->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * nRows; + return pCols; +} + +void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { + // assert(schemaNCols(pSchema) <= pCols->numOfCols); + tdResetDataCols(pCols); + pCols->numOfCols = schemaNCols(pSchema); + + pCols->cols[0].pData = pCols->buf; + for (int i = 1; i < schemaNCols(pSchema); i++) { + pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; } - return pInfo; + return pCols; } void tdFreeDataCols(SDataCols *pCols) { @@ -325,18 +332,14 @@ void tdFreeDataCols(SDataCols *pCols) { } void tdResetDataCols(SDataCols *pCols) { - pCols->firstKey = INT64_MAX; - pCols->lastKey = INT64_MIN; pCols->numOfPoints = 0; - for (int i = 0; i < pCols->numOfCols; i++) { + for (int i = 0; i < pCols->maxCols; i++) { pCols->cols[i].len = 0; } } void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema) { TSKEY key = dataRowKey(row); - if (pCols->numOfPoints == 0) pCols->firstKey = key; - pCols->lastKey = key; for (int i = 0; i < pCols->numOfCols; i++) { SDataCol *pCol = pCols->cols + i; memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, colOffset(schemaColAt(pSchema, i))), From 0d8cc7972f4513a9823b11f2c375e7a82d5fb9a8 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Wed, 25 Mar 2020 23:23:37 +0800 Subject: [PATCH 56/85] TD-34 --- src/common/inc/dataformat.h | 3 +- src/common/src/dataformat.c | 17 ++++++--- src/vnode/tsdb/src/tsdbMain.c | 69 ++++++++++------------------------- 3 files changed, 32 insertions(+), 57 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 37a3ccea5a..e123efd11e 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -108,6 +108,7 @@ typedef struct SDataCol { int8_t type; int bytes; int len; + int offset; void * pData; } SDataCol; @@ -129,7 +130,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); void tdFreeDataCols(SDataCols *pCols); -void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema); +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 84ac1efd71..da55663d0b 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -317,8 +317,13 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { pCols->numOfCols = schemaNCols(pSchema); pCols->cols[0].pData = pCols->buf; - for (int i = 1; i < schemaNCols(pSchema); i++) { - pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; + for (int i = 0; i < schemaNCols(pSchema); i++) { + if (i > 0) { + pCols->cols[i].pData = (char *)(pCols->cols[i - 1].pData) + schemaColAt(pSchema, i - 1)->bytes * pCols->maxPoints; + } + pCols->cols[i].type = colType(schemaColAt(pSchema, i)); + pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i)); + pCols->cols[i].offset = colOffset(schemaColAt(pSchema, i)); } return pCols; @@ -338,14 +343,14 @@ void tdResetDataCols(SDataCols *pCols) { } } -void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols, STSchema *pSchema) { +void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) { TSKEY key = dataRowKey(row); for (int i = 0; i < pCols->numOfCols; i++) { SDataCol *pCol = pCols->cols + i; - memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, colOffset(schemaColAt(pSchema, i))), - colBytes(schemaColAt(pSchema, i))); - pCol->len += colBytes(schemaColAt(pSchema, i)); + memcpy((void *)((char *)(pCol->pData) + pCol->len), dataRowAt(row, pCol->offset), pCol->bytes); + pCol->len += pCol->bytes; } + pCols->numOfPoints++; } /** diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 7b1836378d..a8e04e216a 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -85,6 +85,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -761,7 +762,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { return 0; } -static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCol **cols, STSchema *pSchema) { +static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { int numOfRows = 0; do { SSkipListNode *node = tSkipListIterGet(pIter); @@ -769,12 +770,8 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max SDataRow row = SL_GET_NODE_DATA(node); if (dataRowKey(row) > maxKey) break; - // Convert row data to column data - // for (int i = 0; i < schemaNCols(pSchema); i++) { - // STColumn *pCol = schemaColAt(pSchema, i); - // memcpy(cols[i]->data + TYPE_BYTES[colType(pCol)] * numOfRows, dataRowAt(row, pCol->offset), - // TYPE_BYTES[colType(pCol)]); - // } + + tdAppendDataRowToDataCol(row, pCols); numOfRows++; if (numOfRows > maxRowsToRead) break; @@ -823,7 +820,7 @@ static void *tsdbCommitData(void *arg) { STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCache *pCache = pRepo->tsdbCache; STsdbCfg * pCfg = &(pRepo->config); - if (pCache->imem == NULL) return; + if (pCache->imem == NULL) return NULL; // Create the iterator to read from cache SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); @@ -832,52 +829,23 @@ static void *tsdbCommitData(void *arg) { return NULL; } - int maxCols = pMeta->maxCols; - int maxBytes = pMeta->maxRowBytes; - SDataCol **cols = (SDataCol **)malloc(sizeof(SDataCol *) * maxCols); - void *buf = malloc((maxBytes + sizeof(SDataCol)) * pCfg->maxRowsPerFileBlock); + // Create a data column buffer for commit + SDataCols *pCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock); + if (pCols == NULL) { + // TODO: deal with the error + return NULL; + } int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); for (int fid = sfid; fid <= efid; fid++) { - TSKEY minKey = 0, maxKey = 0; - tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - - // tsdbOpenFileForWrite(pRepo, fid); - - for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL || pTable->imem == NULL) continue; - if (iters[tid] == NULL) { // create table iterator - iters[tid] = tSkipListCreateIter(pTable->imem->pData); - // TODO: deal with the error - if (iters[tid] == NULL) break; - if (!tSkipListIterNext(iters[tid])) { - // assert(0); - } - } - - // Init row data part - cols[0] = (SDataCol *)buf; - for (int col = 1; col < schemaNCols(pTable->schema); col++) { - cols[col] = (SDataCol *)((char *)(cols[col - 1]) + sizeof(SDataCol) + colBytes(schemaColAt(pTable->schema, col-1)) * pCfg->maxRowsPerFileBlock); - } - - // Loop the iterator - int rowsRead = 0; - while ((rowsRead = tsdbReadRowsFromCache(iters[tid], maxKey, pCfg->maxRowsPerFileBlock, cols, pTable->schema)) > - 0) { - // printf("rowsRead:%d-----------\n", rowsRead); - int k = 0; - } - } + tsdbCommitToFile(pRepo, fid, iters, pCols); } + tdFreeDataCols(pCols); tsdbDestroyTableIters(iters, pCfg->maxTables); - free(buf); - free(cols); tsdbLockRepo(arg); tdListMove(pCache->imem->list, pCache->pool.memPool); @@ -896,7 +864,7 @@ static void *tsdbCommitData(void *arg) { return NULL; } -static int tsdbCommitToFile(STsdbRepo *pRepo, SSkipListIterator **iters, int fid) { +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; @@ -908,11 +876,12 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, SSkipListIterator **iters, int fid SSkipListIterator *pIter = iters[tid]; if (pIter == NULL) continue; + tdInitDataCols(pCols, pTable->schema); - // Read data - // while () { - - // } + while (tsdbReadRowsFromCache(pIter, maxKey, pCfg->maxRowsPerFileBlock, pCols)) { + // TODO + int k = 0; + } } return 0; From cc543aaf50cb312c943add51357746cb8c99e549 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 26 Mar 2020 23:26:48 +0800 Subject: [PATCH 57/85] TD-34 --- src/common/inc/dataformat.h | 17 +++--- src/common/src/dataformat.c | 1 + src/util/inc/tutil.h | 8 +++ src/util/src/tutil.c | 69 +++++++++++++++++++++ src/vnode/tsdb/inc/tsdbFile.h | 30 ++++++---- src/vnode/tsdb/src/tsdbFile.c | 107 +++++++++++++++++++++++++++++++-- src/vnode/tsdb/src/tsdbMain.c | 110 ++++++++++++++++++++++++++++++++-- 7 files changed, 314 insertions(+), 28 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index e123efd11e..52b2d1e156 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -105,11 +105,12 @@ SDataRow tdDataRowDup(SDataRow row); // ----------------- Data column structure typedef struct SDataCol { - int8_t type; - int bytes; - int len; - int offset; - void * pData; + int8_t type; + int16_t colId; + int bytes; + int len; + int offset; + void * pData; } SDataCol; typedef struct { @@ -122,9 +123,9 @@ typedef struct { SDataCol cols[]; } SDataCols; -#define keyCol(cols) (&((cols)->cols[0])) // Key column -#define dataColsKeyFirst(cols) ((int64_t *)(keyCol(cols)->pData))[0] -#define dataColsKeyLast(cols) ((int64_t *)(keyCol(cols)->pData))[(cols)->numOfPoints - 1] +#define keyCol(pCols) (&((pCols)->cols[0])) // Key column +#define dataColsKeyFirst(pCols) ((int64_t *)(keyCol(pCols)->pData))[0] +#define dataColsKeyLast(pCols) ((int64_t *)(keyCol(pCols)->pData))[(pCols)->numOfPoints - 1] SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index da55663d0b..8f6a40805f 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -324,6 +324,7 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { pCols->cols[i].type = colType(schemaColAt(pSchema, i)); pCols->cols[i].bytes = colBytes(schemaColAt(pSchema, i)); pCols->cols[i].offset = colOffset(schemaColAt(pSchema, i)); + pCols->cols[i].colId = colColId(schemaColAt(pSchema, i)); } return pCols; diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index df97dde5ac..b80aad1ceb 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -176,6 +176,14 @@ uint32_t ip2uint(const char *const ip_addr); void taosSetAllocMode(int mode, const char* path, bool autoDump); void taosDumpMemoryLeak(); +#define TD_EQ 0x1 +#define TD_GT 0x2 +#define TD_LT 0x4 +#define TD_GE (TD_EQ | TD_GT) +#define TD_LE (TD_EQ | TD_LT) +void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, + int (*compar)(const void *, const void *), int flags); + #ifdef TAOS_MEM_CHECK void * taos_malloc(size_t size, const char *file, uint32_t line); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 9c384b25ba..cbd08954cc 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -617,3 +617,72 @@ char *taosCharsetReplace(char *charsetstr) { return strdup(charsetstr); } + +#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx)) +void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *), int flags) { + // TODO: need to check the correctness of this function + int l = 0; + int r = nmemb; + int idx = 0; + int comparison; + + if (flags == TD_EQ) { + return bsearch(key, base, nmemb, size, compar); + } else if (flags == TD_GE) { + if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0); + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx) < 0)) { + return elePtrAt(base, size, idx); + } else { + if (idx + 1 > nmemb - 1) { + return NULL; + } else { + return elePtrAt(base, size, idx + 1); + } + } + } else if (flags == TD_LE) { + if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1); + if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL; + + while (l < r) { + idx = (l + r) / 2; + comparison = (*compar)(key, elePtrAt(base, size, idx)); + if (comparison < 0) { + r = idx; + } else if (comparison > 0) { + l = idx + 1; + } else { + return elePtrAt(base, size, idx); + } + } + + if ((*compar)(key, elePtrAt(base, size, idx)) > 0) { + return elePtrAt(base, size, idx); + } else { + if (idx == 0) { + return NULL; + } else { + return elePtrAt(base, size, idx - 1); + } + } + + } else { + assert(0); + return NULL; + } + + return NULL; +} diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 9a4d94c58f..22563275cd 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -17,6 +17,7 @@ #include +#include "dataformat.h" #include "taosdef.h" #include "tglobalcfg.h" @@ -69,20 +70,26 @@ typedef struct { STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles); void tsdbCloseFileH(STsdbFileH *pFileH); int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); +int tsdbOpenFile(SFile *pFile, int oflag); +SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); typedef struct { int32_t len; - int32_t padding; // For padding purpose - int64_t offset; -} SCompIdx; + int32_t offset; + int32_t hasLast : 1; + int32_t numOfSuperBlocks : 31; + int32_t checksum; + TSKEY maxKey; +} SCompIdx; /* sizeof(SCompIdx) = 24 */ /** - * if numOfSubBlocks == -1, then the SCompBlock is a sub-block - * if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to - * the data block offset and length - * if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the - * binary + * if numOfSubBlocks == 0, then the SCompBlock is a sub-block + * if numOfSubBlocks >= 1, then the SCompBlock is a super-block + * - if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to + * the data block offset and length + * - if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the + * binary */ typedef struct { int64_t last : 1; // If the block in data file or last file @@ -101,11 +108,12 @@ typedef struct { int32_t delimiter; // For recovery usage int32_t checksum; // TODO: decide if checksum logic in this file or make it one API int64_t uid; - int32_t padding; // For padding purpose - int32_t numOfBlocks; // TODO: make the struct padding SCompBlock blocks[]; } SCompInfo; +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); + // TODO: take pre-calculation into account typedef struct { int16_t colId; // Column ID @@ -122,6 +130,8 @@ typedef struct { SCompCol cols[]; } SCompData; +int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols); + void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index f622c38b5f..7d0bdbd845 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -39,6 +39,7 @@ static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname); static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile); static int tsdbWriteFileHead(SFile *pFile); static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); +static SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * maxFiles); @@ -70,9 +71,7 @@ int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) SFileGroup fGroup; SFileGroup *pFGroup = &fGroup; - if (fid < TSDB_MIN_FILE_ID(pFileH) || fid > TSDB_MAX_FILE_ID(pFileH) || - bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey) == - NULL) { + if (tsdbSearchFGroup(pFileH, fid) == NULL) { pFGroup->fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(dataDir, fid, type, maxTables, &(pFGroup->files[type])) < 0) { @@ -107,6 +106,86 @@ int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { return 0; } +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); + if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + + if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1; + // TODO: need to check the correctness + return 0; +} + +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) { + SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); + + if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1; + + if (read(pFile->fd, buf, pIdx->len) < 0) return -1; + + // TODO: need to check the correctness + + return 0; +} + +static int tsdbWriteBlockToFileImpl(SFile * pFile, // File to write + SDataCols * pCols, // Data column buffer + int numOfPointsToWrie, // Number of points to write to the file + SCompBlock *pBlock // SCompBlock to hold block information to return + ) { + // pBlock->last = 0; + // pBlock->offset = lseek(pFile->fd, 0, SEEK_END); + // // pBlock->algorithm = ; + // pBlock->numOfPoints = pCols->numOfPoints; + // // pBlock->sversion = ; + // // pBlock->len = ; + // pBlock->numOfSubBlocks = 1; + // pBlock->keyFirst = dataColsKeyFirst(pCols); + // pBlock->keyLast = dataColsKeyLast(pCols); + // for (int i = 0; i < pCols->numOfCols; i++) { + // // TODO: if all col value is NULL, do not save it + // pBlock->numOfCols++; + // pCompData->numOfCols++; + // SCompCol *pCompCol = pCompData->cols + i; + // pCompCol->colId = pCols->cols[i].colId; + // pCompCol->type = pCols->cols[i].type; + + // // pCompCol->len = ; + // // pCompCol->offset = ; + // } + + return 0; +} + +int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols) { + memset((void *)pBlock, 0, sizeof(SCompBlock)); + SFile *pFile = NULL; + SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols); + if (pCompData == NULL) return -1; + pCompData->delimiter = TSDB_FILE_DELIMITER; + // pCompData->uid = ; + + if (isMerge) { + TSKEY keyFirst = dataColsKeyFirst(pCols); + // 1. Binary search the block the data can merged into + + if (1/* the data should only merged into last file */) { + } else { + } + } else { + // Write directly to the file without merge + if (1/*pCols->numOfPoints < pCfg->minRowsPerFileBlock*/) { + // TODO: write the data to the last file + } else { + // TODO: wirte the data to the data file + } + } + + // TODO: need to update pIdx + + if (pCompData) free(pCompData); + return 0; +} + static int compFGroupKey(const void *key, const void *fgroup) { int fid = *(int *)key; SFileGroup *pFGroup = (SFileGroup *)fgroup; @@ -158,7 +237,7 @@ static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname) return 0; } -static int tsdbOpenFileForWrite(SFile *pFile, int oflag) { // TODO: change the function +int tsdbOpenFile(SFile *pFile, int oflag) { // TODO: change the function if (TSDB_IS_FILE_OPENED(pFile)) return -1; pFile->fd = open(pFile->fname, oflag, 0755); @@ -167,6 +246,16 @@ static int tsdbOpenFileForWrite(SFile *pFile, int oflag) { // TODO: change the f return 0; } +SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { + SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); + if (pGroup == NULL) return NULL; + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbOpenFile(&(pGroup->files[type]), O_RDWR); + } + return pGroup; +} + static int tsdbCloseFile(SFile *pFile) { if (!TSDB_IS_FILE_OPENED(pFile)) return -1; int ret = close(pFile->fd); @@ -186,7 +275,7 @@ static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, return -1; } - if (tsdbOpenFileForWrite(pFile, O_WRONLY | O_CREAT) < 0) { + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) { // TODO: deal with the ERROR here return -1; } @@ -212,4 +301,12 @@ void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t file TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; +} + +static SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { + if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) + return NULL; + void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); + if (ptr == NULL) return NULL; + return (SFileGroup *)ptr; } \ No newline at end of file diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index a8e04e216a..8e433ecb5b 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -764,6 +764,7 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) { static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { int numOfRows = 0; + do { SSkipListNode *node = tSkipListIterGet(pIter); if (node == NULL) break; @@ -776,6 +777,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max numOfRows++; if (numOfRows > maxRowsToRead) break; } while (tSkipListIterNext(pIter)); + return numOfRows; } @@ -865,24 +867,122 @@ static void *tsdbCommitData(void *arg) { } static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { + int flag = 0; + STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; + SFile tFile, lFile; + SFileGroup *pGroup = NULL; + SCompIdx * pIndices = NULL; + SCompInfo * pCompInfo = NULL; + size_t compInfoSize = 0; + SCompBlock compBlock; + SCompBlock *pBlock = &compBlock; + TSKEY minKey = 0, maxKey = 0; tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; + STable * pTable = pMeta->tables[tid]; SSkipListIterator *pIter = iters[tid]; + int isLoadCompBlocks = 0; if (pIter == NULL) continue; tdInitDataCols(pCols, pTable->schema); - while (tsdbReadRowsFromCache(pIter, maxKey, pCfg->maxRowsPerFileBlock, pCols)) { - // TODO - int k = 0; - } + int numOfWrites = 0; + // while (tsdbReadRowsFromCache(pIter, maxKey, pCfg->maxRowsPerFileBlock, pCols)) { + // break; + // if (!flag) { + // // There are data to commit to this file, we need to create/open it for read/write. + // // At the meantime, we set the flag to prevent further create/open operations + // if (tsdbCreateFGroup(pFileH, pRepo->rootDir, fid, pCfg->maxTables) < 0) { + // // TODO: deal with the ERROR here + // } + // // Open files for commit + // pGroup = tsdbOpenFilesForCommit(pFileH, fid); + // if (pGroup == NULL) { + // // TODO: deal with the ERROR here + // } + // // TODO: open .h file and if neccessary, open .l file + // {} + // pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); + // if (pIndices == NULL) { + // // TODO: deal with the ERROR + // } + // // load the SCompIdx part + // if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { + // // TODO: deal with the ERROR here + // } + + // // TODO: sendfile those not need changed table content + // for (int ttid = 0; ttid < tid; ttid++) { + // // SCompIdx *pIdx = &pIndices[ttid]; + // // if (pIdx->len > 0) { + // // lseek(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, pIdx->offset, 0, SEEK_CUR); + // // sendfile(fd, pGroup->files[TSDB_FILE_TYPE_HEAD].fd, NULL, pIdx->len); + // // } + // } + // flag = 1; + // } + + // SCompIdx *pIdx = &pIndices[tid]; + + // /* The first time to write to the table, need to decide + // * if it is neccessary to load the SComplock part. If it + // * is needed, just load it, or, just use sendfile and + // * append it. + // */ + // if (numOfWrites == 0 && pIdx->offset > 0) { + // if (dataColsKeyFirst(pCols) <= pIdx->maxKey || pIdx->hasLast) { + // pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len); + // if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { + // // TODO: deal with the ERROR here + // } + // if (pCompInfo->uid == pTable->tableId.uid) isLoadCompBlocks = 1; + // } else { + // // TODO: sendfile the prefix part + // } + // } + + // // if (tsdbWriteBlockToFile(pGroup, pCompInfo, pIdx, isLoadCompBlocks, pBlock, pCols) < 0) { + // // // TODO: deal with the ERROR here + // // } + + // // pCompInfo = tsdbMergeBlock(pCompInfo, pBlock); + + + // // if (1 /* the SCompBlock part is not loaded*/) { + // // // Append to .data file generate a SCompBlock and record it + // // } else { + // // } + + // // // TODO: need to reset the pCols + + // numOfWrites++; + // } + + // if (pCols->numOfPoints > 0) { + // // TODO: still has data to commit, commit it + // } + + // if (1/* SCompBlock part is loaded, write it to .head file*/) { + // // TODO + // } else { + // // TODO: use sendfile send the old part and append the newly added part + // } } + // Write the SCompIdx part + + // Close all files and return + if (flag) { + // TODO + } + + if (pIndices) free(pIndices); + if (pCompInfo) free(pCompInfo); + return 0; } \ No newline at end of file From 497300f1ebb2e41c152aac9744d41e4de5ec7d0c Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 27 Mar 2020 10:41:53 +0800 Subject: [PATCH 58/85] Fix meta problem --- src/vnode/tsdb/src/tsdbMeta.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 397807f547..761006a009 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -381,7 +381,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { int32_t headSize = 0; // first tag column - STColumn* s = pSTable->tagSchema->columns[0]; //??? + STColumn* s = schemaColAt(pSTable->tagSchema, 0); tSkipListRandNodeInfo(pSTable->pIndex, &level, &headSize); SSkipListNode* pNode = calloc(1, headSize + s->bytes + POINTER_BYTES); @@ -389,7 +389,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { SSkipList* list = pSTable->pIndex; - memcpy(SL_GET_NODE_KEY(list, pNode), dataRowTuple(pTable->tagVal), s->columns[0].bytes); + memcpy(SL_GET_NODE_KEY(list, pNode), dataRowTuple(pTable->tagVal), colBytes(s)); memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES); tSkipListPut(list, pNode); From 6fae35afd024ca6f800a2955a512f8429141e66b Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 27 Mar 2020 17:55:54 +0800 Subject: [PATCH 59/85] [TD-32]add super table tag filter support at vnode side. --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscSQLParser.c | 11 +- src/client/src/tscServer.c | 49 +++-- src/client/src/tscUtil.c | 3 +- src/inc/taosmsg.h | 4 +- src/query/inc/qast.h | 4 +- src/query/src/qast.c | 218 ++++++++++---------- src/query/src/queryExecutor.c | 41 ++-- src/util/inc/tarray.h | 16 +- src/util/src/tarray.c | 32 +++ src/vnode/detail/src/vnodeUtil.c | 3 +- src/vnode/tsdb/inc/tsdb.h | 11 +- src/vnode/tsdb/src/tsdbRead.c | 329 ++++++++++++++++++++++++++++++- 13 files changed, 556 insertions(+), 167 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 402b6b3706..6bec41d146 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -177,7 +177,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetSTableQueryCondPos(STagCond* pCond, uint64_t tableIndex); -void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str); void tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 6d5d08292d..804d4e5268 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3511,7 +3511,7 @@ int tableNameCompar(const void* lhs, const void* rhs) { return ret > 0 ? 1 : -1; } -static int32_t setTableCondForMetricQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr, +static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* account, tSQLExpr* pExpr, int16_t tableCondIndex, SStringBuilder* sb) { const char* msg = "table name too long"; @@ -3740,7 +3740,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, return ret; } - tsSetMetricQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); + tsSetSTableQueryCond(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid, c); doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); @@ -3815,7 +3815,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql // 7. query condition for table name pQueryInfo->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - ret = setTableCondForMetricQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); + ret = setTableCondForSTableQuery(pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); taosStringBuilderDestroy(&sb); if (!validateFilterExpr(pQueryInfo)) { @@ -5444,8 +5444,9 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); // two table: the first one is for current table, and the secondary is for the super table. - tscAddEmptyMetaInfo(pQueryInfo); - assert(pQueryInfo->numOfTables == 2); + if (pQueryInfo->numOfTables < 2) { + tscAddEmptyMetaInfo(pQueryInfo); + } const int32_t TABLE_INDEX = 0; const int32_t STABLE_INDEX = 1; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b2bd9110e7..22f8a85757 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -601,12 +601,21 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; -// SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; if (pQueryInfo->colList.numOfCols <= 0) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); return -1; } + + if (pQueryInfo->intervalTime < 0) { + tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); + return -1; + } + + if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { + tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); + return -1; + } char *pStart = pCmd->payload + tsRpcHeadSize; @@ -643,8 +652,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { numOfTables = 1; } - pQueryMsg->numOfTables = htonl(numOfTables); - if (pQueryInfo->order.order == TSQL_SO_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->stime); pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); @@ -653,6 +660,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); } + pQueryMsg->numOfTables = htonl(numOfTables); pQueryMsg->order = htons(pQueryInfo->order.order); pQueryMsg->orderColId = htons(pQueryInfo->order.orderColId); pQueryMsg->interpoType = htons(pQueryInfo->interpoType); @@ -662,23 +670,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; - - if (pQueryInfo->intervalTime < 0) { - tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; - } - - if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { - tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; - } - pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { // query on meter - pQueryMsg->tagLength = 0; - } else { // query on super table - pQueryMsg->tagLength = htons(0); - } pQueryMsg->queryType = htons(pQueryInfo->type); pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs); @@ -742,7 +734,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { bool hasArithmeticFunction = false; SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; - for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); @@ -856,6 +847,24 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); } + if (pQueryInfo->tagCond.numOfTagCond > 0) { + STagCond* pTagCond = &pQueryInfo->tagCond; + + SCond *pCond = tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid); + if (pCond != NULL && pCond->cond != NULL) { + size_t condLen = strlen(pCond->cond) + 1; + + bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); + if (!ret) { + tscError("%p mbs to ucs4 failed:%d", pSql, tsGetSTableQueryCondPos(pTagCond, pTableMeta->uid)); + return 0; + } + + pQueryMsg->tagCondLen = htons(condLen); + pMsg += condLen * TSDB_NCHAR_SIZE; + } + } + msgLen = pMsg - pStart; tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8df84744ce..ec6881db3f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -102,7 +102,8 @@ SCond* tsGetSTableQueryCondPos(STagCond* pTagCond, uint64_t uid) { return NULL; } -void tsSetMetricQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { +// todo refactor by using SArray +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, const char* str) { size_t len = strlen(str); if (len == 0) { return; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 1345d80fe5..999e293d42 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -474,14 +474,14 @@ typedef struct { int64_t intervalOffset; // start offset for interval query int64_t slidingTime; // value for sliding window char slidingTimeUnit; // time interval type, for revisement of interval(1d) - int16_t tagLength; // tag length in current query + uint16_t tagCondLen; // tag length in current query int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx uint64_t groupbyTagIds; int64_t limit; int64_t offset; - int16_t queryType; // denote another query process + uint16_t queryType; // denote another query process int16_t numOfOutputCols; // final output columns numbers int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 1e89ffb49d..d52eb24726 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include #include "os.h" #include "taosmsg.h" @@ -93,8 +94,7 @@ void tSQLBinaryExprToString(tSQLBinaryExpr *pExpr, char *dst, int32_t *len); void tSQLBinaryExprDestroy(tSQLBinaryExpr **pExprs, void (*fp)(void*)); -void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExprs, struct tSkipList *pSkipList, tQueryResultset *result, - SBinaryFilterSupp *param); +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, char *, int32_t)); diff --git a/src/query/src/qast.c b/src/query/src/qast.c index a0cbf12169..b301bbf043 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -14,6 +14,8 @@ */ #include "qast.h" +#include +#include #include "os.h" #include "qsqlparser.h" #include "qsyntaxtreefunction.h" @@ -705,126 +707,120 @@ static bool filterItem(tSQLBinaryExpr *pExpr, const void *pItem, SBinaryFilterSu * @param pSchema tag schemas * @param fp filter callback function */ -static UNUSED_FUNC void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, SBinaryFilterSupp *param) { - int32_t n = 0; - for (int32_t i = 0; i < pResult->num; ++i) { - void *pItem = pResult->pRes[i]; +static void tSQLBinaryTraverseOnResult(tSQLBinaryExpr *pExpr, SArray *pResult, SBinaryFilterSupp *param) { + size_t size = taosArrayGetSize(pResult); + + SArray* array = taosArrayInit(size, POINTER_BYTES); + for (int32_t i = 0; i < size; ++i) { + void *pItem = taosArrayGetP(pResult, i); if (filterItem(pExpr, pItem, param)) { - pResult->pRes[n++] = pResult->pRes[i]; + taosArrayPush(array, &pItem); } } - - pResult->num = n; + + taosArrayCopy(pResult, array); } -//static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, tQueryResultset *pResult, tSkipList *pSkipList, -// SBinaryFilterSupp *param) { -// int32_t n = 0; -// SSkipListIterator iter = {0}; -// -// int32_t ret = tSkipListIteratorReset(pSkipList, &iter); -// assert(ret == 0); -// -// pResult->pRes = calloc(pSkipList->nSize, POINTER_BYTES); -// -// while (tSkipListIteratorNext(&iter)) { -// tSkipListNode *pNode = tSkipListIteratorGet(&iter); -// if (filterItem(pExpr, pNode, param)) { -// pResult->pRes[n++] = pNode; -// } -// } -// -// pResult->num = n; -//} +static void tSQLBinaryTraverseOnSkipList(tSQLBinaryExpr *pExpr, SArray *pResult, SSkipList *pSkipList, + SBinaryFilterSupp *param) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + if (filterItem(pExpr, pNode, param)) { + taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); + } + } +} // post-root order traverse syntax tree -//void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, tSkipList *pSkipList, tQueryResultset *result, -// SBinaryFilterSupp *param) { -// if (pExpr == NULL) { -// return; -// } -// -// tSQLSyntaxNode *pLeft = pExpr->pLeft; -// tSQLSyntaxNode *pRight = pExpr->pRight; -// -// // recursive traverse left child branch -// if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { -// uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; -// -// if (weight == 0 && result->num > 0 && pSkipList == NULL) { -// /** -// * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. -// * Since no index presented, the filter operation is done by scan all elements in the result set. -// * -// * if the query is a high selectivity filter, only small portion of meters are retrieved. -// */ -// tSQLBinaryTraverseOnResult(pExpr, result, param); -// } else if (weight == 0) { -// /** -// * apply the hierarchical expression to every node in skiplist for find the qualified nodes -// */ +void tSQLBinaryExprTraverse(tSQLBinaryExpr *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { + if (pExpr == NULL) { + return; + } + + tSQLSyntaxNode *pLeft = pExpr->pLeft; + tSQLSyntaxNode *pRight = pExpr->pRight; + + // recursive traverse left child branch + if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { + uint8_t weight = pLeft->pExpr->filterOnPrimaryKey + pRight->pExpr->filterOnPrimaryKey; + + if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ + tSQLBinaryTraverseOnResult(pExpr, result, param); + } else if (weight == 0) { + /** + * apply the hierarchical expression to every node in skiplist for find the qualified nodes + */ + assert(taosArrayGetSize(result) == 0); + tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); + } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { + tQueryResultset rLeft = {0}; + tQueryResultset rRight = {0}; + + tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); + tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); + + if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS + intersect(&rLeft, &rRight, result); + } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or + merge(&rLeft, &rRight, result); + } else { + assert(false); + } + + free(rLeft.pRes); + free(rRight.pRes); + } else { + /* + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, + * then, we conduct the secondary filter operation based on the result from the initial filter stage. + */ + assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); + + tSQLBinaryExpr *pFirst = NULL; + tSQLBinaryExpr *pSecond = NULL; + if (pLeft->pExpr->filterOnPrimaryKey == 1) { + pFirst = pLeft->pExpr; + pSecond = pRight->pExpr; + } else { + pFirst = pRight->pExpr; + pSecond = pLeft->pExpr; + } + + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + + // we filter the result based on the skiplist index in the first place + tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skiplist index as a parameter + */ + tSQLBinaryExprTraverse(pSecond, NULL, result, param); + } + } else { // column project + assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + + param->setupInfoFn(pExpr, param->pExtInfo); + if (pSkipList == NULL) { + tSQLListTraverseOnResult(pExpr, param->fp, result); + } else { // assert(result->num == 0); -// tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); -// } else if (weight == 2 || (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_OR)) { -// tQueryResultset rLeft = {0}; -// tQueryResultset rRight = {0}; -// -// tSQLBinaryExprTraverse(pLeft->pExpr, pSkipList, &rLeft, param); -// tSQLBinaryExprTraverse(pRight->pExpr, pSkipList, &rRight, param); -// -// if (pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) { // CROSS -// intersect(&rLeft, &rRight, result); -// } else if (pExpr->nSQLBinaryOptr == TSDB_RELATION_OR) { // or -// merge(&rLeft, &rRight, result); -// } else { -// assert(false); -// } -// -// free(rLeft.pRes); -// free(rRight.pRes); -// } else { -// /* -// * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here -// * -// * first, we filter results based on the skiplist index, which is the initial filter stage, -// * then, we conduct the secondary filter operation based on the result from the initial filter stage. -// */ -// assert(pExpr->nSQLBinaryOptr == TSDB_RELATION_AND); -// -// tSQLBinaryExpr *pFirst = NULL; -// tSQLBinaryExpr *pSecond = NULL; -// if (pLeft->pExpr->filterOnPrimaryKey == 1) { -// pFirst = pLeft->pExpr; -// pSecond = pRight->pExpr; -// } else { -// pFirst = pRight->pExpr; -// pSecond = pLeft->pExpr; -// } -// -// assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); -// -// // we filter the result based on the skiplist index in the first place -// tSQLBinaryExprTraverse(pFirst, pSkipList, result, param); -// -// /* -// * recursively perform the filter operation based on the initial results, -// * So, we do not set the skiplist index as a parameter -// */ -// tSQLBinaryExprTraverse(pSecond, NULL, result, param); -// } -// } else { // column project -// assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); -// -// param->setupInfoFn(pExpr, param->pExtInfo); -// if (pSkipList == NULL) { -// tSQLListTraverseOnResult(pExpr, param->fp, result); -// } else { -// assert(result->num == 0); -//// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); -// } -// } -//} +// tSQLDoFilterInitialResult(pSkipList, param->fp, pExpr->info, result); + } + } +} void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, char *, int32_t)) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index a1167ff440..86baf44f45 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5310,11 +5310,6 @@ static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { return -1; } - if (pQueryTableMsg->tagLength < 0) { - dError("qmsg:%p illegal value of tag length %d", pQueryTableMsg, pQueryTableMsg->tagLength); - return -1; - } - return 0; } @@ -5353,7 +5348,8 @@ static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArra * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr) { +static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, + wchar_t** tagCond) { pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); @@ -5372,7 +5368,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); - pQueryTableMsg->tagLength = htons(pQueryTableMsg->tagLength); + pQueryTableMsg->tagCondLen = htons(pQueryTableMsg->tagCondLen); pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); @@ -5428,7 +5424,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId bool hasArithmeticFunction = false; *pExpr = calloc(pQueryTableMsg->numOfOutputCols, POINTER_BYTES); - SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { @@ -5501,6 +5496,14 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { v[i] = htobe64(v[i]); } + + pMsg += sizeof(int64_t) * pQueryTableMsg->numOfOutputCols; + } + + // the tag query condition expression string is located at the end of query msg + if (pQueryTableMsg->tagCondLen > 0) { + *tagCond = calloc(1, pQueryTableMsg->tagCondLen * TSDB_NCHAR_SIZE); + memcpy(*tagCond, pMsg, pQueryTableMsg->tagCondLen * TSDB_NCHAR_SIZE); } dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " @@ -5993,9 +5996,9 @@ void vnodeFreeQInfo(SQInfo *pQInfo) { static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, SArray *pTableIdList, void* tsdb, SQInfo **pQInfo) { int32_t code = TSDB_CODE_SUCCESS; - + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pSqlExprs, pTableIdList); - if (pQInfo == NULL) { + if ((*pQInfo) == NULL) { code = TSDB_CODE_SERV_OUT_OF_MEMORY; goto _error; } @@ -6024,6 +6027,7 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE tsBufNextPos(pTSBuf); } + // filter the qualified if ((code = initQInfo(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -6050,7 +6054,9 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ SArray *pTableIdList = NULL; SSqlFuncExprMsg** pExprMsg = NULL; - if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList, &pExprMsg)) != TSDB_CODE_SUCCESS) { + wchar_t* tagCond = NULL; + + if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { return code; } @@ -6076,7 +6082,18 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQ if ((pGroupbyExpr == NULL && pQueryTableMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } - + + // super table query + if ((pQueryTableMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { + STableId* id = taosArrayGet(pTableIdList, 0); + + SArray* res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryTableMsg->tagCondLen); + if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode + code = TSDB_CODE_SUCCESS; + goto _query_over; + } + } + code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); _query_over: diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 55bdc849ad..6b70780d10 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -87,7 +87,21 @@ size_t taosArrayGetSize(const SArray* pArray); void* taosArrayInsert(SArray* pArray, size_t index, void* pData); /** - * + * remove data entry of the given index + * @param pArray + * @param index + */ +void taosArrayRemove(SArray* pArray, size_t index); + +/** + * copy the whole array from source to destination + * @param pDst + * @param pSrc + */ +void taosArrayCopy(SArray* pDst, SArray* pSrc); + +/** + * destroy array list * @param pArray */ void taosArrayDestroy(SArray* pArray); diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index fb2dac827e..eb1d213317 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -128,6 +128,38 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { return dst; } +void taosArrayRemove(SArray* pArray, size_t index) { + assert(index < pArray->size); + + if (index == pArray->size - 1) { + taosArrayPop(pArray); + return; + } + + size_t remain = pArray->size - index - 1; + memmove(pArray->pData + index * pArray->elemSize, pArray->pData + (index + 1) * pArray->elemSize, remain * pArray->elemSize); + pArray->size -= 1; +} + +void taosArrayCopy(SArray* pDst, SArray* pSrc) { + assert(pSrc != NULL && pDst != NULL); + + if (pDst->capacity < pSrc->size) { + void* pData = realloc(pDst->pData, pSrc->size * pSrc->elemSize); + if (pData == NULL) { // todo handle oom + + } else { + pDst->pData = pData; + pDst->capacity = pSrc->size; + } + } + + memcpy(pDst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); + pDst->elemSize = pSrc->elemSize; + pDst->capacity = pSrc->size; + pDst->size = pSrc->size; +} + void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; diff --git a/src/vnode/detail/src/vnodeUtil.c b/src/vnode/detail/src/vnodeUtil.c index 43c24bae6a..1d5f45e5e3 100644 --- a/src/vnode/detail/src/vnodeUtil.c +++ b/src/vnode/detail/src/vnodeUtil.c @@ -164,8 +164,7 @@ static int32_t vnodeBuildExprFromArithmeticStr(SSqlFunctionExpr* pExpr, SQueryMe SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols); dTrace("qmsg:%p create binary expr from string:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); - tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz, - pExpr->pBase.arg[0].argBytes); + tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz); if (pBinExpr == NULL) { dError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); diff --git a/src/vnode/tsdb/inc/tsdb.h b/src/vnode/tsdb/inc/tsdb.h index 92d8ad757b..ed6bb044c6 100644 --- a/src/vnode/tsdb/inc/tsdb.h +++ b/src/vnode/tsdb/inc/tsdb.h @@ -207,11 +207,6 @@ typedef struct SDataBlockInfo { int32_t sid; } SDataBlockInfo; -typedef struct STableIDList { - STableId *tableIds; - int32_t num; -} STableIDList; - typedef struct { } SFields; @@ -325,15 +320,15 @@ tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stable * @param pQueryHandle * @return table sid list. the invoker is responsible for the release of this the sid list. */ -STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); +SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle); /** - * Get the qualified table sid for a super table according to the tag query expression. + * Get the qualified table id for a super table according to the tag query expression. * @param stableid. super table sid * @param pTagCond. tag query condition * */ -STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond); +SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index fc196bb62f..0140e2e11e 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -13,9 +13,12 @@ * along with this program. If not, see . */ +#include #include "os.h" #include "tutil.h" +#include "../../../query/inc/qast.h" +#include "../../../query/inc/tsqlfunction.h" #include "tsdb.h" #include "tsdbFile.h" #include "tsdbMeta.h" @@ -413,6 +416,328 @@ SArray *tsdbRetrieveDataRow(tsdb_query_handle_t *pQueryHandle, SArray *pIdList, tsdb_query_handle_t *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr) {} -STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) {} +SArray *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle) {} -STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond) {} +static SArray* createTableIdArrayList(struct STsdbRepo* tsdb, int64_t uid) { + STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pTable != NULL); //assert pTable is a super table + + size_t size = tSkipListGetSize(pTable->pIndex); + SArray* pList = taosArrayInit(size, sizeof(STableId)); + + SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); + while(tSkipListIterNext(iter)) { + STable* t = *(STable**) tSkipListIterGet(iter); + taosArrayPush(pList, &t->tableId); + } + + return pList; +} + +typedef struct SSyntaxTreeFilterSupporter { + SSchema* pTagSchema; + int32_t numOfTags; + int32_t optr; +} SSyntaxTreeFilterSupporter; + +/** + * convert the result pointer to STabObj instead of tSkipListNode + * @param pRes + */ +static void tansformQueryResult(SArray* pRes) { + if (pRes == NULL || taosArrayGetSize(pRes) == 0) { + return; + } + + size_t size = taosArrayGetSize(pRes); + for (int32_t i = 0; i < size; ++i) { +// pRes->pRes[i] = ((tSkipListNode*)(pRes->pRes[i]))->pData; + } +} + +void tSQLListTraverseDestroyInfo(void* param) { + if (param == NULL) { + return; + } + + tQueryInfo* pInfo = (tQueryInfo*)param; + tVariantDestroy(&(pInfo->q)); + free(param); +} + +static char* convertTagQueryStr(const wchar_t* str, size_t len) { + char* mbs = NULL; + + if (len > 0) { + mbs = calloc(1, (len + 1) * TSDB_NCHAR_SIZE); + taosUcs4ToMbs((void*) str, len * TSDB_NCHAR_SIZE, mbs); //todo add log + } + + return mbs; +} + +static int32_t compareStrVal(const void* pLeft, const void* pRight) { + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareWStrVal(const void* pLeft, const void* pRight) { + int32_t ret = wcscmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareIntVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_INT64_VAL(pRight)); +} + +static int32_t compareIntDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_INT64_VAL(pLeft), GET_DOUBLE_VAL(pRight)); +} + +static int32_t compareDoubleVal(const void* pLeft, const void* pRight) { + DEFAULT_COMP(GET_DOUBLE_VAL(pLeft), GET_DOUBLE_VAL(pRight)); +} + +static int32_t compareDoubleIntVal(const void* pLeft, const void* pRight) { + double ret = (*(double*)pLeft) - (*(int64_t*)pRight); + if (fabs(ret) < DBL_EPSILON) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } +} + +static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const char* pattern = pRight; + const char* str = pLeft; + + int32_t ret = patternMatch(pattern, str, strlen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + const wchar_t* pattern = pRight; + const wchar_t* str = pLeft; + + int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +static __compar_fn_t getFilterComparator(int32_t type, int32_t filterType, int32_t optr) { + __compar_fn_t comparator = NULL; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BOOL: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareIntDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + if (filterType >= TSDB_DATA_TYPE_BOOL && filterType <= TSDB_DATA_TYPE_BIGINT) { + comparator = compareDoubleIntVal; + } else if (filterType >= TSDB_DATA_TYPE_FLOAT && filterType <= TSDB_DATA_TYPE_DOUBLE) { + comparator = compareDoubleVal; + } + break; + } + + case TSDB_DATA_TYPE_BINARY: { + assert(filterType == TSDB_DATA_TYPE_BINARY); + + if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparator = compareStrPatternComp; + } else { /* normal relational comparator */ + comparator = compareStrVal; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + assert(filterType == TSDB_DATA_TYPE_NCHAR); + + if (optr == TSDB_RELATION_LIKE) { + comparator = compareWStrPatternComp; + } else { + comparator = compareWStrVal; + } + + break; + } + default: + comparator = compareIntVal; + break; + } + + return comparator; +} + +static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pSchema, int32_t* index, + int32_t* offset) { + *index = 0; + *offset = 0; + + // filter on table name(TBNAME) + if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { + *index = TSDB_TBNAME_COLUMN_INDEX; + *offset = TSDB_TBNAME_COLUMN_INDEX; + return; + } + + while ((*index) < pSupporter->numOfTags) { + if (pSupporter->pTagSchema[*index].bytes == pSchema->bytes && + pSupporter->pTagSchema[*index].type == pSchema->type && + strcmp(pSupporter->pTagSchema[*index].name, pSchema->name) == 0) { + break; + } else { + (*offset) += pSupporter->pTagSchema[(*index)++].bytes; + } + } +} + +void filterPrepare(void* expr, void* param) { + tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr; + if (pExpr->info != NULL) { + return; + } + + int32_t i = 0, offset = 0; + pExpr->info = calloc(1, sizeof(tQueryInfo)); + + tQueryInfo* pInfo = pExpr->info; + SSyntaxTreeFilterSupporter* pSupporter = (SSyntaxTreeFilterSupporter*)param; + + tVariant* pCond = pExpr->pRight->pVal; + SSchema* pSchema = pExpr->pLeft->pSchema; + + getTagColumnInfo(pSupporter, pSchema, &i, &offset); + assert((i >= 0 && i < TSDB_MAX_TAGS) || (i == TSDB_TBNAME_COLUMN_INDEX)); + assert((offset >= 0 && offset < TSDB_MAX_TAGS_LEN) || (offset == TSDB_TBNAME_COLUMN_INDEX)); + + pInfo->sch = *pSchema; + pInfo->colIdx = i; + pInfo->optr = pExpr->nSQLBinaryOptr; + pInfo->offset = offset; + pInfo->compare = getFilterComparator(pSchema->type, pCond->nType, pInfo->optr); + + tVariantAssign(&pInfo->q, pCond); + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); +} + +bool tSkipListNodeFilterCallback(const void* pNode, void* param) { + tQueryInfo* pInfo = (tQueryInfo*)param; + + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + + char buf[TSDB_MAX_TAGS_LEN] = {0}; + + char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + int8_t type = pInfo->sch.type; + + int32_t ret = 0; + if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { + ret = pInfo->compare(val, pInfo->q.pz); + } else { + tVariant t = {0}; + tVariantCreateFromBinary(&t, val, (uint32_t) pInfo->sch.bytes, type); + + ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); + } + + switch (pInfo->optr) { + case TSDB_RELATION_EQUAL: { + return ret == 0; + } + case TSDB_RELATION_NOT_EQUAL: { + return ret != 0; + } + case TSDB_RELATION_LARGE_EQUAL: { + return ret >= 0; + } + case TSDB_RELATION_LARGE: { + return ret > 0; + } + case TSDB_RELATION_LESS_EQUAL: { + return ret <= 0; + } + case TSDB_RELATION_LESS: { + return ret < 0; + } + case TSDB_RELATION_LIKE: { + return ret == 0; + } + + default: + assert(false); + } + return true; +} + +static int32_t mgmtFilterMeterByIndex(STable* pSTable, SArray* pRes, const char* pCond) { + STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); + + tSQLBinaryExpr* pExpr = NULL; + tSQLBinaryExprFromString(&pExpr, stcol, schemaNCols(pSTable->tagSchema), pCond, strlen(pCond)); + + // failed to build expression, no result, return immediately + if (pExpr == NULL) { + mError("table:%" PRIu64 ", no result returned, error in super table query expression:%s", pSTable->tableId.uid, pCond); + tfree(pCond); + + return TSDB_CODE_OPS_NOT_SUPPORT; + } + + // query according to the binary expression + SSyntaxTreeFilterSupporter s = {.pTagSchema = stcol, .numOfTags = schemaNCols(pSTable->tagSchema)}; + + SBinaryFilterSupp supp = {.fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, + .setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare, + .pExtInfo = &s}; + + tSQLBinaryExprTraverse(pExpr, pSTable->pIndex, pRes, &supp); + tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo); + + tansformQueryResult(pRes); + + return TSDB_CODE_SUCCESS; +} + +SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *pTagCond, size_t len) { + // no condition, all tables created according to the stable will involved in querying + if (pTagCond == NULL || wcslen(pTagCond) == 0) { + return createTableIdArrayList(tsdb, uid); + } else { + char* str = convertTagQueryStr(pTagCond, len); + SArray* result = taosArrayInit(8, POINTER_BYTES); + + STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); + assert(pSTable != NULL); + + if (mgmtFilterMeterByIndex(pSTable, result, str) == TSDB_CODE_SUCCESS) { + return result; + } + } +} From 7032405e95f63a1b0575a312b6a606506102ffc4 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 27 Mar 2020 20:19:34 +0800 Subject: [PATCH 60/85] TD-34 --- src/vnode/tsdb/inc/tsdbFile.h | 16 ++- src/vnode/tsdb/src/tsdbFile.c | 44 ++++++--- src/vnode/tsdb/src/tsdbMain.c | 180 ++++++++++++++++++++-------------- 3 files changed, 152 insertions(+), 88 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 22563275cd..c761f8520e 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -25,6 +25,8 @@ extern "C" { #endif +#define TSDB_FILE_HEAD_SIZE 512 + #define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) #define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) @@ -69,6 +71,7 @@ typedef struct { STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles); void tsdbCloseFileH(STsdbFileH *pFileH); +int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose); int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); int tsdbOpenFile(SFile *pFile, int oflag); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); @@ -104,6 +107,9 @@ typedef struct { TSKEY keyLast; } SCompBlock; +#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1) +#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) + typedef struct { int32_t delimiter; // For recovery usage int32_t checksum; // TODO: decide if checksum logic in this file or make it one API @@ -111,8 +117,7 @@ typedef struct { SCompBlock blocks[]; } SCompInfo; -int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); -int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); +#define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx)) // TODO: take pre-calculation into account typedef struct { @@ -129,6 +134,13 @@ typedef struct { int64_t uid; // For recovery usage SCompCol cols[]; } SCompData; +int tsdbCopyCompBlockToFile(SFile *outFile, SFile *inFile, SCompInfo *pCompInfo, int index, int isOutLast); + +int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); +int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); +int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf); +int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf); +// TODO: need an API to merge all sub-block data into one int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols); diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 7d0bdbd845..f22274531d 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -24,7 +24,6 @@ #include "tsdbFile.h" -#define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F const char *tsdbFileSuffix[] = { @@ -35,8 +34,7 @@ const char *tsdbFileSuffix[] = { static int compFGroupKey(const void *key, const void *fgroup); static int compFGroup(const void *arg1, const void *arg2); -static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname); -static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile); +static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname); static int tsdbWriteFileHead(SFile *pFile); static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); static SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); @@ -71,10 +69,10 @@ int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) SFileGroup fGroup; SFileGroup *pFGroup = &fGroup; - if (tsdbSearchFGroup(pFileH, fid) == NULL) { + if (tsdbSearchFGroup(pFileH, fid) == NULL) { // if not exists, create one pFGroup->fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(dataDir, fid, type, maxTables, &(pFGroup->files[type])) < 0) { + if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], maxTables, &(pFGroup->files[type]), type == TSDB_FILE_TYPE_HEAD ? 1 : 0, 1) < 0) { // TODO: deal with the ERROR here, remove those creaed file return -1; } @@ -105,6 +103,10 @@ int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { return 0; } +int tsdbCopyCompBlockToFile(SFile *outFile, SFile *inFile, SCompInfo *pCompInfo, int index, int isOutLast) { + // TODO + return 0; +} int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) { SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]); @@ -127,6 +129,22 @@ int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) { return 0; } +int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) { + // assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); + + if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1; + size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols; + if (read(pFile->fd, buf, size) < 0) return -1; + + return 0; +} + +int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) { + if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1; + if (read(pFile->fd, buf, pCol->len) < 0) return -1; + return 0; +} + static int tsdbWriteBlockToFileImpl(SFile * pFile, // File to write SDataCols * pCols, // Data column buffer int numOfPointsToWrie, // Number of points to write to the file @@ -229,10 +247,10 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { return 0; } -static int tsdbGetFileName(char *dataDir, int fileId, int8_t type, char *fname) { - if (dataDir == NULL || fname == NULL || !IS_VALID_TSDB_FILE_TYPE(type)) return -1; +static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname) { + if (dataDir == NULL || fname == NULL) return -1; - sprintf(fname, "%s/f%d%s", dataDir, fileId, tsdbFileSuffix[type]); + sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix); return 0; } @@ -264,12 +282,12 @@ static int tsdbCloseFile(SFile *pFile) { return ret; } -static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, SFile *pFile) { +int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) { memset((void *)pFile, 0, sizeof(SFile)); - pFile->type = type; pFile->fd = -1; - tsdbGetFileName(dataDir, fileId, type, pFile->fname); + tsdbGetFileName(dataDir, fileId, suffix, pFile->fname); + if (access(pFile->fname, F_OK) == 0) { // File already exists return -1; @@ -280,7 +298,7 @@ static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, return -1; } - if (type == TSDB_FILE_TYPE_HEAD) { + if (writeHeader) { if (tsdbWriteHeadFileIdx(pFile, maxTables) < 0) { tsdbCloseFile(pFile); return -1; @@ -292,7 +310,7 @@ static int tsdbCreateFile(char *dataDir, int fileId, int8_t type, int maxTables, return -1; } - tsdbCloseFile(pFile); + if (toClose) tsdbCloseFile(pFile); return 0; } diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 8e433ecb5b..3945ffabda 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -8,6 +8,7 @@ #include #include #include +#include #include // #include "taosdef.h" @@ -45,6 +46,7 @@ #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" #define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7 +#define TSDB_MAX_LAST_FILE_SIZE (1024 * 1024 * 10) // 10M enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING }; @@ -775,7 +777,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max tdAppendDataRowToDataCol(row, pCols); numOfRows++; - if (numOfRows > maxRowsToRead) break; + if (numOfRows >= maxRowsToRead) break; } while (tSkipListIterNext(pIter)); return numOfRows; @@ -842,7 +844,10 @@ static void *tsdbCommitData(void *arg) { int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); for (int fid = sfid; fid <= efid; fid++) { - tsdbCommitToFile(pRepo, fid, iters, pCols); + if (tsdbCommitToFile(pRepo, fid, iters, pCols) < 0) { + // TODO: deal with the error here + // assert(0); + } } tdFreeDataCols(pCols); @@ -867,7 +872,8 @@ static void *tsdbCommitData(void *arg) { } static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { - int flag = 0; + int hasDataToCommit = 0; + int isNewLastFile = 0; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; @@ -887,97 +893,125 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters STable * pTable = pMeta->tables[tid]; SSkipListIterator *pIter = iters[tid]; int isLoadCompBlocks = 0; + char dataDir[128] = "\0"; if (pIter == NULL) continue; tdInitDataCols(pCols, pTable->schema); int numOfWrites = 0; - // while (tsdbReadRowsFromCache(pIter, maxKey, pCfg->maxRowsPerFileBlock, pCols)) { - // break; - // if (!flag) { - // // There are data to commit to this file, we need to create/open it for read/write. - // // At the meantime, we set the flag to prevent further create/open operations - // if (tsdbCreateFGroup(pFileH, pRepo->rootDir, fid, pCfg->maxTables) < 0) { - // // TODO: deal with the ERROR here - // } - // // Open files for commit - // pGroup = tsdbOpenFilesForCommit(pFileH, fid); - // if (pGroup == NULL) { - // // TODO: deal with the ERROR here - // } - // // TODO: open .h file and if neccessary, open .l file - // {} - // pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); - // if (pIndices == NULL) { - // // TODO: deal with the ERROR - // } - // // load the SCompIdx part - // if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { - // // TODO: deal with the ERROR here - // } + int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; // We keep 20% of space for merge purpose + // Loop to read columns from cache + while (tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols)) { + if (!hasDataToCommit) { + // There are data to commit to this fileId, we need to create/open it for read/write. + // At the meantime, we set the flag to prevent further create/open operations + tsdbGetDataDirName(pRepo, dataDir); - // // TODO: sendfile those not need changed table content - // for (int ttid = 0; ttid < tid; ttid++) { - // // SCompIdx *pIdx = &pIndices[ttid]; - // // if (pIdx->len > 0) { - // // lseek(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, pIdx->offset, 0, SEEK_CUR); - // // sendfile(fd, pGroup->files[TSDB_FILE_TYPE_HEAD].fd, NULL, pIdx->len); - // // } - // } - // flag = 1; - // } + if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { + // TODO: deal with the ERROR here + } + // Open files for commit + pGroup = tsdbOpenFilesForCommit(pFileH, fid); + if (pGroup == NULL) { + // TODO: deal with the ERROR here + } + // TODO: open .h file and if neccessary, open .l file + tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &tFile, 1, 0); + if (1 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { + // TODO: make it not to write the last file every time + tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); + isNewLastFile = 1; + } - // SCompIdx *pIdx = &pIndices[tid]; + // load the SCompIdx part + pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); + if (pIndices == NULL) { // TODO: deal with the ERROR + } + if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { // TODO: deal with the ERROR here + } - // /* The first time to write to the table, need to decide - // * if it is neccessary to load the SComplock part. If it - // * is needed, just load it, or, just use sendfile and - // * append it. - // */ - // if (numOfWrites == 0 && pIdx->offset > 0) { - // if (dataColsKeyFirst(pCols) <= pIdx->maxKey || pIdx->hasLast) { - // pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len); - // if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { - // // TODO: deal with the ERROR here - // } - // if (pCompInfo->uid == pTable->tableId.uid) isLoadCompBlocks = 1; - // } else { - // // TODO: sendfile the prefix part - // } - // } + // sendfile those not need to changed table content + lseek(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, + SEEK_SET); + lseek(tFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); + for (int ttid = 0; ttid < tid; ttid++) { + SCompIdx * tIdx= &pIndices[ttid]; + if (tIdx->len <= 0) continue; + if (isNewLastFile && tIdx->hasLast) { + // TODO: Need to load the SCompBlock part and copy to new last file + pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, tIdx->len); + if (pCompInfo == NULL) { /* TODO */} + if (tsdbLoadCompBlocks(pGroup, tIdx, (void *)pCompInfo) < 0) {/* TODO */} + SCompBlock *pLastBlock = TSDB_COMPBLOCK_AT(pCompInfo, tIdx->numOfSuperBlocks - 1); + int numOfSubBlocks = pLastBlock->numOfSubBlocks; + assert(pLastBlock->last); + if (tsdbCopyCompBlockToFile(&pGroup->files[TSDB_FILE_TYPE_LAST], &lFile, pCompInfo, tIdx->numOfSuperBlocks, 1 /* isOutLast*/) < 0) {/* TODO */} + { + if (numOfSubBlocks > 1) tIdx->len -= (sizeof(SCompBlock) * numOfSubBlocks); + tIdx->checksum = 0; + } + write(tFile.fd, (void *)pCompInfo, tIdx->len); + tFile.size += tIdx->len; + } else { + sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, tFile.fd, NULL, tIdx->len); + tFile.size += tIdx->len; + } + } - // // if (tsdbWriteBlockToFile(pGroup, pCompInfo, pIdx, isLoadCompBlocks, pBlock, pCols) < 0) { - // // // TODO: deal with the ERROR here - // // } + hasDataToCommit = 1; + } - // // pCompInfo = tsdbMergeBlock(pCompInfo, pBlock); + SCompIdx *pIdx = &pIndices[tid]; + + /* The first time to write to the table, need to decide + * if it is neccessary to load the SComplock part. If it + * is needed, just load it, or, just use sendfile and + * append it. + */ + if (numOfWrites == 0 && pIdx->offset > 0) { + if (dataColsKeyFirst(pCols) <= pIdx->maxKey || pIdx->hasLast) { + pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len); + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { + // TODO: deal with the ERROR here + } + if (pCompInfo->uid == pTable->tableId.uid) isLoadCompBlocks = 1; + } else { + // TODO: sendfile the prefix part + } + } + + // if (tsdbWriteBlockToFile(pGroup, pCompInfo, pIdx, isLoadCompBlocks, pBlock, pCols) < 0) { + // // TODO: deal with the ERROR here + // } + + // pCompInfo = tsdbMergeBlock(pCompInfo, pBlock); - // // if (1 /* the SCompBlock part is not loaded*/) { - // // // Append to .data file generate a SCompBlock and record it - // // } else { - // // } + // if (1 /* the SCompBlock part is not loaded*/) { + // // Append to .data file generate a SCompBlock and record it + // } else { + // } - // // // TODO: need to reset the pCols + // // TODO: need to reset the pCols - // numOfWrites++; - // } + numOfWrites++; + } - // if (pCols->numOfPoints > 0) { - // // TODO: still has data to commit, commit it - // } + if (pCols->numOfPoints > 0) { + // TODO: still has data to commit, commit it + } - // if (1/* SCompBlock part is loaded, write it to .head file*/) { - // // TODO - // } else { - // // TODO: use sendfile send the old part and append the newly added part - // } + if (1/* SCompBlock part is loaded, write it to .head file*/) { + // TODO + } else { + // TODO: use sendfile send the old part and append the newly added part + } } // Write the SCompIdx part // Close all files and return - if (flag) { + if (hasDataToCommit) { // TODO } From ff144921811211b242e7e5e056ec597fffa9c26a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 28 Mar 2020 10:34:51 +0800 Subject: [PATCH 61/85] [TD-32] support multi-table query --- src/client/src/tscParseInsert.c | 10 +- src/client/src/tscSQLParser.c | 8 +- src/client/src/tscServer.c | 11 ++ src/client/src/tscUtil.c | 2 +- src/dnode/src/dnodeRead.c | 10 -- src/inc/taosmsg.h | 3 +- src/query/inc/queryExecutor.h | 1 - src/query/src/qast.c | 10 +- src/query/src/queryExecutor.c | 199 +++++++++++++++++--------------- src/vnode/tsdb/src/tsdbRead.c | 83 +++++++------ 10 files changed, 173 insertions(+), 164 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 9c7a3e0bf0..6d01538d8f 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1238,8 +1238,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _clean; } - if (pCmd->pDataBlocks->nSize > 0) { - // merge according to vgId + if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { goto _error_clean; } @@ -1294,12 +1293,7 @@ int tsParseInsertSql(SSqlObj *pSql) { int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) { int32_t ret = TSDB_CODE_SUCCESS; - -// if (NULL == pSql->asyncTblPos) { -// tscCleanSqlCmd(&pSql->cmd); -// } else { - tscTrace("continue parse sql: %s", pSql->asyncTblPos); -// } + tscTrace("continue parse sql: %s", pSql->asyncTblPos); if (tscIsInsertOrImportData(pSql->sqlstr)) { /* diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 804d4e5268..2505d544e4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2789,7 +2789,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { +static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) { tSQLExprList* pList = pExpr->pParam; if (pList->nExpr <= 0) { return TSDB_CODE_INVALID_SQL; @@ -2815,7 +2815,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuild return TSDB_CODE_SUCCESS; } -static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { +static int32_t tablenameCondToString(tSQLExpr* pExpr, SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); taosStringBuilderAppendString(sb, pExpr->val.pz); @@ -3756,8 +3756,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql return TSDB_CODE_SUCCESS; } - const char* msg = "invalid filter expression"; const char* msg1 = "invalid expression"; + const char* msg2 = "invalid filter expression"; int32_t ret = TSDB_CODE_SUCCESS; @@ -3819,7 +3819,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql taosStringBuilderDestroy(&sb); if (!validateFilterExpr(pQueryInfo)) { - return invalidSqlErrMsg(pQueryInfo->msg, msg); + return invalidSqlErrMsg(pQueryInfo->msg, msg2); } doAddJoinTagsColumnsIntoTagList(pQueryInfo, &condExpr); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 22f8a85757..498a338389 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -847,6 +847,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); } + // serialize tag column query condition if (pQueryInfo->tagCond.numOfTagCond > 0) { STagCond* pTagCond = &pQueryInfo->tagCond; @@ -865,6 +866,16 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } + // tbname in/like query expression should be sent to mgmt node + STagCond* pTagCond = &pQueryInfo->tagCond; + if (pTagCond->tbnameCond.cond != NULL) { + size_t s = strlen(pTagCond->tbnameCond.cond); + memcpy(pMsg, pTagCond->tbnameCond.cond, s); + + pQueryMsg->nameCondLen = htons(s); + pMsg += s; + } + msgLen = pMsg - pStart; tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ec6881db3f..18f491aa9e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -608,7 +608,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { * the payloadLen should be actual message body size * the old value of payloadLen is the allocated payload size */ - pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize; + pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize - sizeof(SMsgDesc); assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0); return TSDB_CODE_SUCCESS; diff --git a/src/dnode/src/dnodeRead.c b/src/dnode/src/dnodeRead.c index 965727c9d0..cb25cbfd52 100644 --- a/src/dnode/src/dnodeRead.c +++ b/src/dnode/src/dnodeRead.c @@ -243,16 +243,6 @@ static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMs taos_queue queue = dnodeGetVnodeRworker(pVnode); taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); - -// SReadMsg readMsg = { -// .rpcMsg = {0}, -// .pCont = qhandle, -// .contLen = 0, -// .pRpcContext = pMsg->pRpcContext, -// }; -// -// taos_queue queue = dnodeGetVnodeRworker(pVnode); -// taosWriteQitem(queue, TSDB_MSG_TYPE_QUERY, &readMsg); } static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 999e293d42..9ac06e4199 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -475,9 +475,10 @@ typedef struct { int64_t slidingTime; // value for sliding window char slidingTimeUnit; // time interval type, for revisement of interval(1d) uint16_t tagCondLen; // tag length in current query + uint16_t nameCondLen; // table name in/like query expression string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; - int16_t orderType; // used in group by xx order by xxx + int16_t orderType; // used in group by xx order by xxx uint64_t groupbyTagIds; int64_t limit; int64_t offset; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index dc7aff105f..2e1d86a942 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -172,7 +172,6 @@ typedef struct SQueryRuntimeEnv { typedef struct SQInfo { void* signature; -// void* param; // pointer to the RpcReadMsg TSKEY startTime; TSKEY elapsedTime; int32_t pointsInterpo; diff --git a/src/query/src/qast.c b/src/query/src/qast.c index b301bbf043..c6c65eba27 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -16,6 +16,7 @@ #include "qast.h" #include #include +#include "../../client/inc/tschemautil.h" #include "os.h" #include "qsqlparser.h" #include "qsyntaxtreefunction.h" @@ -107,7 +108,7 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols, return NULL; } - size_t nodeSize = sizeof(tSQLSyntaxNode); + size_t nodeSize = sizeof(tSQLSyntaxNode); tSQLSyntaxNode *pNode = NULL; if (pToken->type == TK_ID || pToken->type == TK_TBNAME) { @@ -239,9 +240,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode } static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0; - - t0 = tStrGetToken(str, i, false, 0, NULL); + SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } @@ -343,7 +342,8 @@ void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t return; } - int32_t pos = 0; + int32_t pos = 0; + tSQLSyntaxNode *pStxNode = createSyntaxTree(pSchema, numOfCols, src, &pos); if (pStxNode != NULL) { assert(pStxNode->nodeType == TSQL_NODE_EXPR); diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 86baf44f45..2f21936808 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5265,11 +5265,11 @@ void qTableQuery(SQInfo *pQInfo) { // vnodeDecRefCount(pQInfo); } -static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { +static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { int32_t j = 0; - while (j < pQueryTableMsg->numOfCols) { - if (pExprMsg->colInfo.colId == pQueryTableMsg->colList[j].colId) { + while (j < pQueryMsg->numOfCols) { + if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { break; } @@ -5279,44 +5279,44 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncEx return j; } -bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { - int32_t j = getColumnIndexInSource(pQueryTableMsg, pExprMsg); - return j < pQueryTableMsg->numOfCols; +bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { + int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg); + return j < pQueryMsg->numOfCols; } -static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { - if (pQueryTableMsg->intervalTime < 0) { - dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryTableMsg, pQueryTableMsg->intervalTime); +static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryMsg) { + if (pQueryMsg->intervalTime < 0) { + dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime); return -1; } - if (pQueryTableMsg->numOfCols <= 0 || pQueryTableMsg->numOfCols > TSDB_MAX_COLUMNS) { - dError("qmsg:%p illegal value of numOfCols %d", pQueryTableMsg, pQueryTableMsg->numOfCols); + if (pQueryMsg->numOfCols <= 0 || pQueryMsg->numOfCols > TSDB_MAX_COLUMNS) { + dError("qmsg:%p illegal value of numOfCols %d", pQueryMsg, pQueryMsg->numOfCols); return -1; } - if (pQueryTableMsg->numOfTables <= 0) { - dError("qmsg:%p illegal value of numOfTables %d", pQueryTableMsg, pQueryTableMsg->numOfTables); + if (pQueryMsg->numOfTables <= 0) { + dError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); return -1; } - if (pQueryTableMsg->numOfGroupCols < 0) { - dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryTableMsg, pQueryTableMsg->numOfGroupCols); + if (pQueryMsg->numOfGroupCols < 0) { + dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); return -1; } - if (pQueryTableMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryTableMsg->numOfOutputCols <= 0) { - dError("qmsg:%p illegal value of output columns %d", pQueryTableMsg, pQueryTableMsg->numOfOutputCols); + if (pQueryMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutputCols <= 0) { + dError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutputCols); return -1; } return 0; } -static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArray** pTableIdList) { - assert(pQueryTableMsg->numOfTables > 0); +static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { + assert(pQueryMsg->numOfTables > 0); - *pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); @@ -5326,7 +5326,7 @@ static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArra taosArrayPush(*pTableIdList, pTableIdInfo); pMsg += sizeof(STableIdInfo); - for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { + for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); @@ -5341,49 +5341,47 @@ static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArra } /** - * pQueryTableMsg->head has been converted before this function is called. + * pQueryMsg->head has been converted before this function is called. * - * @param pQueryTableMsg + * @param pQueryMsg * @param pTableIdList * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - wchar_t** tagCond) { - pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); +static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, + wchar_t** tagCond, char** nameCond) { + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); - pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); - pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); - pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); - pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); - pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); - pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); - - pQueryTableMsg->order = htons(pQueryTableMsg->order); - pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId); + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htons(pQueryMsg->queryType); - pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); - - pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); - pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); - pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); - pQueryTableMsg->tagCondLen = htons(pQueryTableMsg->tagCondLen); - - pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); - pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); - pQueryTableMsg->tsNumOfBlocks = htonl(pQueryTableMsg->tsNumOfBlocks); - pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder); + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); + pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->nameCondLen = htons(pQueryMsg->nameCondLen); + pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); + pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); // query msg safety check - if (validateQueryMeterMsg(pQueryTableMsg) != 0) { + if (validateQueryMeterMsg(pQueryMsg) != 0) { return TSDB_CODE_INVALID_QUERY_MSG; } - char *pMsg = (char *)(pQueryTableMsg->colList) + sizeof(SColumnInfo) * pQueryTableMsg->numOfCols; + char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; - for (int32_t col = 0; col < pQueryTableMsg->numOfCols; ++col) { - SColumnInfo* pColInfo = &pQueryTableMsg->colList[col]; + for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { + SColumnInfo* pColInfo = &pQueryMsg->colList[col]; pColInfo->colId = htons(pColInfo->colId); pColInfo->type = htons(pColInfo->type); @@ -5423,10 +5421,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId bool hasArithmeticFunction = false; - *pExpr = calloc(pQueryTableMsg->numOfOutputCols, POINTER_BYTES); + *pExpr = calloc(pQueryMsg->numOfOutputCols, POINTER_BYTES); SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; - for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); @@ -5457,7 +5455,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId return TSDB_CODE_INVALID_QUERY_MSG; } } else { - if (!vnodeValidateExprColumnInfo(pQueryTableMsg, pExprMsg)) { + if (!vnodeValidateExprColumnInfo(pQueryMsg, pExprMsg)) { return TSDB_CODE_INVALID_QUERY_MSG; } } @@ -5465,55 +5463,59 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId pExprMsg = (SSqlFuncExprMsg *)pMsg; } - pQueryTableMsg->colNameLen = htonl(pQueryTableMsg->colNameLen); + pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen); if (hasArithmeticFunction) { // column name array - assert(pQueryTableMsg->colNameLen > 0); - pQueryTableMsg->colNameList = (int64_t)pMsg; - pMsg += pQueryTableMsg->colNameLen; + assert(pQueryMsg->colNameLen > 0); + pQueryMsg->colNameList = (int64_t)pMsg; + pMsg += pQueryMsg->colNameLen; } - pMsg = createTableIdList(pQueryTableMsg, pMsg, pTableIdList); + pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); - if (pQueryTableMsg->numOfGroupCols > 0) { // group by tag columns -// if (pQueryTableMsg->numOfGroupCols > 0) { -// pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); + if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns +// if (pQueryMsg->numOfGroupCols > 0) { +// pQueryMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryMsg->numOfTagsCols]); // } else { -// pQueryTableMsg->groupbyTagIds = 0; +// pQueryMsg->groupbyTagIds = 0; // } - pQueryTableMsg->orderByIdx = htons(pQueryTableMsg->orderByIdx); - pQueryTableMsg->orderType = htons(pQueryTableMsg->orderType); + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); + pQueryMsg->orderType = htons(pQueryMsg->orderType); - pMsg += sizeof(SColIndexEx) * pQueryTableMsg->numOfGroupCols; + pMsg += sizeof(SColIndexEx) * pQueryMsg->numOfGroupCols; } else { - pQueryTableMsg->groupbyTagIds = 0; + pQueryMsg->groupbyTagIds = 0; } - pQueryTableMsg->interpoType = htons(pQueryTableMsg->interpoType); - if (pQueryTableMsg->interpoType != TSDB_INTERPO_NONE) { - pQueryTableMsg->defaultVal = (uint64_t)(pMsg); + pQueryMsg->interpoType = htons(pQueryMsg->interpoType); + if (pQueryMsg->interpoType != TSDB_INTERPO_NONE) { + pQueryMsg->defaultVal = (uint64_t)(pMsg); int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { v[i] = htobe64(v[i]); } - pMsg += sizeof(int64_t) * pQueryTableMsg->numOfOutputCols; + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols; } // the tag query condition expression string is located at the end of query msg - if (pQueryTableMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryTableMsg->tagCondLen * TSDB_NCHAR_SIZE); - memcpy(*tagCond, pMsg, pQueryTableMsg->tagCondLen * TSDB_NCHAR_SIZE); + if (pQueryMsg->tagCondLen > 0) { + *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); + } + + if (pQueryMsg->nameCondLen > 0) { + *nameCond = strndup(pMsg, pQueryMsg->nameCondLen); } dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64, - pQueryTableMsg, pQueryTableMsg->numOfTables, pQueryTableMsg->window.skey, pQueryTableMsg->window.ekey, - pQueryTableMsg->numOfGroupCols, pQueryTableMsg->order, pQueryTableMsg->orderType, - pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, - pQueryTableMsg->numOfCols, pQueryTableMsg->intervalTime, pQueryTableMsg->interpoType, pQueryTableMsg->tsLen, - pQueryTableMsg->limit, pQueryTableMsg->offset); + pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, + pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->orderType, + pQueryMsg->orderByIdx, pQueryMsg->numOfOutputCols, + pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, + pQueryMsg->limit, pQueryMsg->offset); return 0; } @@ -6047,54 +6049,59 @@ _error: return code; } -int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { - assert(pQueryTableMsg != NULL); +int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) { + assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; SArray *pTableIdList = NULL; SSqlFuncExprMsg** pExprMsg = NULL; wchar_t* tagCond = NULL; + char* nameCond = NULL; - if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &nameCond)) != TSDB_CODE_SUCCESS) { return code; } - if (pQueryTableMsg->numOfTables <= 0) { - dError("Invalid number of tables to query, numOfTables:%d", pQueryTableMsg->numOfTables); + if (pQueryMsg->numOfTables <= 0) { + dError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } // todo check vnode status if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { - dError("qmsg:%p, SQueryTableMsg wrong format", pQueryTableMsg); + dError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); code = TSDB_CODE_INVALID_QUERY_MSG; goto _query_over; } SSqlFunctionExpr *pExprs = NULL; - if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { + if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { goto _query_over; } - SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryTableMsg, &code); - if ((pGroupbyExpr == NULL && pQueryTableMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { + SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, &code); + if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } // super table query - if ((pQueryTableMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { + SArray* res = NULL; + if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { STableId* id = taosArrayGet(pTableIdList, 0); + id->uid = -1; - SArray* res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryTableMsg->tagCondLen); + res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen); if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode code = TSDB_CODE_SUCCESS; goto _query_over; } + } else { + res = pTableIdList; } - code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); + code = createQInfo(pQueryMsg, pGroupbyExpr, pExprs, res, tsdb, pQInfo); _query_over: if (code != TSDB_CODE_SUCCESS) { @@ -6103,16 +6110,16 @@ _query_over: // if failed to add ref for all meters in this query, abort current query // if (code != TSDB_CODE_SUCCESS) { - // vnodeDecQueryRefCount(pQueryTableMsg, pMeterObjList, incNumber); + // vnodeDecQueryRefCount(pQueryMsg, pMeterObjList, incNumber); // } // - // tfree(pQueryTableMsg->pSqlFuncExprs); + // tfree(pQueryMsg->pSqlFuncExprs); // tfree(pMeterObjList); // ret = vnodeSendQueryRspMsg(pObj, code, pObj->qhandle); // - // tfree(pQueryTableMsg->pSidExtInfo); - // for(int32_t i = 0; i < pQueryTableMsg->numOfCols; ++i) { - // vnodeFreeColumnInfo(&pQueryTableMsg->colList[i]); + // tfree(pQueryMsg->pSidExtInfo); + // for(int32_t i = 0; i < pQueryMsg->numOfCols; ++i) { + // vnodeFreeColumnInfo(&pQueryMsg->colList[i]); // } // // atomic_fetch_add_32(&vnodeSelectReqNum, 1); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 0140e2e11e..3119fdc7cc 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -73,14 +73,15 @@ typedef struct SQueryFilesInfo { char dbFilePathPrefix[PATH_MAX]; } SQueryFilesInfo; -typedef struct STableQueryRec { +typedef struct STableQueryInfo { + STableId tableId; TSKEY lastKey; STable * pTableObj; int64_t offsetInHeaderFile; int32_t numOfBlocks; int32_t start; SCompBlock *pBlock; -} STableQueryRec; +} STableQueryInfo; typedef struct { SCompBlock *compBlock; @@ -89,7 +90,7 @@ typedef struct { typedef struct STableDataBlockInfoEx { SCompBlockFields pBlock; - STableQueryRec * pMeterDataInfo; + STableQueryInfo* pMeterDataInfo; int32_t blockIndex; int32_t groupIdx; /* number of group is less than the total number of meters */ } STableDataBlockInfoEx; @@ -100,12 +101,11 @@ typedef struct STsdbQueryHandle { SQueryHandlePos cur; // current position SQueryHandlePos start; // the start position, used for secondary/third iteration int32_t unzipBufSize; - char *unzipBuffer; - char *secondaryUnzipBuffer; + char *unzipBuffer; + char *secondaryUnzipBuffer; SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ - SQueryFilesInfo vnodeFileInfo; int16_t numOfRowsPerPage; @@ -113,21 +113,22 @@ typedef struct STsdbQueryHandle { int16_t order; STimeWindow window; // the primary query time window that applies to all queries int32_t blockBufferSize; - SCompBlock *pBlock; + SCompBlock* pBlock; int32_t numOfBlocks; SField ** pFields; SArray * pColumns; // column list, SColumnInfoEx array list - SArray * pTableIdList; // table id object list bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. - - STableDataBlockInfoEx *pDataBlockInfoEx; - STableQueryRec * pTableQueryInfo; - int32_t tableIndex; - bool isFirstSlot; - void * qinfo; // query info handle, for debug purpose + SArray* pTableQueryInfo; + int32_t activeIndex; + + int32_t tableIndex; + bool isFirstSlot; + void * qinfo; // query info handle, for debug purpose + SSkipListIterator* memIter; + STableDataBlockInfoEx *pDataBlockInfoEx; } STsdbQueryHandle; int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { @@ -263,25 +264,27 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; - pQueryHandle->pTableIdList = idList; pQueryHandle->pColumns = pColumnInfo; pQueryHandle->loadDataAfterSeek = false; pQueryHandle->isFirstSlot = true; - // only support table query - assert(taosArrayGetSize(idList) == 1); + size_t size = taosArrayGetSize(idList); + assert(size >= 1); + + pQueryHandle->pTableQueryInfo = taosArrayInit(size, sizeof(STableQueryInfo)); + for(int32_t i = 0; i < size; ++i) { + STableId id = *(STableId*) taosArrayGet(idList, i); + + STableQueryInfo info = { + .lastKey = pQueryHandle->window.skey, + .tableId = id, + .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed + }; + + taosArrayPush(pQueryHandle->pTableQueryInfo, &info); + } - pQueryHandle->pTableQueryInfo = calloc(1, sizeof(STableQueryRec)); - STableQueryRec* pTableQRec = pQueryHandle->pTableQueryInfo; - - pTableQRec->lastKey = pQueryHandle->window.skey; - - STableIdInfo* idInfo = taosArrayGet(pQueryHandle->pTableIdList, 0); - - STable *pTable = tsdbGetTableByUid(tsdbGetMeta(pQueryHandle->pTsdb), idInfo->uid); - assert(pTable != NULL); - - pTableQRec->pTableObj = pTable; + pQueryHandle->activeIndex = 0; // malloc buffer in order to load data from file int32_t numOfCols = taosArrayGetSize(pColumnInfo); @@ -313,7 +316,9 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STable *pTable = pHandle->pTableQueryInfo->pTableObj; + STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex); + + STable *pTable = pTableQInfo->pTableObj; // no data in cache, abort if (pTable->mem == NULL && pTable->imem == NULL) { @@ -321,7 +326,7 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { } // all data in mem are checked already. - if (pHandle->pTableQueryInfo->lastKey > pTable->mem->keyLast) { + if (pTableQInfo->lastKey > pTable->mem->keyLast) { return false; } @@ -364,9 +369,9 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max // copy data from cache into data block SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STableIdInfo* idInfo = taosArrayGet(pHandle->pTableIdList, 0); - STable *pTable = pHandle->pTableQueryInfo->pTableObj; + STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex); + STable *pTable = pTableQInfo->pTableObj; TSKEY skey = 0, ekey = 0; int32_t rows = 0; @@ -382,14 +387,14 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { } SDataBlockInfo blockInfo = { - .uid = idInfo->uid, - .sid = idInfo->sid, + .uid = pTable->tableId.uid, + .sid = pTable->tableId.tid, .size = rows, .window = {.skey = skey, .ekey = ekey} }; // update the last key value - pHandle->pTableQueryInfo->lastKey = ekey + 1; + pTableQInfo->lastKey = ekey + 1; return blockInfo; } @@ -427,7 +432,9 @@ static SArray* createTableIdArrayList(struct STsdbRepo* tsdb, int64_t uid) { SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); while(tSkipListIterNext(iter)) { - STable* t = *(STable**) tSkipListIterGet(iter); + SSkipListNode* pNode = tSkipListIterGet(iter); + STable* t = *(STable**) SL_GET_NODE_DATA(pNode); + taosArrayPush(pList, &t->tableId); } @@ -696,7 +703,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { return true; } -static int32_t mgmtFilterMeterByIndex(STable* pSTable, SArray* pRes, const char* pCond) { +static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond) { STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); tSQLBinaryExpr* pExpr = NULL; @@ -736,7 +743,7 @@ SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *p STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); assert(pSTable != NULL); - if (mgmtFilterMeterByIndex(pSTable, result, str) == TSDB_CODE_SUCCESS) { + if (doQueryTableList(pSTable, result, str) == TSDB_CODE_SUCCESS) { return result; } } From f29b0fcaf7cfb1b0b509108a85a36911fedc2337 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 28 Mar 2020 12:08:41 +0800 Subject: [PATCH 62/85] TD-34 --- src/common/inc/dataformat.h | 1 + src/common/src/dataformat.c | 4 + src/vnode/tsdb/inc/tsdbFile.h | 5 +- src/vnode/tsdb/src/tsdbFile.c | 55 +++++- src/vnode/tsdb/src/tsdbMain.c | 313 +++++++++++++++++++++------------- 5 files changed, 257 insertions(+), 121 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index 52b2d1e156..a7016061e4 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -132,6 +132,7 @@ void tdResetDataCols(SDataCols *pCols); void tdInitDataCols(SDataCols *pCols, STSchema *pSchema); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols); +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); #ifdef __cplusplus } diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 8f6a40805f..56565b6e5c 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -353,6 +353,10 @@ void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) { } pCols->numOfPoints++; } +// Pop pointsToPop points from the SDataCols +void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) { + +} /** * Return the first part length of a data row for a schema diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index c761f8520e..e4592ffa2e 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -134,12 +134,15 @@ typedef struct { int64_t uid; // For recovery usage SCompCol cols[]; } SCompData; -int tsdbCopyCompBlockToFile(SFile *outFile, SFile *inFile, SCompInfo *pCompInfo, int index, int isOutLast); + +int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols); int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf); int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf); int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf); +int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData); + // TODO: need an API to merge all sub-block data into one int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols); diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index f22274531d..c1b7ae2382 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -103,8 +103,59 @@ int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { return 0; } -int tsdbCopyCompBlockToFile(SFile *outFile, SFile *inFile, SCompInfo *pCompInfo, int index, int isOutLast) { - // TODO + +int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) { + SCompBlock *pBlock = pStartBlock; + for (int i = 0; i < numOfBlocks; i++) { + if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1; + for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) { + SCompCol *pCompCol = &(pCompData->cols[iCol]); + pCols->numOfPoints += pBlock->numOfPoints; + int k = 0; + for (; k < pCols->numOfCols; k++) { + if (pCompCol->colId == pCols->cols[k].colId) break; + } + + if (tsdbLoadColData(pFile, pCompCol, pBlock->offset, + (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0) + return -1; + } + pStartBlock++; + } + return 0; +} + +int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) { + SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx); + SCompBlock *pStartBlock = NULL; + SCompBlock *pBlock = NULL; + int numOfBlocks = pSuperBlock->numOfSubBlocks; + + if (numOfBlocks == 1) + pStartBlock = pSuperBlock; + else + pStartBlock = TSDB_COMPBLOCK_AT(pCompInfo, pSuperBlock->offset); + + int maxNumOfCols = 0; + pBlock = pStartBlock; + for (int i = 0; i < numOfBlocks; i++) { + if (pBlock->numOfCols > maxNumOfCols) maxNumOfCols = pBlock->numOfCols; + pBlock++; + } + + SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * maxNumOfCols); + if (pCompData == NULL) return -1; + + // Load data from the block + if (tsdbLoadDataBlock(pOutFile, pStartBlock, numOfBlocks, pCols, pCompData)); + + // Write data block to the file + { + // TODO + } + + + if (pCompData) free(pCompData); return 0; } diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 3945ffabda..599af73813 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -88,6 +88,8 @@ static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols); +static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey); +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -872,13 +874,12 @@ static void *tsdbCommitData(void *arg) { } static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols) { - int hasDataToCommit = 0; int isNewLastFile = 0; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; - SFile tFile, lFile; + SFile hFile, lFile; SFileGroup *pGroup = NULL; SCompIdx * pIndices = NULL; SCompInfo * pCompInfo = NULL; @@ -889,125 +890,181 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters TSKEY minKey = 0, maxKey = 0; tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable * pTable = pMeta->tables[tid]; - SSkipListIterator *pIter = iters[tid]; - int isLoadCompBlocks = 0; - char dataDir[128] = "\0"; + // Check if there are data to commit to this file + int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); + if (!hasDataToCommit) return 0; // No data to commit, just return - if (pIter == NULL) continue; - tdInitDataCols(pCols, pTable->schema); - - int numOfWrites = 0; - int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; // We keep 20% of space for merge purpose - // Loop to read columns from cache - while (tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols)) { - if (!hasDataToCommit) { - // There are data to commit to this fileId, we need to create/open it for read/write. - // At the meantime, we set the flag to prevent further create/open operations - tsdbGetDataDirName(pRepo, dataDir); - - if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { - // TODO: deal with the ERROR here - } - // Open files for commit - pGroup = tsdbOpenFilesForCommit(pFileH, fid); - if (pGroup == NULL) { - // TODO: deal with the ERROR here - } - // TODO: open .h file and if neccessary, open .l file - tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &tFile, 1, 0); - if (1 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { - // TODO: make it not to write the last file every time - tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); - isNewLastFile = 1; - } - - // load the SCompIdx part - pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); - if (pIndices == NULL) { // TODO: deal with the ERROR - } - if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { // TODO: deal with the ERROR here - } - - // sendfile those not need to changed table content - lseek(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, - SEEK_SET); - lseek(tFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); - for (int ttid = 0; ttid < tid; ttid++) { - SCompIdx * tIdx= &pIndices[ttid]; - if (tIdx->len <= 0) continue; - if (isNewLastFile && tIdx->hasLast) { - // TODO: Need to load the SCompBlock part and copy to new last file - pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, tIdx->len); - if (pCompInfo == NULL) { /* TODO */} - if (tsdbLoadCompBlocks(pGroup, tIdx, (void *)pCompInfo) < 0) {/* TODO */} - SCompBlock *pLastBlock = TSDB_COMPBLOCK_AT(pCompInfo, tIdx->numOfSuperBlocks - 1); - int numOfSubBlocks = pLastBlock->numOfSubBlocks; - assert(pLastBlock->last); - if (tsdbCopyCompBlockToFile(&pGroup->files[TSDB_FILE_TYPE_LAST], &lFile, pCompInfo, tIdx->numOfSuperBlocks, 1 /* isOutLast*/) < 0) {/* TODO */} - { - if (numOfSubBlocks > 1) tIdx->len -= (sizeof(SCompBlock) * numOfSubBlocks); - tIdx->checksum = 0; - } - write(tFile.fd, (void *)pCompInfo, tIdx->len); - tFile.size += tIdx->len; - } else { - sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, tFile.fd, NULL, tIdx->len); - tFile.size += tIdx->len; - } - } - - hasDataToCommit = 1; - } - - SCompIdx *pIdx = &pIndices[tid]; - - /* The first time to write to the table, need to decide - * if it is neccessary to load the SComplock part. If it - * is needed, just load it, or, just use sendfile and - * append it. - */ - if (numOfWrites == 0 && pIdx->offset > 0) { - if (dataColsKeyFirst(pCols) <= pIdx->maxKey || pIdx->hasLast) { - pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len); - if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { - // TODO: deal with the ERROR here - } - if (pCompInfo->uid == pTable->tableId.uid) isLoadCompBlocks = 1; - } else { - // TODO: sendfile the prefix part - } - } - - // if (tsdbWriteBlockToFile(pGroup, pCompInfo, pIdx, isLoadCompBlocks, pBlock, pCols) < 0) { - // // TODO: deal with the ERROR here - // } - - // pCompInfo = tsdbMergeBlock(pCompInfo, pBlock); - - - // if (1 /* the SCompBlock part is not loaded*/) { - // // Append to .data file generate a SCompBlock and record it - // } else { - // } - - // // TODO: need to reset the pCols - - numOfWrites++; - } - - if (pCols->numOfPoints > 0) { - // TODO: still has data to commit, commit it - } - - if (1/* SCompBlock part is loaded, write it to .head file*/) { - // TODO - } else { - // TODO: use sendfile send the old part and append the newly added part - } + // Create and open files for commit + tsdbGetDataDirName(pRepo, dataDir); + if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) {/* TODO */} + pGroup = tsdbOpenFilesForCommit(pFileH, fid); + if (pGroup == NULL) {/* TODO */} + tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 0); + if (1 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { + // TODO: make it not to write the last file every time + tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); + isNewLastFile = 1; } + // Load the SCompIdx + pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) *pCfg->maxTables); + if (pIndices == NULL) {/* TODO*/} + if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) {/* TODO */} + +// Loop to commit data in each table + for (int tid = 0; tid < pCfg->maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + SSkipListIterator *pIter = iters[tid]; + SCompIdx *pIdx = &pIndices[tid]; + + if (!tsdbHasDataInRange(pIter, minKey, maxKey)) { + // This table does not have data in this range, just copy its head part and last + // part (if neccessary) to new file + if (pIdx->len != 0) { // has old data + if (isNewLastFile && pIdx->hasLast) { + // Need to load SCompBlock part and copy to new file + if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) {/* TODO */} + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) {/* TODO */} + + // Copy the last block from old last file to new file + // tsdbCopyBlockData() + } else { + pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); + sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); + hFile.size += pIdx->len; + } + } + continue; + } + + // while () { + + // } + } + + // for (int tid = 0; tid < pCfg->maxTables; tid++) { + // STable * pTable = pMeta->tables[tid]; + // SSkipListIterator *pIter = iters[tid]; + // int isLoadCompBlocks = 0; + // char dataDir[128] = "\0"; + + // if (pIter == NULL) { + // if (hasDataToCommit && isNewLastFile()) + // continue; + // } + // tdInitDataCols(pCols, pTable->schema); + + // int numOfWrites = 0; + // int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; // We keep 20% of space for merge purpose + // // Loop to read columns from cache + // while (tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols)) { + // if (!hasDataToCommit) { + // // There are data to commit to this fileId, we need to create/open it for read/write. + // // At the meantime, we set the flag to prevent further create/open operations + // tsdbGetDataDirName(pRepo, dataDir); + + // if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { + // // TODO: deal with the ERROR here + // } + // // Open files for commit + // pGroup = tsdbOpenFilesForCommit(pFileH, fid); + // if (pGroup == NULL) { + // // TODO: deal with the ERROR here + // } + // // TODO: open .h file and if neccessary, open .l file + // tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &tFile, 1, 0); + // if (1 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { + // // TODO: make it not to write the last file every time + // tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); + // isNewLastFile = 1; + // } + + // // load the SCompIdx part + // pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); + // if (pIndices == NULL) { // TODO: deal with the ERROR + // } + // if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { // TODO: deal with the ERROR here + // } + + // // sendfile those not need to changed table content + // lseek(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, + // SEEK_SET); + // lseek(tFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); + // for (int ttid = 0; ttid < tid; ttid++) { + // SCompIdx * tIdx= &pIndices[ttid]; + // if (tIdx->len <= 0) continue; + // if (isNewLastFile && tIdx->hasLast) { + // // TODO: Need to load the SCompBlock part and copy to new last file + // pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, tIdx->len); + // if (pCompInfo == NULL) { /* TODO */} + // if (tsdbLoadCompBlocks(pGroup, tIdx, (void *)pCompInfo) < 0) {/* TODO */} + // SCompBlock *pLastBlock = TSDB_COMPBLOCK_AT(pCompInfo, tIdx->numOfSuperBlocks - 1); + // int numOfSubBlocks = pLastBlock->numOfSubBlocks; + // assert(pLastBlock->last); + // if (tsdbCopyCompBlockToFile(&pGroup->files[TSDB_FILE_TYPE_LAST], &lFile, pCompInfo, tIdx->numOfSuperBlocks, 1 /* isOutLast*/) < 0) {/* TODO */} + // { + // if (numOfSubBlocks > 1) tIdx->len -= (sizeof(SCompBlock) * numOfSubBlocks); + // tIdx->checksum = 0; + // } + // write(tFile.fd, (void *)pCompInfo, tIdx->len); + // tFile.size += tIdx->len; + // } else { + // sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, tFile.fd, NULL, tIdx->len); + // tFile.size += tIdx->len; + // } + // } + + // hasDataToCommit = 1; + // } + + // SCompIdx *pIdx = &pIndices[tid]; + + // /* The first time to write to the table, need to decide + // * if it is neccessary to load the SComplock part. If it + // * is needed, just load it, or, just use sendfile and + // * append it. + // */ + // if (numOfWrites == 0 && pIdx->offset > 0) { + // if (dataColsKeyFirst(pCols) <= pIdx->maxKey || pIdx->hasLast) { + // pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len); + // if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { + // // TODO: deal with the ERROR here + // } + // if (pCompInfo->uid == pTable->tableId.uid) isLoadCompBlocks = 1; + // } else { + // // TODO: sendfile the prefix part + // } + // } + + // int numOfPointsWritten = tsdbWriteBlockToFile(pGroup, pCompInfo, pIdx, isLoadCompBlocks, pBlock, pCols); + // if (numOfPointsWritten < 0) { + // // TODO: deal with the ERROR here + // } + + // // pCompInfo = tsdbMergeBlock(pCompInfo, pBlock); + + + // // if (1 /* the SCompBlock part is not loaded*/) { + // // // Append to .data file generate a SCompBlock and record it + // // } else { + // // } + + // // // TODO: need to reset the pCols + + // numOfWrites++; + // } + + // if (pCols->numOfPoints > 0) { + // // TODO: still has data to commit, commit it + // } + + // if (1/* SCompBlock part is loaded, write it to .head file*/) { + // // TODO + // } else { + // // TODO: use sendfile send the old part and append the newly added part + // } + // } + // Write the SCompIdx part // Close all files and return @@ -1018,5 +1075,25 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters if (pIndices) free(pIndices); if (pCompInfo) free(pCompInfo); + return 0; +} + +static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey) { + if (pIter == NULL) return 0; + + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) return 0; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) >= minKey && dataRowKey(row) <= maxKey) return 1; + + return 0; +} + +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { + for (int i = 0; i < nIters; i++) { + SSkipListIterator *pIter = iters[i]; + if (tsdbHasDataInRange(pIter, minKey, maxKey)) return 1; + } return 0; } \ No newline at end of file From e45701a6675ab0e4df81dbe4b33945a99908bb34 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 28 Mar 2020 13:01:03 +0800 Subject: [PATCH 63/85] [TD-32] fix bugs in super table query --- src/client/src/tscAsync.c | 2 +- src/client/src/tscSecondaryMerge.c | 2 +- src/client/src/tscSubquery.c | 3 +- src/query/inc/qtsbuf.h | 8 +- src/query/inc/queryExecutor.h | 24 +- src/query/src/queryExecutor.c | 635 +++++++++++++---------------- src/vnode/tsdb/src/tsdbMeta.c | 4 +- 7 files changed, 310 insertions(+), 368 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 366faa15dc..985a6741f9 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -514,7 +514,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscTansformSQLFunctionForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); } else { - tscTrace("%p get tableMeta/metricMeta successfully", pSql); + tscTrace("%p get tableMeta successfully", pSql); } tscDoQuery(pSql); diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index d69f6d295f..0ca079bc6e 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes); + (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1); if (*pMemBuffer == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 21ce270466..ccf187ac64 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1011,8 +1011,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = 0; + int32_t numOfSubQueries = 1; // int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; + assert(numOfSubQueries > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qtsbuf.h index 1afdb0cd6c..8e014e5feb 100644 --- a/src/query/inc/qtsbuf.h +++ b/src/query/inc/qtsbuf.h @@ -48,10 +48,10 @@ typedef struct STSElem { } STSElem; typedef struct STSCursor { - int32_t vnodeIndex; - int32_t blockIndex; - int32_t tsIndex; - int32_t order; + int32_t vnodeIndex; + int32_t blockIndex; + int32_t tsIndex; + uint32_t order; } STSCursor; typedef struct STSBlock { diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 2e1d86a942..5adce04efa 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -32,12 +32,6 @@ typedef struct SData { char data[]; } SData; -enum { -// ST_QUERY_KILLED = 0, // query killed - ST_QUERY_PAUSED = 1, // query paused, due to full of the response buffer - ST_QUERY_COMPLETED = 2, // query completed -}; - struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -60,18 +54,20 @@ typedef struct SWindowStatus { } SWindowStatus; typedef struct SWindowResult { - uint16_t numOfRows; + uint16_t numOfRows; // number of rows of current time window SPosInfo pos; // Position of current result in disk-based output buffer SResultInfo* resultInfo; // For each result column, there is a resultInfo STimeWindow window; // The time window that current result covers. - SWindowStatus status; + SWindowStatus status; // this result status: closed or opened } SWindowResult; typedef struct SResultRec { - int64_t total; - int64_t size; - int64_t capacity; - int32_t threshold; // the threshold size, when the number of rows in result buffer, return to client + int64_t total; // total generated result size in rows + int64_t size; // current result set size in rows + int64_t capacity; // capacity of current result output buffer + + // result size threshold in rows. If the result buffer is larger than this, pause query and return to client + int32_t threshold; } SResultRec; typedef struct SWindowResInfo { @@ -99,7 +95,6 @@ typedef struct SSingleColumnFilterInfo { void* pData; } SSingleColumnFilterInfo; -/* intermediate pos during multimeter query involves interval */ typedef struct STableQueryInfo { int64_t lastKey; STimeWindow win; @@ -107,7 +102,7 @@ typedef struct STableQueryInfo { int16_t queryRangeSet; // denote if the query range is set, only available for interval query int64_t tag; STSCursor cur; - int32_t sid; // for retrieve the page id list + int32_t tid; // for retrieve the page id list SWindowResInfo windowResInfo; } STableQueryInfo; @@ -116,7 +111,6 @@ typedef struct STableDataInfo { int32_t numOfBlocks; int32_t start; // start block index int32_t tableIndex; - void* pMeterObj; int32_t groupIdx; // group id in table list STableQueryInfo* pTableQInfo; } STableDataInfo; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 2f21936808..224512f0ae 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -375,29 +375,6 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { return false; } -/** - * - * @param pQuery - * @param pDataBlockInfo - * @param forwardStep - * @return TRUE means query not completed, FALSE means query is completed - */ -static bool queryPaused(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, int32_t forwardStep) { - // output buffer is full, pause current query - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - // assert((QUERY_IS_ASC_QUERY(pQuery) && forwardStep + pQuery->pos <= pDataBlockInfo->size) || - // (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->pos - forwardStep + 1 >= 0)); - // - return true; - } - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - return true; - } - - return false; -} - static bool isTopBottomQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; @@ -1690,7 +1667,7 @@ bool notHasQueryTimeRange(SQuery *pQuery) { (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); } -bool needSupplementaryScan(SQuery *pQuery) { +static bool needReverseScan(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { @@ -2664,7 +2641,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfRes = 0; SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); int32_t forwardStep = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &numOfRes, &pRuntimeEnv->windowResInfo, pDataBlock); @@ -2950,7 +2926,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return leftTimestamp > rightTimestamp ? 1 : -1; } -int32_t mergeMetersResultToOneGroups(SQInfo *pQInfo) { +int32_t mergeResultsToGroup(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -2990,7 +2966,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQInfo->numOfGroupResultPages = 0; // current results of group has been sent to client, try next group - if (mergeMetersResultToOneGroups(pQInfo) != TSDB_CODE_SUCCESS) { + if (mergeResultsToGroup(pQInfo) != TSDB_CODE_SUCCESS) { return; // failed to save data in the disk } @@ -3071,9 +3047,9 @@ int32_t doMergeMetersResultsToGroupRes(SQInfo *pQInfo, STableDataInfo *pTableDat // todo opt for the case of one table per group int32_t numOfMeters = 0; for (int32_t i = start; i < end; ++i) { - int32_t sid = pTableDataInfo[i].pTableQInfo->sid; + int32_t tid = pTableDataInfo[i].pTableQInfo->tid; - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, sid); + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, tid); if (list.size > 0 && pTableDataInfo[i].pTableQInfo->windowResInfo.size > 0) { pTableList[numOfMeters] = &pTableDataInfo[i]; numOfMeters += 1; @@ -3240,10 +3216,9 @@ void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pRes } } -void setMeterDataInfo(STableDataInfo *pTableDataInfo, void *pMeterObj, int32_t meterIdx, int32_t groupId) { - pTableDataInfo->pMeterObj = pMeterObj; +void setTableDataInfo(STableDataInfo *pTableDataInfo, int32_t tableIndex, int32_t groupId) { pTableDataInfo->groupIdx = groupId; - pTableDataInfo->tableIndex = meterIdx; + pTableDataInfo->tableIndex = tableIndex; } static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo *pWindowResInfo, int32_t order) { @@ -3297,7 +3272,7 @@ void disableFunctForTableSuppleScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order pQuery->order.order = pQuery->order.order ^ 1u; } -void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { +void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -3322,7 +3297,7 @@ void disableFunctForSuppleScan(SQInfo *pQInfo, int32_t order) { pQuery->order.order = (pQuery->order.order) ^ 1u; } -void enableFunctForMasterScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { +void enableFuncForForwardScan(SQueryRuntimeEnv *pRuntimeEnv, int32_t order) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { @@ -3483,7 +3458,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery * pQuery = pRuntimeEnv->pQuery; SQueryStatus qStatus = {0}; - if (!needSupplementaryScan(pQuery)) { + if (!needReverseScan(pQuery)) { return; } @@ -3503,7 +3478,7 @@ static void doSingleMeterSupplementScan(SQueryRuntimeEnv *pRuntimeEnv) { doScanAllDataBlocks(pRuntimeEnv); queryStatusRestore(pRuntimeEnv, &qStatus); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } @@ -3562,7 +3537,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { return toContinue; } -void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { +void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); @@ -3588,10 +3563,7 @@ void vnodeScanAllData(SQueryRuntimeEnv *pRuntimeEnv) { break; } - /* - * set the correct start position, and load the corresponding block in buffer for next - * round scan all data blocks. - */ + // set the correct start position, and load the corresponding block in buffer for next round scan all data blocks. int32_t ret = tsdbDataBlockSeek(pRuntimeEnv->pQueryHandle, pos); status = pQuery->status; @@ -3675,18 +3647,13 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -STableQueryInfo *createMeterQueryInfo(SQInfo *pQInfo, int32_t sid, TSKEY skey, TSKEY ekey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - +STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, int32_t tid, STimeWindow win) { STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); - pTableQueryInfo->win = (STimeWindow){ - .skey = skey, - .ekey = ekey, - }; - pTableQueryInfo->lastKey = skey; + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; - pTableQueryInfo->sid = sid; + pTableQueryInfo->tid = tid; pTableQueryInfo->cur.vnodeIndex = -1; initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, 100, 100, TSDB_DATA_TYPE_INT); @@ -3702,7 +3669,7 @@ void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) free(pTableQueryInfo); } -void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo, TSKEY skey, TSKEY ekey) { +void changeMeterQueryInfoForSuppleQuery(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; } @@ -4014,9 +3981,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock_(SQInfo *pQInfo, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; @@ -4363,40 +4329,32 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { } } -static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { +static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - - // dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); + + int64_t st = taosGetTimestampMs(); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + tsdb_query_handle_t *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(pQInfo)) { break; } - // prepare the STableDataInfo struct for each table - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - // SMeterObj * pMeterObj = getMeterObj(pSupporter->pMetersHashTable, blockInfo.sid); - - // pQInfo->pObj = pMeterObj; - // pRuntimeEnv->pMeterObj = pMeterObj; - - STableDataInfo *pTableDataInfo = NULL; - // for (int32_t i = 0; i < pSupporter->pSidSet->numOfTables; ++i) { - // if (pMeterDataInfo[i].pMeterObj == pMeterObj) { - // pTableDataInfo = &pMeterDataInfo[i]; - // break; - // } - // } - - assert(pTableDataInfo != NULL); - STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; - - if (pTableDataInfo->pTableQInfo == NULL) { - // pTableDataInfo->pTableQInfo = createMeterQueryInfo(pQInfo, pMeterObj->sid, pQuery->skey, pQuery->ekey); + STableDataInfo* pTableDataInfo = NULL; + + // todo opt performance + for(int32_t i = 0; i < numOfTables; ++i) { + if (pQInfo->pTableDataInfo[i].pTableQInfo->tid == blockInfo.sid) { + pTableDataInfo = &pQInfo->pTableDataInfo[i]; + } } + assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); + STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; + restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); SDataStatis *pStatis = NULL; @@ -4408,14 +4366,18 @@ static void queryOnDataBlocks(SQInfo *pQInfo, STableDataInfo *pMeterDataInfo) { } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, pTableDataInfo->tableIndex, pTableQueryInfo); + if (ret != TSDB_CODE_SUCCESS) { - // pQInfo->killed = 1; - return; + pQInfo->code = ret; + return taosGetTimestampMs() - st; } } - // stableApplyFunctionsOnBlock_(pSupporter, pTableDataInfo, &blockInfo, pStatis, pDataBlock, searchFn); + stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } + + int64_t et = taosGetTimestampMs(); + return et - st; } static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index, @@ -4499,7 +4461,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start pointInterpSupporterSetData(pQInfo, &pointInterpSupporter); pointInterpSupporterDestroy(&pointInterpSupporter); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); // first/last_row query, do not invoke the finalize for super table query doFinalizeResult(pRuntimeEnv); @@ -4679,7 +4641,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { } } - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); pQuery->size = getNumOfResult(pRuntimeEnv); doSkipResults(pRuntimeEnv); @@ -4778,95 +4740,85 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { #endif } -static void doOrderedScan(SQInfo *pQInfo) { - SQuery *pQuery = &pQInfo->runtimeEnv.pQuery; -#if 0 -// if (pQInfo->runtimeEnv. == NULL) { -// pSupporter->pMeterDataInfo = calloc(pSupporter->pSidSet->numOfTables, sizeof(STableDataInfo)); -// } +static void createTableDataInfo(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - STableIdInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; + // todo make sure the table are added the reference count to gauranteed that all involved tables are valid + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - tSidSet* pSidset = pSupporter->pSidSet; - int32_t groupId = 0; - - for (int32_t i = 0; i < pSidset->numOfTables; ++i) { // load all meter meta info - SMeterObj *pMeterObj = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[i]->sid); - if (pMeterObj == NULL) { - dError("QInfo:%p failed to find required sid:%d", pQInfo, pMeterSidExtInfo[i]->sid); - continue; + if (pQInfo->pTableDataInfo == NULL) { + pQInfo->pTableDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * numOfTables); + if (pQInfo->pTableDataInfo == NULL) { + dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + return; } - if (i >= pSidset->starterPos[groupId + 1]) { - groupId += 1; + int32_t groupId = 0; + for (int32_t i = 0; i < numOfTables; ++i) { // load all meter meta info + STableId *id = taosArrayGet(pQInfo->pTableIdList, i); + STableDataInfo *pInfo = &pQInfo->pTableDataInfo[i]; + + setTableDataInfo(pInfo, i, groupId); + pInfo->pTableQInfo = createTableQueryInfo(&pQInfo->runtimeEnv, id->tid, pQuery->window); } - - STableDataInfo *pOneMeterDataInfo = &pSupporter->pMeterDataInfo[i]; - assert(pOneMeterDataInfo->pMeterObj == NULL); - - setMeterDataInfo(pOneMeterDataInfo, pMeterObj, i, groupId); - pOneMeterDataInfo->pTableQInfo = createMeterQueryInfo(pSupporter, pMeterObj->sid, pQuery->skey, pQuery->ekey); } - - queryOnDataBlocks(pQInfo, pSupporter->pMeterDataInfo); - if (pQInfo->code != TSDB_CODE_SUCCESS) { - return; - } -#endif } -static void setupMeterQueryInfoForSupplementQuery(SQInfo *pQInfo) { +static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - int32_t num = taosHashGetSize(pQInfo->pTableIdList); - for (int32_t i = 0; i < num; ++i) { - // STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey); + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); } } -static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - if (!needSupplementaryScan(pQuery)) { - dTrace("QInfo:%p no need to do supplementary scan, query completed", pQInfo); - return; - } - +static void doSaveContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - // disableFunctForSuppleScan(pSupporter, pQuery->order.order); - + disableFuncForReverseScan(pQInfo, pQuery->order.order); + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; } + + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + prepareQueryInfoForReverseScan(pQInfo); +} -#if 0 - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - setupMeterQueryInfoForSupplementQuery(pSupporter); +static void doRestoreContext(SQInfo* pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - int64_t st = taosGetTimestampMs(); - - doOrderedScan(pQInfo); - - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); - - /* - * restore the env - * the meter query info is not reset to the original state - */ - SWAP(pSupporter->rawSKey, pSupporter->rawEKey, TSKEY); - enableFunctForMasterScan(pRuntimeEnv, pQuery->order.order); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } -#endif + + enableFuncForForwardScan(pRuntimeEnv, pQuery->order.order); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { +static void doCloseAllTimeWindowAfterScan(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + size_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + + if (isIntervalQuery(pQuery)) { + for (int32_t i = 0; i < numOfTables; ++i) { + STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + } + } else { // close results for group result + closeAllTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + } +} + +static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -4875,7 +4827,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { * if the subgroupIdx > 0, the query process must be completed yet, we only need to * copy the data into output buffer */ - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW @@ -4894,46 +4846,49 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p current:%lldd, total:%lldd", pQInfo, pQuery->rec.size, pQuery->rec.total); return; } -#if 0 - pSupporter->pMeterDataInfo = (STableDataInfo *)calloc(1, sizeof(STableDataInfo) * pSupporter->numOfMeters); - if (pSupporter->pMeterDataInfo == NULL) { - dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); - pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; - return; - } - dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey, - pSupporter->rawEKey, pQuery->order.order, pSupporter->pSidSet->numOfSubSet); + dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); - dTrace("QInfo:%p main query scan start", pQInfo); - int64_t st = taosGetTimestampMs(); - doOrderedScan(pQInfo); + // create the query support structures + createTableDataInfo(pQInfo); - int64_t et = taosGetTimestampMs(); - dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, + // do check all qualified data blocks + int64_t el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, pQuery->order.order ^ 1u); - if (pQuery->intervalTime > 0) { - for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { - STableQueryInfo *pTableQueryInfo = pSupporter->pMeterDataInfo[i].pTableQInfo; - closeAllTimeWindow(&pTableQueryInfo->windowResInfo); - } - } else { // close results for group result - closeAllTimeWindow(&pRuntimeEnv->windowResInfo); - } - - doMultiMeterSupplementaryScan(pQInfo); - - if (isQueryKilled(pQInfo)) { - dTrace("QInfo:%p query killed, abort", pQInfo); + // query error occurred or query is killed, abort current execution + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - if (pQuery->intervalTime > 0 || isSumAvgRateQuery(pQuery)) { - assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); + // close all time window results + doCloseAllTimeWindowAfterScan(pQInfo); + + if (needReverseScan(pQuery)) { + doSaveContext(pQInfo); - if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) { - copyResToQueryResultBuf(pSupporter, pQuery); + el = queryOnDataBlocks(pQInfo); + dTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); + + doRestoreContext(pQInfo); + } else { + dTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); + return; + } + + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { + dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); + return; + } + + if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { +// assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); + + if (mergeResultsToGroup(pQInfo) == TSDB_CODE_SUCCESS) { + copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); @@ -4944,10 +4899,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { } // handle the limitation of output buffer - pQInfo->size += pQuery->size; - dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->size, pQInfo->size, - pQInfo->pointsReturned); -#endif + dTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); } /* @@ -4960,7 +4912,7 @@ static void tableFixedOutputProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { @@ -4993,7 +4945,7 @@ static void tableMultiOutputProcessor(SQInfo *pQInfo) { } while (1) { - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); doFinalizeResult(pRuntimeEnv); if (isQueryKilled(pQInfo)) { @@ -5039,7 +4991,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { initCtxOutputBuf(pRuntimeEnv); - vnodeScanAllData(pRuntimeEnv); + scanAllDataBlocks(pRuntimeEnv); if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; @@ -5076,7 +5028,7 @@ static void tableIntervalProcessor(SQInfo *pQInfo) { while (1) { tableIntervalProcessImpl(pRuntimeEnv); - if (pQuery->intervalTime > 0) { + if (isIntervalQuery(pQuery)) { pQInfo->subgroupIdx = 0; // always start from 0 pQuery->rec.size = 0; copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -5213,15 +5165,15 @@ static void singleTableQueryImpl(SQInfo* pQInfo) { sem_post(&pQInfo->dataReady); } -void multiTableQueryImpl(SQInfo* pQInfo) { +static void multiTableQueryImpl(SQInfo* pQInfo) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; pQuery->rec.size = 0; int64_t st = taosGetTimestampUs(); - if (pQuery->intervalTime > 0 || + if (isIntervalQuery(pQuery) || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { - vnodeMultiMeterQueryProcessor(pQInfo); + multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)); @@ -5242,29 +5194,6 @@ void multiTableQueryImpl(SQInfo* pQInfo) { sem_post(&pQInfo->dataReady); } -void qTableQuery(SQInfo *pQInfo) { - if (pQInfo == NULL || pQInfo->signature != pQInfo) { - dTrace("%p freed abort query", pQInfo); - return; - } - - if (isQueryKilled(pQInfo)) { - dTrace("QInfo:%p it is already killed, abort", pQInfo); - return; - } - - dTrace("QInfo:%p query task is launched", pQInfo); - - int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); - if (numOfTables == 1) { - singleTableQueryImpl(pQInfo); - } else { - multiTableQueryImpl(pQInfo); - } - - // vnodeDecRefCount(pQInfo); -} - static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { int32_t j = 0; @@ -5284,7 +5213,7 @@ bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pEx return j < pQueryMsg->numOfCols; } -static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryMsg) { +static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { if (pQueryMsg->intervalTime < 0) { dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime); return -1; @@ -5349,7 +5278,7 @@ static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - wchar_t** tagCond, char** nameCond) { + wchar_t** tagCond) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); @@ -5374,7 +5303,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); // query msg safety check - if (validateQueryMeterMsg(pQueryMsg) != 0) { + if (validateQueryMsg(pQueryMsg) != 0) { return TSDB_CODE_INVALID_QUERY_MSG; } @@ -5503,10 +5432,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE); } - - if (pQueryMsg->nameCondLen > 0) { - *nameCond = strndup(pMsg, pQueryMsg->nameCondLen); - } dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 @@ -5919,7 +5844,7 @@ _clean_memory: return NULL; } -bool isQInfoValid(void *param) { +static bool isValidQInfo(void *param) { SQInfo *pQInfo = (SQInfo *)param; if (pQInfo == NULL) { return false; @@ -5933,68 +5858,7 @@ bool isQInfoValid(void *param) { return (sig == (uint64_t)pQInfo); } -void vnodeFreeQInfo(SQInfo *pQInfo) { - if (!isQInfoValid(pQInfo)) { - return; - } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - setQueryKilled(pQInfo); - - dTrace("QInfo:%p start to free SQInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - tfree(pQuery->sdata[col]); - } - - // for (int col = 0; col < pQuery->numOfCols; ++col) { - // vnodeFreeColumnInfo(&pQuery->colList[col].data); - // } - // - // if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - // tfree(pQuery->tsData); - // } - - sem_destroy(&(pQInfo->dataReady)); - vnodeQueryFreeQInfoEx(pQInfo); - - for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { - SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; - if (pColFilter->numOfFilters > 0) { - tfree(pColFilter->pFilters); - } - } - - tfree(pQuery->pFilterInfo); - tfree(pQuery->colList); - tfree(pQuery->sdata); - - if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; - - if (pBinExprInfo->numOfCols > 0) { - tfree(pBinExprInfo->pReqColumns); - tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr, NULL); - } - } - - tfree(pQuery->pSelectExpr); - } - - if (pQuery->defaultVal != NULL) { - tfree(pQuery->defaultVal); - } - - tfree(pQuery->pGroupbyExpr); - tfree(pQuery); - - dTrace("QInfo:%p QInfo is freed", pQInfo); - - // destroy signature, in order to avoid the query process pass the object safety check - memset(pQInfo, 0, sizeof(SQInfo)); - tfree(pQInfo); -} - +static void freeQInfo(SQInfo *pQInfo); static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, SArray *pTableIdList, void* tsdb, SQInfo **pQInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -6045,10 +5909,128 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE _error: // table query ref will be decrease during error handling - vnodeFreeQInfo(*pQInfo); + freeQInfo(*pQInfo); return code; } +static void freeQInfo(SQInfo *pQInfo) { + if (!isValidQInfo(pQInfo)) { + return; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + setQueryKilled(pQInfo); + + dTrace("QInfo:%p start to free SQInfo", pQInfo); + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { + tfree(pQuery->sdata[col]); + } + + // for (int col = 0; col < pQuery->numOfCols; ++col) { + // vnodeFreeColumnInfo(&pQuery->colList[col].data); + // } + // + // if (pQuery->colList[0].colIdx != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + // tfree(pQuery->tsData); + // } + + sem_destroy(&(pQInfo->dataReady)); + vnodeQueryFreeQInfoEx(pQInfo); + + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; + if (pColFilter->numOfFilters > 0) { + tfree(pColFilter->pFilters); + } + } + + tfree(pQuery->pFilterInfo); + tfree(pQuery->colList); + tfree(pQuery->sdata); + + if (pQuery->pSelectExpr != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; + + if (pBinExprInfo->numOfCols > 0) { + tfree(pBinExprInfo->pReqColumns); + tSQLBinaryExprDestroy(&pBinExprInfo->pBinExpr, NULL); + } + } + + tfree(pQuery->pSelectExpr); + } + + if (pQuery->defaultVal != NULL) { + tfree(pQuery->defaultVal); + } + + tfree(pQuery->pGroupbyExpr); + tfree(pQuery); + + dTrace("QInfo:%p QInfo is freed", pQInfo); + + // destroy signature, in order to avoid the query process pass the object safety check + memset(pQInfo, 0, sizeof(SQInfo)); + tfree(pQInfo); +} + +static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + /* + * get the file size and set the numOfRows to be the file size, since for tsComp query, + * the returned row size is equalled to 1 + * TODO handle the case that the file is too large to send back one time + */ + if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { + struct stat fstat; + if (stat(pQuery->sdata[0]->data, &fstat) == 0) { + *numOfRows = fstat.st_size; + return fstat.st_size; + } else { + dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); + return 0; + } + } else { + return pQuery->rowSize * (*numOfRows); + } +} + +static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { + // the remained number of retrieved rows, not the interpolated result + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // load data from file to msg buffer + if (isTSCompQuery(pQuery)) { + int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); + + // make sure file exist + if (FD_VALID(fd)) { + size_t s = lseek(fd, 0, SEEK_END); + dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); + + lseek(fd, 0, SEEK_SET); + read(fd, data, s); + close(fd); + + unlink(pQuery->sdata[0]->data); + } else { + dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, + pQuery->sdata[0]->data, strerror(errno)); + } + } else { + doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); + } + + pQuery->rec.total += pQuery->rec.size; + dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); + + return TSDB_CODE_SUCCESS; + + // todo if interpolation exists, the result may be dump to client by several rounds +} + int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) { assert(pQueryMsg != NULL); @@ -6057,9 +6039,7 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) SArray *pTableIdList = NULL; SSqlFuncExprMsg** pExprMsg = NULL; wchar_t* tagCond = NULL; - char* nameCond = NULL; - - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &nameCond)) != TSDB_CODE_SUCCESS) { + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { return code; } @@ -6126,8 +6106,31 @@ _query_over: return TSDB_CODE_SUCCESS; } +void qTableQuery(SQInfo *pQInfo) { + if (pQInfo == NULL || pQInfo->signature != pQInfo) { + dTrace("%p freed abort query", pQInfo); + return; + } + + if (isQueryKilled(pQInfo)) { + dTrace("QInfo:%p it is already killed, abort", pQInfo); + return; + } + + dTrace("QInfo:%p query task is launched", pQInfo); + + int32_t numOfTables = taosArrayGetSize(pQInfo->pTableIdList); + if (numOfTables == 1) { + singleTableQueryImpl(pQInfo); + } else { + multiTableQueryImpl(pQInfo); + } + + // vnodeDecRefCount(pQInfo); +} + int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { - if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } @@ -6148,62 +6151,6 @@ int32_t qRetrieveQueryResultInfo(SQInfo *pQInfo) { return (pQInfo->code >= 0)? pQInfo->code:(-pQInfo->code); } -static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - - /* - * get the file size and set the numOfRows to be the file size, since for tsComp query, - * the returned row size is equalled to 1 - * TODO handle the case that the file is too large to send back one time - */ - if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fstat; - if (stat(pQuery->sdata[0]->data, &fstat) == 0) { - *numOfRows = fstat.st_size; - return fstat.st_size; - } else { - dError("QInfo:%p failed to get file info, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); - return 0; - } - } else { - return pQuery->rowSize * (*numOfRows); - } -} - -static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { - // the remained number of retrieved rows, not the interpolated result - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - - // make sure file exist - if (FD_VALID(fd)) { - size_t s = lseek(fd, 0, SEEK_END); - dTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - - lseek(fd, 0, SEEK_SET); - read(fd, data, s); - close(fd); - - unlink(pQuery->sdata[0]->data); - } else { - dError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, - pQuery->sdata[0]->data, strerror(errno)); - } - } else { - doCopyQueryResultToMsg(pQInfo, pQuery->rec.size, data); - } - - pQuery->rec.total += pQuery->rec.size; - dTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.size, pQuery->rec.total); - - return TSDB_CODE_SUCCESS; - - // todo if interpolation exists, the result may be dump to client by several rounds -} - bool qHasMoreResultsToRetrieve(SQInfo* pQInfo) { if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { return false; @@ -6222,7 +6169,7 @@ bool qHasMoreResultsToRetrieve(SQInfo* pQInfo) { } int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { - if (pQInfo == NULL || !isQInfoValid(pQInfo)) { + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } @@ -6251,7 +6198,7 @@ int32_t qDumpRetrieveResult(SQInfo *pQInfo, SRetrieveTableRsp** pRsp, int32_t* c if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { (*pRsp)->completed = 1; // notify no more result to client - vnodeFreeQInfo(pQInfo); + freeQInfo(pQInfo); } return code; diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 761006a009..1dd1c3b29d 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -349,10 +349,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { } else { // add non-super table to the array pMeta->tables[pTable->tableId.tid] = pTable; - if (pTable->type == TSDB_CHILD_TABLE) { - // add STABLE to the index + if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index tsdbAddTableIntoIndex(pMeta, pTable); } + pMeta->nTables++; } From 016b5e8550a845b8171f149b8f04e169e55ac43d Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 28 Mar 2020 14:19:48 +0800 Subject: [PATCH 64/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 198 +++++++++------------------------- 1 file changed, 53 insertions(+), 145 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 599af73813..a3367d2e51 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -892,13 +892,15 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters // Check if there are data to commit to this file int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); - if (!hasDataToCommit) return 0; // No data to commit, just return + if (!hasDataToCommit) return 0; // No data to commit, just return // Create and open files for commit tsdbGetDataDirName(pRepo, dataDir); - if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) {/* TODO */} + if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { /* TODO */ + } pGroup = tsdbOpenFilesForCommit(pFileH, fid); - if (pGroup == NULL) {/* TODO */} + if (pGroup == NULL) { /* TODO */ + } tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 0); if (1 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { // TODO: make it not to write the last file every time @@ -907,28 +909,32 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters } // Load the SCompIdx - pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) *pCfg->maxTables); - if (pIndices == NULL) {/* TODO*/} - if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) {/* TODO */} + pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); + if (pIndices == NULL) { /* TODO*/ + } + if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { /* TODO */ + } -// Loop to commit data in each table + // Loop to commit data in each table for (int tid = 0; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; + STable * pTable = pMeta->tables[tid]; SSkipListIterator *pIter = iters[tid]; - SCompIdx *pIdx = &pIndices[tid]; + SCompIdx * pIdx = &pIndices[tid]; if (!tsdbHasDataInRange(pIter, minKey, maxKey)) { // This table does not have data in this range, just copy its head part and last // part (if neccessary) to new file - if (pIdx->len != 0) { // has old data + if (pIdx->offset > 0) { // has old data if (isNewLastFile && pIdx->hasLast) { // Need to load SCompBlock part and copy to new file - if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) {/* TODO */} - if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) {/* TODO */} + if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */ + } + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ + } - // Copy the last block from old last file to new file + // TODO: Copy the last block from old last file to new file // tsdbCopyBlockData() - } else { + } else { pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); hFile.size += pIdx->len; @@ -937,140 +943,42 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters continue; } - // while () { + // Load SCompBlock part if neccessary + int isCompBlockLoaded = 0; + if (pIdx->offset > 0) { + if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) { + // has last block || cache key overlap with commit key + if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ + } + if (pCompInfo->uid == pTable->tableId.uid) isCompBlockLoaded = 1; + } else { + // TODO: No need to load the SCompBlock part, just sendfile the SCompBlock part + // and write those new blocks to it + } + } - // } + tdInitDataCols(pCols, pTable->schema); + + int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; + while (1) { + tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols); + if (pCols->numOfPoints == 0) break; + + // TODO + int pointsWritten = 0; /* tsdbWriteBlockToFile(); */ + tdPopDataColsPoints(pCols, pointsWritten); + } + + // Write the SCompBlock part + if (isCompBlockLoaded) { + // merge the block into old and update pIdx + } else { + // sendfile the SCompBlock part and update the pIdx + } } - // for (int tid = 0; tid < pCfg->maxTables; tid++) { - // STable * pTable = pMeta->tables[tid]; - // SSkipListIterator *pIter = iters[tid]; - // int isLoadCompBlocks = 0; - // char dataDir[128] = "\0"; - - // if (pIter == NULL) { - // if (hasDataToCommit && isNewLastFile()) - // continue; - // } - // tdInitDataCols(pCols, pTable->schema); - - // int numOfWrites = 0; - // int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; // We keep 20% of space for merge purpose - // // Loop to read columns from cache - // while (tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols)) { - // if (!hasDataToCommit) { - // // There are data to commit to this fileId, we need to create/open it for read/write. - // // At the meantime, we set the flag to prevent further create/open operations - // tsdbGetDataDirName(pRepo, dataDir); - - // if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { - // // TODO: deal with the ERROR here - // } - // // Open files for commit - // pGroup = tsdbOpenFilesForCommit(pFileH, fid); - // if (pGroup == NULL) { - // // TODO: deal with the ERROR here - // } - // // TODO: open .h file and if neccessary, open .l file - // tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &tFile, 1, 0); - // if (1 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { - // // TODO: make it not to write the last file every time - // tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); - // isNewLastFile = 1; - // } - - // // load the SCompIdx part - // pIndices = (SCompIdx *)malloc(sizeof(SCompIdx) * pCfg->maxTables); - // if (pIndices == NULL) { // TODO: deal with the ERROR - // } - // if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { // TODO: deal with the ERROR here - // } - - // // sendfile those not need to changed table content - // lseek(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, - // SEEK_SET); - // lseek(tFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); - // for (int ttid = 0; ttid < tid; ttid++) { - // SCompIdx * tIdx= &pIndices[ttid]; - // if (tIdx->len <= 0) continue; - // if (isNewLastFile && tIdx->hasLast) { - // // TODO: Need to load the SCompBlock part and copy to new last file - // pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, tIdx->len); - // if (pCompInfo == NULL) { /* TODO */} - // if (tsdbLoadCompBlocks(pGroup, tIdx, (void *)pCompInfo) < 0) {/* TODO */} - // SCompBlock *pLastBlock = TSDB_COMPBLOCK_AT(pCompInfo, tIdx->numOfSuperBlocks - 1); - // int numOfSubBlocks = pLastBlock->numOfSubBlocks; - // assert(pLastBlock->last); - // if (tsdbCopyCompBlockToFile(&pGroup->files[TSDB_FILE_TYPE_LAST], &lFile, pCompInfo, tIdx->numOfSuperBlocks, 1 /* isOutLast*/) < 0) {/* TODO */} - // { - // if (numOfSubBlocks > 1) tIdx->len -= (sizeof(SCompBlock) * numOfSubBlocks); - // tIdx->checksum = 0; - // } - // write(tFile.fd, (void *)pCompInfo, tIdx->len); - // tFile.size += tIdx->len; - // } else { - // sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, tFile.fd, NULL, tIdx->len); - // tFile.size += tIdx->len; - // } - // } - - // hasDataToCommit = 1; - // } - - // SCompIdx *pIdx = &pIndices[tid]; - - // /* The first time to write to the table, need to decide - // * if it is neccessary to load the SComplock part. If it - // * is needed, just load it, or, just use sendfile and - // * append it. - // */ - // if (numOfWrites == 0 && pIdx->offset > 0) { - // if (dataColsKeyFirst(pCols) <= pIdx->maxKey || pIdx->hasLast) { - // pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len); - // if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { - // // TODO: deal with the ERROR here - // } - // if (pCompInfo->uid == pTable->tableId.uid) isLoadCompBlocks = 1; - // } else { - // // TODO: sendfile the prefix part - // } - // } - - // int numOfPointsWritten = tsdbWriteBlockToFile(pGroup, pCompInfo, pIdx, isLoadCompBlocks, pBlock, pCols); - // if (numOfPointsWritten < 0) { - // // TODO: deal with the ERROR here - // } - - // // pCompInfo = tsdbMergeBlock(pCompInfo, pBlock); - - - // // if (1 /* the SCompBlock part is not loaded*/) { - // // // Append to .data file generate a SCompBlock and record it - // // } else { - // // } - - // // // TODO: need to reset the pCols - - // numOfWrites++; - // } - - // if (pCols->numOfPoints > 0) { - // // TODO: still has data to commit, commit it - // } - - // if (1/* SCompBlock part is loaded, write it to .head file*/) { - // // TODO - // } else { - // // TODO: use sendfile send the old part and append the newly added part - // } - // } - - // Write the SCompIdx part - - // Close all files and return - if (hasDataToCommit) { - // TODO - } + // TODO: close the files and replace the .head and .last file + {} if (pIndices) free(pIndices); if (pCompInfo) free(pCompInfo); From d37876fe31dd03c0745a451d73a7c0d022ff52aa Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 28 Mar 2020 15:03:50 +0800 Subject: [PATCH 65/85] TD-34 --- src/vnode/tsdb/inc/tsdbFile.h | 11 +++++++---- src/vnode/tsdb/src/tsdbFile.c | 18 ++++++++---------- src/vnode/tsdb/src/tsdbMain.c | 21 ++++++++++++++++++--- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index e4592ffa2e..ac96d15126 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -42,13 +42,16 @@ typedef enum { extern const char *tsdbFileSuffix[]; typedef struct { - int8_t type; - int fd; - char fname[128]; int64_t size; // total size of the file int64_t tombSize; // unused file size int32_t totalBlocks; int32_t totalSubBlocks; +} SFileInfo; + +typedef struct { + int fd; + char fname[128]; + SFileInfo info; } SFile; #define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1) @@ -74,7 +77,7 @@ void tsdbCloseFileH(STsdbFileH *pFileH); int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose); int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); int tsdbOpenFile(SFile *pFile, int oflag); -SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); +int tsdbCloseFile(SFile *pFile); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); typedef struct { diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index c1b7ae2382..3567517d79 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -268,7 +268,7 @@ static int compFGroup(const void *arg1, const void *arg2) { static int tsdbWriteFileHead(SFile *pFile) { char head[TSDB_FILE_HEAD_SIZE] = "\0"; - pFile->size += TSDB_FILE_HEAD_SIZE; + pFile->info.size += TSDB_FILE_HEAD_SIZE; // TODO: write version and File statistic to the head lseek(pFile->fd, 0, SEEK_SET); @@ -292,7 +292,7 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) { return -1; } - pFile->size += size; + pFile->info.size += size; free(buf); return 0; @@ -315,6 +315,12 @@ int tsdbOpenFile(SFile *pFile, int oflag) { // TODO: change the function return 0; } +int tsdbCloseFile(SFile *pFile) { + int ret = close(pFile->fd); + pFile->fd = -1; + return ret; +} + SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); if (pGroup == NULL) return NULL; @@ -325,14 +331,6 @@ SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { return pGroup; } -static int tsdbCloseFile(SFile *pFile) { - if (!TSDB_IS_FILE_OPENED(pFile)) return -1; - int ret = close(pFile->fd); - pFile->fd = -1; - - return ret; -} - int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) { memset((void *)pFile, 0, sizeof(SFile)); pFile->fd = -1; diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index a3367d2e51..2a6b60d24d 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -937,7 +937,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters } else { pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); - hFile.size += pIdx->len; + hFile.info.size += pIdx->len; } } continue; @@ -977,8 +977,23 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters } } - // TODO: close the files and replace the .head and .last file - {} + // Write the SCompIdx part + if (lseek(hFile.fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {/* TODO */} + if (write(hFile.fd, (void *)pIndices, sizeof(SCompIdx) * pCfg->maxTables) < 0) {/* TODO */} + + // close the files + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbCloseFile(&pGroup->files[type]); + } + tsdbCloseFile(&hFile); + if (isNewLastFile) tsdbCloseFile(&lFile); + // TODO: replace the .head and .last file + rename(hFile.fname, pGroup->files[TSDB_FILE_TYPE_HEAD].fname); + pGroup->files[TSDB_FILE_TYPE_HEAD].info = hFile.info; + if (isNewLastFile) { + rename(lFile.fname, pGroup->files[TSDB_FILE_TYPE_LAST].fname); + pGroup->files[TSDB_FILE_TYPE_LAST].info = lFile.info; + } if (pIndices) free(pIndices); if (pCompInfo) free(pCompInfo); From 5470057dbcf2760c93c8239f4899fa1016745f6a Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 28 Mar 2020 16:24:59 +0800 Subject: [PATCH 66/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 2a6b60d24d..42792bc1b8 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -332,7 +332,10 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { pRepo->tsdbCache->curBlock = NULL; // TODO: here should set as detached or use join for memory leak - pthread_create(&(pRepo->commitThread), NULL, tsdbCommitData, (void *)repo); + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); + pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo); tsdbUnLockRepo(repo); return 0; From 2b14e5524f6a97231588662fbaeb635778556bfb Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 28 Mar 2020 18:07:56 +0800 Subject: [PATCH 67/85] TD-34 --- src/common/src/dataformat.c | 11 +++++++ src/vnode/tsdb/inc/tsdbFile.h | 2 -- src/vnode/tsdb/src/tsdbFile.c | 59 ----------------------------------- src/vnode/tsdb/src/tsdbMain.c | 34 ++++++++++++++++++-- 4 files changed, 43 insertions(+), 63 deletions(-) diff --git a/src/common/src/dataformat.c b/src/common/src/dataformat.c index 56565b6e5c..0496fc6feb 100644 --- a/src/common/src/dataformat.c +++ b/src/common/src/dataformat.c @@ -355,7 +355,18 @@ void tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols) { } // Pop pointsToPop points from the SDataCols void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) { + int pointsLeft = pCols->numOfPoints - pointsToPop; + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SDataCol *p_col = pCols->cols + iCol; + if (p_col->len > 0) { + p_col->len = TYPE_BYTES[p_col->type] * pointsLeft; + if (pointsLeft > 0) { + memmove((void *)(p_col->pData), (void *)((char *)(p_col->pData) + TYPE_BYTES[p_col->type] * pointsToPop), p_col->len); + } + } + } + pCols->numOfPoints = pointsLeft; } /** diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index ac96d15126..0f654ad0bd 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -148,8 +148,6 @@ int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SD // TODO: need an API to merge all sub-block data into one -int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols); - void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); #ifdef __cplusplus } diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 3567517d79..088a164933 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -196,65 +196,6 @@ int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void return 0; } -static int tsdbWriteBlockToFileImpl(SFile * pFile, // File to write - SDataCols * pCols, // Data column buffer - int numOfPointsToWrie, // Number of points to write to the file - SCompBlock *pBlock // SCompBlock to hold block information to return - ) { - // pBlock->last = 0; - // pBlock->offset = lseek(pFile->fd, 0, SEEK_END); - // // pBlock->algorithm = ; - // pBlock->numOfPoints = pCols->numOfPoints; - // // pBlock->sversion = ; - // // pBlock->len = ; - // pBlock->numOfSubBlocks = 1; - // pBlock->keyFirst = dataColsKeyFirst(pCols); - // pBlock->keyLast = dataColsKeyLast(pCols); - // for (int i = 0; i < pCols->numOfCols; i++) { - // // TODO: if all col value is NULL, do not save it - // pBlock->numOfCols++; - // pCompData->numOfCols++; - // SCompCol *pCompCol = pCompData->cols + i; - // pCompCol->colId = pCols->cols[i].colId; - // pCompCol->type = pCols->cols[i].type; - - // // pCompCol->len = ; - // // pCompCol->offset = ; - // } - - return 0; -} - -int tsdbWriteBlockToFile(SFileGroup *pGroup, SCompInfo *pCompInfo, SCompIdx *pIdx, int isMerge, SCompBlock *pBlock, SDataCols *pCols) { - memset((void *)pBlock, 0, sizeof(SCompBlock)); - SFile *pFile = NULL; - SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols); - if (pCompData == NULL) return -1; - pCompData->delimiter = TSDB_FILE_DELIMITER; - // pCompData->uid = ; - - if (isMerge) { - TSKEY keyFirst = dataColsKeyFirst(pCols); - // 1. Binary search the block the data can merged into - - if (1/* the data should only merged into last file */) { - } else { - } - } else { - // Write directly to the file without merge - if (1/*pCols->numOfPoints < pCfg->minRowsPerFileBlock*/) { - // TODO: write the data to the last file - } else { - // TODO: wirte the data to the data file - } - } - - // TODO: need to update pIdx - - if (pCompData) free(pCompData); - return 0; -} - static int compFGroupKey(const void *key, const void *fgroup) { int fid = *(int *)key; SFileGroup *pFGroup = (SFileGroup *)fgroup; diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 42792bc1b8..5f7dd2513f 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -967,9 +967,26 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols); if (pCols->numOfPoints == 0) break; - // TODO - int pointsWritten = 0; /* tsdbWriteBlockToFile(); */ + int pointsWritten = 0; + // { // TODO : try to write the block data to file + // if (!isCompBlockLoaded) { // Just append + // if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // write directly to .data file + // lseek(pGroup->files[TSDB_FILE_TYPE_DATA], 0, SEEK_END); + + // } else { + // if (isNewLastFile) { // write directly to .l file + + // } else { // write directly to .last file + + // } + // } + // } else { // Need to append + // // SCompBlock *pTBlock = NULL; + // } + // } + pointsWritten = pCols->numOfPoints; tdPopDataColsPoints(pCols, pointsWritten); + maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints; } // Write the SCompBlock part @@ -1022,4 +1039,17 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK if (tsdbHasDataInRange(pIter, minKey, maxKey)) return 1; } return 0; +} + +static int tsdbWriteBlockToFile(STsdbRepo *pRepo, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock) { + STsdbCfg *pCfg = &(pRepo->config); + + memset((void *)pCompBlock, 0, sizeof(SCompBlock)); + + if (pCompInfo == NULL) { + // Just need to append to file + } else { + // Need to merge + } + return 0; } \ No newline at end of file From 12d2e463eb2e55980bd30712de8d18395c9418e3 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 28 Mar 2020 18:58:51 +0800 Subject: [PATCH 68/85] TD-34 --- src/common/inc/dataformat.h | 6 ++-- src/vnode/tsdb/inc/tsdbFile.h | 1 + src/vnode/tsdb/src/tsdbFile.c | 2 -- src/vnode/tsdb/src/tsdbMain.c | 59 ++++++++++++++++++++++++++++++++++- 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/common/inc/dataformat.h b/src/common/inc/dataformat.h index a7016061e4..4e8afd4f0e 100644 --- a/src/common/inc/dataformat.h +++ b/src/common/inc/dataformat.h @@ -119,13 +119,15 @@ typedef struct { int maxPoints; // max number of points int numOfPoints; int numOfCols; // Total number of cols + int sversion; // TODO: set sversion void * buf; SDataCol cols[]; } SDataCols; #define keyCol(pCols) (&((pCols)->cols[0])) // Key column -#define dataColsKeyFirst(pCols) ((int64_t *)(keyCol(pCols)->pData))[0] -#define dataColsKeyLast(pCols) ((int64_t *)(keyCol(pCols)->pData))[(pCols)->numOfPoints - 1] +#define dataColsKeyAt(pCols, idx) ((int64_t *)(keyCol(pCols)->pData))[(idx)] +#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0) +#define dataColsKeyLast(pCols) dataColsKeyAt(pCols, (pCols)->numOfPoints - 1) SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 0f654ad0bd..7e358cc0a2 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -26,6 +26,7 @@ extern "C" { #endif #define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F #define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) #define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 088a164933..9a0db96bd0 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -24,8 +24,6 @@ #include "tsdbFile.h" -#define TSDB_FILE_DELIMITER 0xF00AFA0F - const char *tsdbFileSuffix[] = { ".head", // TSDB_FILE_TYPE_HEAD ".data", // TSDB_FILE_TYPE_DATA diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 5f7dd2513f..d18e766c38 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -1041,13 +1041,70 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK return 0; } +static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, int64_t uid) { + size_t size = sizeof(SCompData) + sizeof(SCompCol) * pCols->numOfCols; + SCompData *pCompData = (SCompData *)malloc(size); + if (pCompData == NULL) return -1; + + pCompData->delimiter = TSDB_FILE_DELIMITER; + pCompData->uid = uid; + pCompData->numOfCols = pCols->numOfCols; + + *offset = lseek(pFile->fd, 0, SEEK_END); + *len = size; + + int toffset = size; + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SCompCol *pCompCol = pCompData->cols + iCol; + SDataCol *pDataCol = pCols->cols + iCol; + + pCompCol->colId = pDataCol->colId; + pCompCol->type = pDataCol->type; + pCompCol->offset = toffset; + + // TODO: add compression + pCompCol->len = TYPE_BYTES[pCompCol->type] * pointsToWrite; + toffset += pCompCol->len; + } + + // Write the block + if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err; + for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { + SDataCol *pDataCol = pCols->cols + iCol; + SCompCol *pCompCol = pCompData->cols + iCol; + if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err; + } + + if (pCompData == NULL) free((void *)pCompData); + return 0; + +_err: + if (pCompData == NULL) free((void *)pCompData); + return -1; +} + static int tsdbWriteBlockToFile(STsdbRepo *pRepo, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock) { STsdbCfg *pCfg = &(pRepo->config); + SCompData *pCompData = NULL; memset((void *)pCompBlock, 0, sizeof(SCompBlock)); if (pCompInfo == NULL) { - // Just need to append to file + if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file + // tsdbWriteBlockToFileImpl() + } else { // Write to .last or .l file + pCompBlock->last = 1; + + } + // pCompBlock->offset = ; + // pCompBlock->len = ; + pCompBlock->algorithm = 2; // TODO : add to configuration + pCompBlock->sversion = pCols->sversion; + pCompBlock->numOfPoints = pCols->numOfPoints; + pCompBlock->numOfSubBlocks = 1; + pCompBlock->numOfCols = pCols->numOfCols; + pCompBlock->keyFirst = dataColsKeyFirst(pCols); + pCompBlock->keyLast = dataColsKeyLast(pCols); } else { // Need to merge } From 8ff864021b0bc75ce223436c87ea949fac1f59b7 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 28 Mar 2020 22:23:38 +0800 Subject: [PATCH 69/85] [TD-32] fix bugs in super table query. --- src/client/src/tscSecondaryMerge.c | 9 +- src/client/src/tscServer.c | 22 ++-- src/client/src/tscSubquery.c | 188 +++++++++++++++-------------- src/client/src/tscUtil.c | 2 +- src/query/src/queryExecutor.c | 182 +++++++++++++++------------- src/vnode/tsdb/src/tsdbRead.c | 81 ++++++++++--- 6 files changed, 277 insertions(+), 207 deletions(-) diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 0ca079bc6e..1d4ed51f32 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -636,10 +636,11 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); -// for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) { -// (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); -// (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; -// } + size_t numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList); + for (int32_t i = 0; i < numOfSubs; ++i) { + (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); + (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; + } if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 498a338389..111ebc4e8c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -509,31 +509,31 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSubmitMsg *pShellMsg; - char * pMsg, *pStart; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - pStart = pSql->cmd.payload + tsRpcHeadSize; - pMsg = pStart; + char* pMsg = pSql->cmd.payload + tsRpcHeadSize; + + // NOTE: shell message size should not include SMsgDesc + int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; - pMsgDesc->numOfVnodes = htonl(1); //set the number of vnodes + + pMsgDesc->numOfVnodes = htonl(1); //todo set the right number of vnodes pMsg += sizeof(SMsgDesc); - pShellMsg = (SSubmitMsg *)pMsg; + SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; + pShellMsg->header.vgId = htonl(pTableMeta->vgId); - pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen); + pShellMsg->header.contLen = htonl(size); pShellMsg->length = pShellMsg->header.contLen; pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit); // number of meters to be inserted - // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here + // pSql->cmd.payloadLen is set during copying data into paylaod pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes)); -// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip), -// htons(pShellMsg->vnode)); return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index ccf187ac64..b0c7b68ab4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1006,14 +1006,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->qhandle = 1; // hack the qhandle check - const uint32_t nBufferSize = (1 << 16); // 64KB + const uint32_t nBufferSize = (1u << 16); // 64KB SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t numOfSubQueries = 1; -// int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes; - + int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList); assert(numOfSubQueries > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); @@ -1119,7 +1117,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tfree(trsupport); } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) { // set no disk space error info @@ -1141,7 +1139,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES pthread_mutex_unlock(&trsupport->queryMutex); - tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code); + tscRetrieveFromDnodeCallBack(trsupport, tres, trsupport->pState->code); } static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1236,7 +1234,86 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq } } -static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { +static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { + int32_t idx = trsupport->subqueryIndex; + SSqlObj * pPObj = trsupport->pParentSqlObj; + tOrderDescriptor *pDesc = trsupport->pOrderDescriptor; + + SSubqueryState* pState = trsupport->pState; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + // data in from current vnode is stored in cache and disk + uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; +// tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, +// pSvd->vnode, numOfRowsFromSubquery, idx); + + tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); + +#ifdef _DEBUG_VIEW + printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); + SSrcColumnInfo colInfo[256] = {0}; + tscGetSrcColumnInfo(colInfo, pQueryInfo); + tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, + trsupport->localBuffer->numOfElems, colInfo); +#endif + + if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { + tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, + tsAvailTmpDirGB, tsMinimalTmpDirGB); + tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + return; + } + + // each result for a vnode is ordered as an independant list, + // then used as an input of loser tree for disk-based merge routine + int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, + pQueryInfo->groupbyExpr.orderType); + if (ret != 0) { // set no disk space error info, and abort retry + return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE); + } + + // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion + // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. + // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. + int32_t numOfTotal = pState->numOfTotal; + + int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); + if (finished < numOfTotal) { + tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); + return tscFreeSubSqlObj(trsupport, pSql); + } + + // all sub-queries are returned, start to local merge process + pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; + + tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, + pState->numOfTotal, pState->numOfRetrievedRows); + + SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); + tscClearInterpInfo(pPQueryInfo); + + tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, + &pPObj->cmd, &pPObj->res); + tscTrace("%p build loser tree completed", pPObj); + + pPObj->res.precision = pSql->res.precision; + pPObj->res.numOfRows = 0; + pPObj->res.row = 0; + + // only free once + tfree(trsupport->pState); + tscFreeSubSqlObj(trsupport, pSql); + + // set the command flag must be after the semaphore been correctly set. + pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; + if (pPObj->res.code == TSDB_CODE_SUCCESS) { + (*pPObj->fp)(pPObj->param, pPObj, 0); + } else { + tscQueueAsyncRes(pPObj); + } +} + +static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { SRetrieveSupport *trsupport = (SRetrieveSupport *)param; int32_t idx = trsupport->subqueryIndex; SSqlObj * pPObj = trsupport->pParentSqlObj; @@ -1265,15 +1342,15 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx); - SVnodeSidList *vnodeInfo = 0; - SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; +// SVnodeSidList *vnodeInfo = 0; +// SVnodeDesc * pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index]; if (numOfRows > 0) { assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, - pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); +// tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql, +// pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64, @@ -1299,85 +1376,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data, pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); - if (ret < 0) { - // set no disk space error info, and abort retry + if (ret < 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } else { + } else if (pRes->completed) { + tscAllDataRetrievedFromDnode(trsupport, pSql); + } else { // continue fetch data from dnode pthread_mutex_unlock(&trsupport->queryMutex); - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); - } - - } else { // all data has been retrieved to client - /* data in from current vnode is stored in cache and disk */ - uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; - tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip, - pSvd->vnode, numOfRowsFromVnode, idx); - - tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); - -#ifdef _DEBUG_VIEW - printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems); - SSrcColumnInfo colInfo[256] = {0}; - tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems, - trsupport->localBuffer->numOfElems, colInfo); -#endif - - if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) { - tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql, - tsAvailTmpDirGB, tsMinimalTmpDirGB); - tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - return; - } - - // each result for a vnode is ordered as an independant list, - // then used as an input of loser tree for disk-based merge routine - int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, - pQueryInfo->groupbyExpr.orderType); - if (ret != 0) { - /* set no disk space error info, and abort retry */ - return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE); - } - - // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion - // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed. - // In this case, the comparsion between finished value and released pState->numOfTotal is not safe. - int32_t numOfTotal = pState->numOfTotal; - - int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1); - if (finished < numOfTotal) { - tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished); - return tscFreeSubSqlObj(trsupport, pSql); - } - - // all sub-queries are returned, start to local merge process - pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; - - tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj, - pState->numOfTotal, pState->numOfRetrievedRows); - - SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); - tscClearInterpInfo(pPQueryInfo); - - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, - &pPObj->cmd, &pPObj->res); - tscTrace("%p build loser tree completed", pPObj); - - pPObj->res.precision = pSql->res.precision; - pPObj->res.numOfRows = 0; - pPObj->res.row = 0; - - // only free once - tfree(trsupport->pState); - tscFreeSubSqlObj(trsupport, pSql); - - // set the command flag must be after the semaphore been correctly set. - pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC; - if (pPObj->res.code == TSDB_CODE_SUCCESS) { - (*pPObj->fp)(pPObj->param, pPObj, 0); - } else { - tscQueueAsyncRes(pPObj); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } + } else { // all data has been retrieved to client + tscAllDataRetrievedFromDnode(trsupport, pSql); } } @@ -1437,7 +1445,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { /* * if a query on a vnode is failed, all retrieve operations from vnode that occurs later - * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack + * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack * function to abort current and remain retrieve process. * * NOTE: threadsafe is required. @@ -1475,7 +1483,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex, pState->code); } - tscRetrieveFromVnodeCallBack(param, tres, pState->code); + tscRetrieveFromDnodeCallBack(param, tres, pState->code); } else { // success, proceed to retrieve data from dnode if (vnodeInfo != NULL) { tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql, @@ -1486,7 +1494,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { trsupport->subqueryIndex); } - taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param); + taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 18f491aa9e..ec6881db3f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -608,7 +608,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { * the payloadLen should be actual message body size * the old value of payloadLen is the allocated payload size */ - pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize - sizeof(SMsgDesc); + pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize; assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0); return TSDB_CODE_SUCCESS; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 224512f0ae..83cbae4266 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2204,7 +2204,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { size_t s = taosArrayGetSize(pQInfo->pTableIdList); num = MAX(s, INITIAL_RESULT_ROWS_VALUE); } else { // for super table query, one page for each subset - // num = pQInfo->pSidSet->numOfSubSet; + num = 1;//pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -3873,7 +3873,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (isIntervalQuery(pQuery))) { totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); } else { - // totalSubset = pQInfo->pSidSet->numOfSubSet; + totalSubset = 1;//pQInfo->pSidSet->numOfSubSet; } return totalSubset; @@ -3909,19 +3909,18 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; int32_t oldOffset = pQInfo->offset; - assert(0); /* * current output space is not enough to keep all the result data of this group, only copy partial results * to SQuery object's result buffer */ - // if (numOfRowsToCopy > pQuery->pointsToRead - numOfResult) { - // numOfRowsToCopy = pQuery->pointsToRead - numOfResult; - // pQInfo->offset += numOfRowsToCopy; - // } else { - // pQInfo->offset = 0; - // pQInfo->subgroupIdx += 1; - // } + if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { + numOfRowsToCopy = pQuery->rec.capacity - numOfResult; + pQInfo->offset += numOfRowsToCopy; + } else { + pQInfo->offset = 0; + pQInfo->subgroupIdx += 1; + } for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; @@ -3932,13 +3931,12 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde } numOfResult += numOfRowsToCopy; - assert(0); - // if (numOfResult == pQuery->rec.pointsToRead) { - // break; - // } + if (numOfResult == pQuery->rec.capacity) { + break; + } } - dTrace("QInfo:%p copy data to SQuery buf completed", GET_QINFO_ADDR(pQuery)); + dTrace("QInfo:%p copy data to SQuery buf completed", pQInfo); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pQuery, numOfResult); @@ -4135,10 +4133,11 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage } void vnodePrintQueryStatistics(SQInfo *pQInfo) { +#if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; -#if 0 + SQueryCostSummary *pSummary = &pRuntimeEnv->summary; if (pRuntimeEnv->pResultBuf == NULL) { pSummary->tmpBufferInDisk = 0; @@ -4179,41 +4178,30 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { +int32_t doInitializeQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; - // only the successful complete requries the sem_post/over = 1 operations. - if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || - (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { - dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, - pQuery->window.ekey, pQuery->order.order); - - sem_post(&pQInfo->dataReady); - setQueryStatus(pQuery, QUERY_COMPLETED); - return TSDB_CODE_SUCCESS; - } - setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); // dataInCache requires lastKey value pQuery->lastKey = pQuery->window.skey; - STsdbQueryCond cond = {0}; - - cond.twindow = (STimeWindow){.skey = pQuery->window.skey, .ekey = pQuery->window.ekey}; - cond.order = pQuery->order.order; - cond.colList = *pQuery->colList; + STsdbQueryCond cond = { + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = *pQuery->colList, + }; SArray *cols = taosArrayInit(pQuery->numOfCols, sizeof(pQuery->colList[0])); for (int32_t i = 0; i < pQuery->numOfCols; ++i) { taosArrayPush(cols, &pQuery->colList[i]); } - - pQInfo->runtimeEnv.pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); - - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + pRuntimeEnv->pQueryHandle = tsdbQueryByTableId(tsdb, &cond, pQInfo->pTableIdList, cols); pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; @@ -4224,15 +4212,34 @@ int32_t initQInfo(SQInfo *pQInfo, void *param, void* tsdb) { } // create runtime environment - code = setupQueryRuntimeEnv(&pQInfo->runtimeEnv, NULL, pQuery->order.order, false); + code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, false); - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { + pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); + + if (isSTableQuery) { + int32_t rows = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (pQuery->intervalTime == 0) { + int16_t type = TSDB_DATA_TYPE_NULL; + + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; + type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + } else { + type = TSDB_DATA_TYPE_INT; // group id + } + + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); + } + + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4349,6 +4356,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { for(int32_t i = 0; i < numOfTables; ++i) { if (pQInfo->pTableDataInfo[i].pTableQInfo->tid == blockInfo.sid) { pTableDataInfo = &pQInfo->pTableDataInfo[i]; + break; } } @@ -4361,7 +4369,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); TSKEY nextKey = blockInfo.window.ekey; - if (pQuery->intervalTime == 0) { + if (!isIntervalQuery(pQuery)) { setExecutionContext(pQInfo, pTableQueryInfo, pTableDataInfo->tableIndex, pTableDataInfo->groupIdx, nextKey); } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); @@ -4487,9 +4495,10 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start */ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - -#if 0 SQuery* pQuery = pRuntimeEnv->pQuery; + setQueryStatus(pQuery, QUERY_COMPLETED); + +#if 0 // tSidSet *pTableIdList = pSupporter->pSidSet; int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode; @@ -4876,9 +4885,10 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { doRestoreContext(pQInfo); } else { dTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); - return; } + setQueryStatus(pQuery, QUERY_COMPLETED); + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { dTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; @@ -5245,14 +5255,16 @@ static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { assert(pQueryMsg->numOfTables > 0); - *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - taosArrayPush(*pTableIdList, pTableIdInfo); + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; + taosArrayPush(*pTableIdList, &id); + pMsg += sizeof(STableIdInfo); for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { @@ -5820,7 +5832,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->pos = -1; - dTrace("QInfo %p is allocated", pQInfo); + pQuery->window.skey = pQueryMsg->window.skey; + pQuery->window.ekey = pQueryMsg->window.ekey; + pQuery->lastKey = pQuery->window.skey; + + if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { + dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); + goto _clean_memory; + } + + vnodeParametersSafetyCheck(pQuery); + + dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); return pQInfo; _clean_memory: @@ -5859,49 +5882,34 @@ static bool isValidQInfo(void *param) { } static void freeQInfo(SQInfo *pQInfo); -static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, - SArray *pTableIdList, void* tsdb, SQInfo **pQInfo) { +static int32_t initializeQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pSqlExprs, pTableIdList); - if ((*pQInfo) == NULL) { - code = TSDB_CODE_SERV_OUT_OF_MEMORY; - goto _error; - } - - SQuery *pQuery = (*pQInfo)->runtimeEnv.pQuery; - dTrace("qmsg:%p create QInfo:%p, QInfo created", pQueryMsg, pQInfo); - - pQuery->window.skey = pQueryMsg->window.skey; - pQuery->window.ekey = pQueryMsg->window.ekey; - pQuery->lastKey = pQuery->window.skey; - - if (sem_init(&(*pQInfo)->dataReady, 0, 0) != 0) { - dError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); - code = TSDB_CODE_APP_ERROR; - goto _error; - } - - vnodeParametersSafetyCheck(pQuery); - STSBuf *pTSBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); - + tsBufResetPos(pTSBuf); tsBufNextPos(pTSBuf); } - - // filter the qualified - if ((code = initQInfo(*pQInfo, pTSBuf, tsdb)) != TSDB_CODE_SUCCESS) { - goto _error; + + // only the successful complete requries the sem_post/over = 1 operations. + if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { + dTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, + pQuery->window.ekey, pQuery->order.order); + + sem_post(&pQInfo->dataReady); + setQueryStatus(pQuery, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; } - // if (pQInfo->over == 1) { - // vnodeAddRefCount(pQInfo); // for retrieve procedure - // return pQInfo; - // } + // filter the qualified + if ((code = doInitializeQInfo(pQInfo, pTSBuf, tsdb, isSTable)) != TSDB_CODE_SUCCESS) { + goto _error; + } // dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, // pQInfo->refCount); @@ -5909,7 +5917,7 @@ static int32_t createQInfo(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyE _error: // table query ref will be decrease during error handling - freeQInfo(*pQInfo); + freeQInfo(pQInfo); return code; } @@ -6068,7 +6076,10 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) // super table query SArray* res = NULL; + bool isSTableQuery = false; if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { + isSTableQuery = true; + STableId* id = taosArrayGet(pTableIdList, 0); id->uid = -1; @@ -6081,8 +6092,13 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) res = pTableIdList; } - code = createQInfo(pQueryMsg, pGroupbyExpr, pExprs, res, tsdb, pQInfo); - + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, res); + if ((*pQInfo) == NULL) { + code = TSDB_CODE_SERV_OUT_OF_MEMORY; + } + + code = initializeQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); + _query_over: if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pTableIdList); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 3119fdc7cc..7b996c8f7a 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -73,7 +73,7 @@ typedef struct SQueryFilesInfo { char dbFilePathPrefix[PATH_MAX]; } SQueryFilesInfo; -typedef struct STableQueryInfo { +typedef struct STableCheckInfo { STableId tableId; TSKEY lastKey; STable * pTableObj; @@ -81,7 +81,8 @@ typedef struct STableQueryInfo { int32_t numOfBlocks; int32_t start; SCompBlock *pBlock; -} STableQueryInfo; + SSkipListIterator* iter; +} STableCheckInfo; typedef struct { SCompBlock *compBlock; @@ -90,14 +91,19 @@ typedef struct { typedef struct STableDataBlockInfoEx { SCompBlockFields pBlock; - STableQueryInfo* pMeterDataInfo; + STableCheckInfo* pMeterDataInfo; int32_t blockIndex; int32_t groupIdx; /* number of group is less than the total number of meters */ } STableDataBlockInfoEx; +enum { + SINGLE_TABLE_MODEL = 1, + MULTI_TABLE_MODEL = 2, +}; + typedef struct STsdbQueryHandle { struct STsdbRepo* pTsdb; - + int8_t model; // access model, single table model or multi-table model SQueryHandlePos cur; // current position SQueryHandlePos start; // the start position, used for secondary/third iteration int32_t unzipBufSize; @@ -120,14 +126,13 @@ typedef struct STsdbQueryHandle { bool locateStart; int32_t realNumOfRows; bool loadDataAfterSeek; // load data after seek. - SArray* pTableQueryInfo; + SArray* pTableCheckInfo; int32_t activeIndex; int32_t tableIndex; bool isFirstSlot; void * qinfo; // query info handle, for debug purpose - SSkipListIterator* memIter; STableDataBlockInfoEx *pDataBlockInfoEx; } STsdbQueryHandle; @@ -271,19 +276,21 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond size_t size = taosArrayGetSize(idList); assert(size >= 1); - pQueryHandle->pTableQueryInfo = taosArrayInit(size, sizeof(STableQueryInfo)); + pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo)); for(int32_t i = 0; i < size; ++i) { STableId id = *(STableId*) taosArrayGet(idList, i); - STableQueryInfo info = { + STableCheckInfo info = { .lastKey = pQueryHandle->window.skey, .tableId = id, .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed }; - taosArrayPush(pQueryHandle->pTableQueryInfo, &info); + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } + pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL; + pQueryHandle->activeIndex = 0; // malloc buffer in order to load data from file @@ -314,11 +321,13 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond return (tsdb_query_handle_t)pQueryHandle; } -bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex); +static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) { + assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); - STable *pTable = pTableQInfo->pTableObj; + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + assert(pTable != NULL); // no data in cache, abort if (pTable->mem == NULL && pTable->imem == NULL) { @@ -326,13 +335,49 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { } // all data in mem are checked already. - if (pTableQInfo->lastKey > pTable->mem->keyLast) { + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { return false; } return true; } +static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { + size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); + assert(numOfTables > 0); + + while(pHandle->activeIndex < numOfTables) { + STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + STable *pTable = pTableCheckInfo->pTableObj; + if (pTable->mem == NULL && pTable->imem == NULL) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + // all data in mem are checked already. + if (pTableCheckInfo->lastKey > pTable->mem->keyLast) { + pHandle->activeIndex += 1; // try next table if exits + continue; + } + + return true; + } + + // all tables has checked already + return false; +} + +// handle data in cache situation +bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { + STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; + if (pHandle->model == SINGLE_TABLE_MODEL) { + return hasMoreDataInCacheForSingleModel(pHandle); + } else { + return hasMoreDataInCacheForMultiModel(pHandle); + } +} + static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) { int numOfRows = 0; @@ -370,7 +415,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex); + STableCheckInfo* pTableQInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); STable *pTable = pTableQInfo->pTableObj; TSKEY skey = 0, ekey = 0; @@ -379,11 +424,11 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { if (pTable->mem != NULL) { // create mem table iterator if it is not created yet - if (pHandle->memIter == NULL) { - pHandle->memIter = tSkipListCreateIter(pTable->mem->pData); + if (pTableQInfo->iter == NULL) { + pTableQInfo->iter = tSkipListCreateIter(pTable->mem->pData); } - rows = tsdbReadRowsFromCache(pHandle->memIter, INT64_MAX, 2, &skey, &ekey, pHandle); + rows = tsdbReadRowsFromCache(pTableQInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle); } SDataBlockInfo blockInfo = { From f0bd200eb62de8568a514892fca492469369c046 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sat, 28 Mar 2020 22:52:01 +0800 Subject: [PATCH 70/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 79 ++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index d18e766c38..984f2ca203 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -1069,10 +1069,12 @@ static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsTo // Write the block if (write(pFile->fd, (void *)pCompData, size) < 0) goto _err; + *len += size; for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { SDataCol *pDataCol = pCols->cols + iCol; SCompCol *pCompCol = pCompData->cols + iCol; if (write(pFile->fd, pDataCol->pData, pCompCol->len) < 0) goto _err; + *len += pCompCol->len; } if (pCompData == NULL) free((void *)pCompData); @@ -1083,22 +1085,46 @@ _err: return -1; } -static int tsdbWriteBlockToFile(STsdbRepo *pRepo, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock) { - STsdbCfg *pCfg = &(pRepo->config); +static int compareKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SCompBlock *pBlock = (SCompBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } + + return 0; +} + +static int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { + STsdbCfg * pCfg = &(pRepo->config); SCompData *pCompData = NULL; + SFile * pFile = NULL; + int numOfPointsToWrite = 0; + int64_t offset = 0; + int32_t len = 0; memset((void *)pCompBlock, 0, sizeof(SCompBlock)); if (pCompInfo == NULL) { - if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file - // tsdbWriteBlockToFileImpl() - } else { // Write to .last or .l file + // Just append the data block to .data or .l or .last file + numOfPointsToWrite = pCols->numOfPoints; + if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // Write to .data file + pFile = &(pGroup->files[TSDB_FILE_TYPE_DATA]); + } else { // Write to .last or .l file pCompBlock->last = 1; - + if (lFile) { + pFile = lFile; + } else { + pFile = &(pGroup->files[TSDB_FILE_TYPE_LAST]); + } } - // pCompBlock->offset = ; - // pCompBlock->len = ; - pCompBlock->algorithm = 2; // TODO : add to configuration + tsdbWriteBlockToFileImpl(pFile, pCols, numOfPointsToWrite, &offset, &len, uid); + pCompBlock->offset = offset; + pCompBlock->len = len; + pCompBlock->algorithm = 2; // TODO : add to configuration pCompBlock->sversion = pCols->sversion; pCompBlock->numOfPoints = pCols->numOfPoints; pCompBlock->numOfSubBlocks = 1; @@ -1106,7 +1132,38 @@ static int tsdbWriteBlockToFile(STsdbRepo *pRepo, SCompIdx *pIdx, SCompInfo *pCo pCompBlock->keyFirst = dataColsKeyFirst(pCols); pCompBlock->keyLast = dataColsKeyLast(pCols); } else { - // Need to merge + // Need to merge the block to either the last block or the other block + TSKEY keyFirst = dataColsKeyFirst(pCols); + SCompBlock *pMergeBlock = NULL; + + // Search the block to merge in + void *ptr = taosbsearch((void *)&keyFirst, (void *)(pCompInfo->blocks), sizeof(SCompBlock), pIdx->numOfSuperBlocks, + compareKeyBlock, TD_GE); + if (ptr == NULL) { + // No block greater or equal than the key, but there are data in the .last file, need to merge the last file block + // and merge the data + pMergeBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks - 1); + } else { + pMergeBlock = (SCompBlock *)ptr; + } + + if (pMergeBlock->last) { + if (pMergeBlock->last + pCols->numOfPoints > pCfg->minRowsPerFileBlock) { + // Need to load the data from .last and combine data in pCols to write to .data file + + } else { // Just append the block to .last or .l file + if (lFile) { + // read the block from .last file and merge with pCols, write to .l file + + } else { + // tsdbWriteBlockToFileImpl(); + } + } + } else { // The block need to merge in .data file + + } + } - return 0; + + return numOfPointsToWrite; } \ No newline at end of file From 393a816c24f52335184502196befe5bbc8c9062f Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 29 Mar 2020 14:14:51 +0800 Subject: [PATCH 71/85] [TD-32] add unit test --- src/client/tests/timeParseTest.cpp | 167 ++++++ src/inc/taosmsg.h | 9 - src/query/CMakeLists.txt | 4 +- src/query/tests/CMakeLists.txt | 15 + src/query/tests/histogramTest.cpp | 121 +++++ src/query/tests/patternMatchTest.cpp | 74 +++ src/query/tests/resultBufferTest.cpp | 35 ++ src/query/tests/tsBufTest.cpp | 451 ++++++++++++++++ src/query/tests/unitTest.cpp | 769 +++++++++++++++++++++++++++ src/util/CMakeLists.txt | 3 +- src/util/tests/CMakeLists.txt | 15 + src/util/tests/cacheTest.cpp | 142 +++++ src/util/tests/hashTest.cpp | 156 ++++++ src/util/tests/stringTest.cpp | 178 +++++++ src/vnode/tsdb/tests/CMakeLists.txt | 7 +- 15 files changed, 2129 insertions(+), 17 deletions(-) create mode 100644 src/client/tests/timeParseTest.cpp create mode 100644 src/query/tests/CMakeLists.txt create mode 100644 src/query/tests/histogramTest.cpp create mode 100644 src/query/tests/patternMatchTest.cpp create mode 100644 src/query/tests/resultBufferTest.cpp create mode 100644 src/query/tests/tsBufTest.cpp create mode 100644 src/query/tests/unitTest.cpp create mode 100644 src/util/tests/CMakeLists.txt create mode 100644 src/util/tests/cacheTest.cpp create mode 100644 src/util/tests/hashTest.cpp create mode 100644 src/util/tests/stringTest.cpp diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp new file mode 100644 index 0000000000..1f53d50c1f --- /dev/null +++ b/src/client/tests/timeParseTest.cpp @@ -0,0 +1,167 @@ +#include +#include +#include + +#include "taos.h" +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +extern void deltaToUtcInitOnce(); +/* test parse time function */ +TEST(testCase, parse_time) { + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + deltaToUtcInitOnce(); + + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); + + + //======================== add some case ============================// + + char b1[] = "9999-12-31 23:59:59.999"; + taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 253402271999999); + + + char b2[] = "2020-01-01 01:01:01.321"; + taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1577811661321); + + taos_options(TSDB_OPTION_TIMEZONE, "America/New_York"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + deltaToUtcInitOnce(); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); +} + + diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 9ac06e4199..4632c67c3c 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -188,15 +188,6 @@ extern char *taosMsg[]; #pragma pack(push, 1) -// typedef struct { -// int32_t vnode; -// int32_t sid; -// int32_t sversion; -// uint64_t uid; -// int16_t numOfRows; -// char payLoad[]; -//} SShellSubmitBlock; - typedef struct { int32_t numOfVnodes; } SMsgDesc; diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index 0e51962f49..a6462f9855 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -12,4 +12,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(query ${SRC}) TARGET_LINK_LIBRARIES(query tsdb tutil m rt) -ENDIF () \ No newline at end of file +ENDIF () + +ADD_SUBDIRECTORY(tests) \ No newline at end of file diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt new file mode 100644 index 0000000000..0ae8600756 --- /dev/null +++ b/src/query/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp new file mode 100644 index 0000000000..c23f0f5924 --- /dev/null +++ b/src/query/tests/histogramTest.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "tutil.h" + +#include "qhistogram.h" + +/* test validate the names for table/database */ +TEST(testCase, histogram_binary_search) { + SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); + + pHisto->numOfEntries = 10; + for (int32_t i = 0; i < 10; ++i) { + pHisto->elems[i].num = 1; + pHisto->elems[i].val = i; + } + + int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + assert(idx == 1); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + assert(idx == 9); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + assert(idx == 10); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + assert(idx == 0); + + idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + assert(idx == 4); + + free(pHisto); +} + +TEST(testCase, histogram_add) { + SHistogramInfo* pHisto = NULL; + + /** + * use arrayList, elapsed time is: + * before: + * 10,000,000 45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3) + * + * after: + * + */ + struct timeval systemTime; + gettimeofday(&systemTime, NULL); + int64_t st = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + for (int32_t i = 0; i < 10000; ++i) { + tHistogramAdd(&pHisto, i); + // tHistogramPrint(pHisto); + } + // + gettimeofday(&systemTime, NULL); + int64_t et = + (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000; + printf("total elapsed time: %ld\n", et - st); + + printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries); + tHistogramPrint(pHisto); + + printf("%ld\n", tHistogramSum(pHisto, 1.5)); + printf("%ld\n", tHistogramSum(pHisto, 2)); + printf("%ld\n", tHistogramSum(pHisto, 3)); + printf("%ld\n", tHistogramSum(pHisto, 4)); + printf("%ld\n", tHistogramSum(pHisto, 5)); + printf("%ld\n", tHistogramSum(pHisto, 6)); + + for (int32_t i = 399; i < 400; ++i) { + printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i)); + } + + double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100}; + double* res = tHistogramUniform(pHisto, ratio, 4); + for (int32_t i = 0; i < 4; ++i) { + printf("%f\n", res[i]); + } + + SHistogramInfo* pHisto1 = NULL; + for (int32_t i = (90000 - 1); i >= 80000; --i) { + tHistogramAdd(&pHisto1, i); + } + tHistogramPrint(pHisto1); + + SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN); + assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems); + tHistogramPrint(pRes); + + tHistogramDestroy(&pHisto); + tHistogramDestroy(&pHisto1); + tHistogramDestroy(&pRes); + free(res); +} + +TEST(testCase, heapsort) { + // int32_t num = 20; + // + // SHeapEntry* pEntry = tHeapCreate(num); + // + // for(int32_t i=0; i +#include +#include +#include + +#include "tsqlfunction.h" + +TEST(testCase, patternMatchTest) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + const char* str = "abcdef"; + int32_t ret = patternMatch("a%b%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tm01"; + ret = patternMatch("tm__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("t%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "tkm1"; + ret = patternMatch("%m1", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = ""; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "1"; + ret = patternMatch("%__", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = ""; + ret = patternMatch("%", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = " "; + ret = patternMatch("_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "!"; + ret = patternMatch("%_", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefg"; + ret = patternMatch("abc%fg", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%fg", str, 7, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 6, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("abc%f_", str, 1, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("ab", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a%", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "abcdefgabcdeju"; + ret = patternMatch("a__", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); +} diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp new file mode 100644 index 0000000000..fca9ac8f9d --- /dev/null +++ b/src/query/tests/resultBufferTest.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +#include "taos.h" +#include "qresultBuf.h" +#include "tsdb.h" + +namespace { +// simple test +void simpleTest() { + SDiskbasedResultBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64); + + int32_t pageId = 0; + int32_t groupId = 0; + + tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + + ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64); + ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); + + SIDList list = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(list.size, 1); + + ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); + + destroyResultBuf(pResultBuf); +} +} // namespace + +TEST(testCase, resultBufferTest) { + simpleTest(); +} diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp new file mode 100644 index 0000000000..2392bd00b5 --- /dev/null +++ b/src/query/tests/tsBufTest.cpp @@ -0,0 +1,451 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "tstoken.h" +#include "ttime.h" +#include "tutil.h" +#include "qtsbuf.h" + +namespace { +/** + * + * @param num total number + * @param step gap between two consecutive ts + * @return + */ +int64_t* createTsList(int32_t num, int64_t start, int32_t step) { + int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t)); + + for (int64_t i = 0; i < num; ++i) { + pList[i] = start + i * step; + } + + return pList; +} + +// simple test +void simpleTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 10; + int64_t tag = 1; + + int64_t* list = createTsList(10, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +// one large list of ts, the ts list need to be split into several small blocks +void largeTSTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + // write 10 ts points + int32_t num = 1000000; + int64_t tag = 1; + + int64_t* list = createTsList(num, 10000000, 30); + tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + + // the data has been flush to disk, no data in cache + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t tag = 1; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->numOfVnodes, 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void multiVnodeTagsTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t start = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + + // 2000 vnodes + for (int32_t j = 0; j < 20; ++j) { + // vnodeId:0 + start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + tsBufDestory(pTSBuf); +} + +void loadDataTest() { + STSBuf* pTSBuf = tsBufCreate(true); + + int32_t num = 10000; + int64_t oldStart = 10000000; + int32_t numOfTags = 50; + int32_t step = 30; + int32_t numOfVnode = 200; + + // 10000 vnodes + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %lld\n", i, list[0]); + + free(list); + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); + + EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + + tsBufFlush(pTSBuf); + EXPECT_EQ(pTSBuf->tsData.len, 0); + EXPECT_EQ(pTSBuf->block.numOfElem, num); + + // create from exists file + STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false); + EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder); + EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode); + EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize); + + EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset); + EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks); + EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen); + + EXPECT_STREQ(pNewBuf->path, pTSBuf->path); + + tsBufResetPos(pNewBuf); + + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + int32_t x = 0; + while (tsBufNextPos(pNewBuf)) { + STSElem elem = tsBufGetElem(pNewBuf); + if (++x == 100000000) { + break; + } + + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + int64_t e = taosGetTimestampUs(); + printf("end:%lld, elapsed:%lld, total obj:%d\n", e, e - s, x); +} + +void randomIncTsTest() {} + +void TSTraverse() { + // 10000 vnodes + int32_t num = 200000; + int64_t oldStart = 10000000; + int32_t numOfTags = 3; + int32_t step = 30; + int32_t numOfVnode = 2; + + STSBuf* pTSBuf = tsBufCreate(true); + + for (int32_t j = 0; j < numOfVnode; ++j) { + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + + free(list); + start += step * num; + + list = createTsList(num, start, step); + tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]); + free(list); + + start += step * num; + } + + EXPECT_EQ(pTSBuf->numOfVnodes, j + 1); + } + + tsBufResetPos(pTSBuf); + + //////////////////////////////////////////////////////////////////////////////////////// + // reverse traverse + int64_t s = taosGetTimestampUs(); + printf("start:%lld\n", s); + + pTSBuf->cur.order = TSQL_SO_DESC; + + // complete reverse traverse + int32_t x = 0; + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_DESC; + + int32_t startVnode = 1; + int32_t startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + int32_t totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag == 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } + + ///////////////////////////////////////////////////////////////////////////////// + // traverse + pTSBuf->cur.order = TSQL_SO_ASC; + tsBufResetPos(pTSBuf); + + // complete forwards traverse + while (tsBufNextPos(pTSBuf)) { + STSElem elem = tsBufGetElem(pTSBuf); + // printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + // specify the data block with vnode and tags value + tsBufResetPos(pTSBuf); + pTSBuf->cur.order = TSQL_SO_ASC; + + startVnode = 1; + startTag = 2; + + tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + + totalOutput = 10; + while (1) { + STSElem elem = tsBufGetElem(pTSBuf); + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + + if (!tsBufNextPos(pTSBuf)) { + break; + } + + if (--totalOutput <= 0) { + totalOutput = 10; + + tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + + if (startTag < 0) { + startVnode -= 1; + startTag = 3; + } + + if (startVnode < 0) { + break; + } + } + } +} + +void performanceTest() {} + +void emptyTagTest() {} + +void invalidFileTest() { + const char* cmd = "touch /tmp/test"; + + // create empty file + system(cmd); + + STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); + EXPECT_TRUE(pNewBuf == NULL); + + pNewBuf = tsBufCreateFromFile("/tmp/911", true); + EXPECT_TRUE(pNewBuf == NULL); +} + +void mergeDiffVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 9); + EXPECT_EQ(pTSBuf1->numOfVnodes, 2); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufDisplay(pTSBuf1); + + tsBufDestory(pTSBuf2); + tsBufDestory(pTSBuf1); +} + +void mergeIdenticalVnodeBufferTest() { + STSBuf* pTSBuf1 = tsBufCreate(true); + STSBuf* pTSBuf2 = tsBufCreate(true); + + int32_t step = 30; + int32_t num = 1000; + int32_t numOfTags = 10; + + // vnodeId:0 + int64_t start = 10000000; + for (int32_t i = 0; i < numOfTags; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { + int64_t* list = createTsList(num, start, step); + + tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + free(list); + + start += step * num; + } + + tsBufFlush(pTSBuf2); + + tsBufMerge(pTSBuf1, pTSBuf2, 12); + EXPECT_EQ(pTSBuf1->numOfVnodes, 1); + EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); + + tsBufResetPos(pTSBuf1); + while (tsBufNextPos(pTSBuf1)) { + STSElem elem = tsBufGetElem(pTSBuf1); + EXPECT_EQ(elem.vnode, 12); + + printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts); + } + + tsBufDestory(pTSBuf1); + tsBufDestory(pTSBuf2); +} +} // namespace + +TEST(testCase, tsBufTest) { + simpleTest(); + largeTSTest(); + multiTagsTest(); + multiVnodeTagsTest(); + loadDataTest(); + invalidFileTest(); + // randomIncTsTest(); + TSTraverse(); + mergeDiffVnodeBufferTest(); + mergeIdenticalVnodeBufferTest(); +} diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp new file mode 100644 index 0000000000..c7520c90f9 --- /dev/null +++ b/src/query/tests/unitTest.cpp @@ -0,0 +1,769 @@ +#include +#include +#include + +#include "taos.h" +#include "tsdb.h" + +#include "../../client/inc/tscUtil.h" +#include "ttime.h" +#include "tutil.h" +#include "tvariant.h" +#include "ttokendef.h" + +namespace { +int32_t testValidateName(char* name) { + SSQLToken token = {0}; + token.z = name; + token.n = strlen(name); + token.type = 0; + + tSQLGetToken(name, &token.type); + return tscValidateName(&token); +} +} + +static void _init_tvariant_bool(tVariant* t) { + t->i64Key = TSDB_FALSE; + t->nType = TSDB_DATA_TYPE_BOOL; +} + +static void _init_tvariant_tinyint(tVariant* t) { + t->i64Key = -27; + t->nType = TSDB_DATA_TYPE_TINYINT; +} + +static void _init_tvariant_int(tVariant* t) { + t->i64Key = -23997659; + t->nType = TSDB_DATA_TYPE_INT; +} + +static void _init_tvariant_bigint(tVariant* t) { + t->i64Key = -3333333333333; + t->nType = TSDB_DATA_TYPE_BIGINT; +} + +static void _init_tvariant_float(tVariant* t) { + t->dKey = -8991212199.8987878776; + t->nType = TSDB_DATA_TYPE_FLOAT; +} + +static void _init_tvariant_binary(tVariant* t) { + tVariantDestroy(t); + + t->pz = (char*)calloc(1, 20); //"2e3"); + t->nType = TSDB_DATA_TYPE_BINARY; + strcpy(t->pz, "2e5"); + t->nLen = strlen(t->pz); +} + +static void _init_tvariant_nchar(tVariant* t) { + tVariantDestroy(t); + + t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE); + t->nType = TSDB_DATA_TYPE_NCHAR; + wcscpy(t->wpz, L"-2000000.8765"); + t->nLen = wcslen(t->wpz); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/* test validate the names for table/database */ +TEST(testCase, db_table_name) { + + char t01[] = "abc"; + EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); + + char t02[] = "'abc'"; + EXPECT_EQ(testValidateName(t02), TSDB_CODE_SUCCESS); + + char t1[] = "abc.def"; + EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS); + printf("%s\n", t1); + + char t2[] = "'abc.def'"; + EXPECT_EQ(testValidateName(t2), TSDB_CODE_SUCCESS); + printf("%s\n", t2); + + char t3[] = "'abc'.def"; + EXPECT_EQ(testValidateName(t3), TSDB_CODE_SUCCESS); + printf("%s\n", t3); + + char t4[] = "'abc'.'def'"; + EXPECT_EQ(testValidateName(t4), TSDB_CODE_SUCCESS); + + char t5[] = "table.'def'"; + EXPECT_EQ(testValidateName(t5), TSDB_CODE_INVALID_SQL); + + char t6[] = "'table'.'def'"; + EXPECT_EQ(testValidateName(t6), TSDB_CODE_INVALID_SQL); + + char t7[] = "'_ab1234'.'def'"; + EXPECT_EQ(testValidateName(t7), TSDB_CODE_SUCCESS); + printf("%s\n", t7); + + char t8[] = "'_ab&^%1234'.'def'"; + EXPECT_EQ(testValidateName(t8), TSDB_CODE_INVALID_SQL); + + char t9[] = "'_123'.'gtest中文'"; + EXPECT_EQ(testValidateName(t9), TSDB_CODE_INVALID_SQL); + + char t10[] = "abc.'gtest中文'"; + EXPECT_EQ(testValidateName(t10), TSDB_CODE_INVALID_SQL); + + char t10_1[] = "abc.'中文gtest'"; + EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_INVALID_SQL); + + char t11[] = "'192.168.0.1'.abc"; + EXPECT_EQ(testValidateName(t11), TSDB_CODE_INVALID_SQL); + + char t12[] = "192.168.0.1.abc"; + EXPECT_EQ(testValidateName(t12), TSDB_CODE_INVALID_SQL); + + char t13[] = "abc."; + EXPECT_EQ(testValidateName(t13), TSDB_CODE_INVALID_SQL); + + char t14[] = ".abc"; + EXPECT_EQ(testValidateName(t14), TSDB_CODE_INVALID_SQL); + + char t15[] = ".'abc'"; + EXPECT_EQ(testValidateName(t15), TSDB_CODE_INVALID_SQL); + + char t16[] = ".abc'"; + EXPECT_EQ(testValidateName(t16), TSDB_CODE_INVALID_SQL); + + char t17[] = "123a.\"abc\""; + EXPECT_EQ(testValidateName(t17), TSDB_CODE_INVALID_SQL); + printf("%s\n", t17); + + char t18[] = "a.\"abc\""; + EXPECT_EQ(testValidateName(t18), TSDB_CODE_SUCCESS); + printf("%s\n", t18); + + char t19[] = "'_ab1234'.'def'.'ab123'"; + EXPECT_EQ(testValidateName(t19), TSDB_CODE_INVALID_SQL); + + char t20[] = "'_ab1234*&^'"; + EXPECT_EQ(testValidateName(t20), TSDB_CODE_INVALID_SQL); + + char t21[] = "'1234_abc'"; + EXPECT_EQ(testValidateName(t21), TSDB_CODE_INVALID_SQL); + + + // =======Containing capital letters================= + char t30[] = "ABC"; + EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS); + + char t31[] = "'ABC'"; + EXPECT_EQ(testValidateName(t31), TSDB_CODE_SUCCESS); + + char t32[] = "ABC.def"; + EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS); + + char t33[] = "'ABC.def"; + EXPECT_EQ(testValidateName(t33), TSDB_CODE_INVALID_SQL); + + char t33_0[] = "abc.DEF'"; + EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_INVALID_SQL); + + char t34[] = "'ABC.def'"; + //int32_t tmp0 = testValidateName(t34); + EXPECT_EQ(testValidateName(t34), TSDB_CODE_SUCCESS); + + char t35[] = "'ABC'.def"; + EXPECT_EQ(testValidateName(t35), TSDB_CODE_SUCCESS); + + char t36[] = "'ABC'.'DEF'"; + EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS); + + char t37[] = "abc.'DEF'"; + EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS); + + char t37_1[] = "abc.'_123DEF'"; + EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS); + + char t38[] = "'abc'.'DEF'"; + EXPECT_EQ(testValidateName(t38), TSDB_CODE_SUCCESS); + + // do not use key words + char t39[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t39), TSDB_CODE_INVALID_SQL); + + char t40[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t40), TSDB_CODE_INVALID_SQL); + + char t41[] = "'_abXYZ1234'.'deFF'"; + EXPECT_EQ(testValidateName(t41), TSDB_CODE_SUCCESS); + + char t42[] = "'_abDEF&^%1234'.'DIef'"; + EXPECT_EQ(testValidateName(t42), TSDB_CODE_INVALID_SQL); + + char t43[] = "'_123'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t43), TSDB_CODE_INVALID_SQL); + + char t44[] = "'aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t44), TSDB_CODE_INVALID_SQL); + + char t45[] = "'ABC'."; + EXPECT_EQ(testValidateName(t45), TSDB_CODE_INVALID_SQL); + + char t46[] = ".'ABC'"; + EXPECT_EQ(testValidateName(t46), TSDB_CODE_INVALID_SQL); + + char t47[] = "a.\"aTWc\""; + EXPECT_EQ(testValidateName(t47), TSDB_CODE_SUCCESS); + + // ================has space ================= + char t60[] = " ABC "; + EXPECT_EQ(testValidateName(t60), TSDB_CODE_INVALID_SQL); + + char t60_1[] = " ABC "; + EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_INVALID_SQL); + + char t61[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS); + + char t61_1[] = "' ABC '"; + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS); + + char t62[] = " ABC . def "; + EXPECT_EQ(testValidateName(t62), TSDB_CODE_INVALID_SQL); + + char t63[] = "' ABC . def "; + EXPECT_EQ(testValidateName(t63), TSDB_CODE_INVALID_SQL); + + char t63_0[] = " abc . DEF ' "; + EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_INVALID_SQL); + + char t64[] = " ' ABC . def ' "; + //int32_t tmp1 = testValidateName(t64); + EXPECT_EQ(testValidateName(t64), TSDB_CODE_INVALID_SQL); + + char t65[] = " ' ABC '. def "; + EXPECT_EQ(testValidateName(t65), TSDB_CODE_INVALID_SQL); + + char t66[] = "' ABC '.' DEF '"; + EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS); + + char t67[] = "abc . ' DEF '"; + EXPECT_EQ(testValidateName(t67), TSDB_CODE_INVALID_SQL); + + char t68[] = "' abc '.' DEF '"; + EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS); + + // do not use key words + char t69[] = "table.'DEF'"; + EXPECT_EQ(testValidateName(t69), TSDB_CODE_INVALID_SQL); + + char t70[] = "'table'.'DEF'"; + EXPECT_EQ(testValidateName(t70), TSDB_CODE_INVALID_SQL); + + char t71[] = "'_abXYZ1234 '.' deFF '"; + EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS); + + char t72[] = "'_abDEF&^%1234'.' DIef'"; + EXPECT_EQ(testValidateName(t72), TSDB_CODE_INVALID_SQL); + + char t73[] = "'_123'.' Gtest中文'"; + EXPECT_EQ(testValidateName(t73), TSDB_CODE_INVALID_SQL); + + char t74[] = "' aABC'.'Gtest中文'"; + EXPECT_EQ(testValidateName(t74), TSDB_CODE_INVALID_SQL); + + char t75[] = "' ABC '."; + EXPECT_EQ(testValidateName(t75), TSDB_CODE_INVALID_SQL); + + char t76[] = ".' ABC'"; + EXPECT_EQ(testValidateName(t76), TSDB_CODE_INVALID_SQL); + + char t77[] = " a . \"aTWc\" "; + EXPECT_EQ(testValidateName(t77), TSDB_CODE_INVALID_SQL); + + char t78[] = " a.\"aTWc \""; + EXPECT_EQ(testValidateName(t78), TSDB_CODE_INVALID_SQL); + + + // ===============muti string by space =================== + // There's no such case. + //char t160[] = "A BC"; + //EXPECT_EQ(testValidateName(t160), TSDB_CODE_INVALID_SQL); + //printf("end:%s\n", t160); + + // There's no such case. + //char t161[] = "' A BC '"; + //EXPECT_EQ(testValidateName(t161), TSDB_CODE_INVALID_SQL); + + char t162[] = " AB C . de f "; + EXPECT_EQ(testValidateName(t162), TSDB_CODE_INVALID_SQL); + + char t163[] = "' AB C . de f "; + EXPECT_EQ(testValidateName(t163), TSDB_CODE_INVALID_SQL); + + char t163_0[] = " ab c . DE F ' "; + EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_INVALID_SQL); + + char t164[] = " ' AB C . de f ' "; + //int32_t tmp2 = testValidateName(t164); + EXPECT_EQ(testValidateName(t164), TSDB_CODE_INVALID_SQL); + + char t165[] = " ' A BC '. de f "; + EXPECT_EQ(testValidateName(t165), TSDB_CODE_INVALID_SQL); + + char t166[] = "' AB C '.' DE F '"; + EXPECT_EQ(testValidateName(t166), TSDB_CODE_INVALID_SQL); + + char t167[] = "ab c . ' D EF '"; + EXPECT_EQ(testValidateName(t167), TSDB_CODE_INVALID_SQL); + + char t168[] = "' a bc '.' DE F '"; + EXPECT_EQ(testValidateName(t168), TSDB_CODE_INVALID_SQL); + +} + +/* test parse time function */ +TEST(testCase, parse_time) { + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + char t1[] = "2018-1-1 1:1:1.952798"; + char t13[] = "1970-1-1 0:0:0"; + + int64_t time = 0, time1 = 0; + + taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND); + + char t2[] = "2018-1-1T1:1:1.952Z"; + taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI); + + EXPECT_EQ(time, 1514739661952 + 28800000); + + char t3[] = "2018-1-1 1:01:01.952"; + taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 1514739661952); + + char t4[] = "2018-1-1 1:01:01.9"; + char t5[] = "2018-1-1 1:01:1.900"; + char t6[] = "2018-01-01 1:1:1.90"; + char t7[] = "2018-01-01 01:01:01.9"; + char t8[] = "2018-01-01 01:01:01.9007865"; + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t9[] = "2017-4-3 1:1:2.980"; + char t10[] = "2017-4-3T2:1:2.98+9:00"; + taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t11[] = "2017-4-3T2:1:2.98+09:00"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + char t12[] = "2017-4-3T2:1:2.98+0900"; + taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI); + taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, time1); + + taos_options(TSDB_OPTION_TIMEZONE, "UTC"); + taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai"); + char t14[] = "1970-1-1T0:0:0Z"; + taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 0); + + char t40[] = "1970-1-1 0:0:0.999999999"; + taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND); + + char t41[] = "1997-1-1 0:0:0.999999999"; + taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999); + + int64_t k = timezone; + char t42[] = "1997-1-1T0:0:0.999999999Z"; + taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); + + //////////////////////////////////////////////////////////////////// + // illegal timestamp format + char t15[] = "2017-12-33 0:0:0"; + EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1); + + char t16[] = "2017-12-31 99:0:0"; + EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1); + + char t17[] = "2017-12-31T9:0:0"; + EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1); + + char t18[] = "2017-12-31T9:0:0.Z"; + EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1); + + char t19[] = "2017-12-31 9:0:0.-1"; + EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1); + + char t20[] = "2017-12-31 9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0); + EXPECT_EQ(time, 1514682000100); + + char t21[] = "2017-12-31T9:0:0.1+12:99"; + EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1); + + char t22[] = "2017-12-31 9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0); + + char t23[] = "2017-12-31T9:0:0.1+13:1"; + EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0); +} + +TEST(testCase, tvariant_convert) { + // 1. bool data to all other data types + tVariant t = {0}; + _init_tvariant_bool(&t); + + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, 0); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "FALSE"); + tVariantDestroy(&t); + + _init_tvariant_bool(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"FALSE"); + tVariantDestroy(&t); + + // 2. tinyint to other data types + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -27); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-27"); + tVariantDestroy(&t); + + _init_tvariant_tinyint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-27"); + tVariantDestroy(&t); + + // 3. int to other data + // types////////////////////////////////////////////////////////////////// + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -23997659); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-23997659"); + tVariantDestroy(&t); + + _init_tvariant_int(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-23997659"); + tVariantDestroy(&t); + + // 4. bigint to other data + // type////////////////////////////////////////////////////////////////////////////// + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_EQ(t.dKey, -3333333333333); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-3333333333333"); + tVariantDestroy(&t); + + _init_tvariant_bigint(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-3333333333333"); + tVariantDestroy(&t); + + // 5. float to other data + // types//////////////////////////////////////////////////////////////////////// + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -8991212199); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -8991212199.8987885); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-8991212199.898788"); + tVariantDestroy(&t); + + _init_tvariant_float(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-8991212199.898788"); + tVariantDestroy(&t); + + // 6. binary to other data types + // ////////////////////////////////////////////////////////////////// + t.pz = "true"; + t.nLen = strlen(t.pz); + t.nType = TSDB_DATA_TYPE_BINARY; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, 200000); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "2e5"); + tVariantDestroy(&t); + + _init_tvariant_binary(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"2e5"); + tVariantDestroy(&t); + + // 7. nchar to other data types + // ////////////////////////////////////////////////////////////////// + t.wpz = L"FALSE"; + t.nLen = wcslen(t.wpz); + t.nType = TSDB_DATA_TYPE_NCHAR; + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + EXPECT_EQ(t.i64Key, 0); + + _init_tvariant_nchar(&t); + EXPECT_LE(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0); + EXPECT_EQ(t.i64Key, -2000000); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0); + EXPECT_DOUBLE_EQ(t.dKey, -2000000.8765); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0); + EXPECT_STREQ(t.pz, "-2000000.8765"); + tVariantDestroy(&t); + + _init_tvariant_nchar(&t); + EXPECT_EQ(tVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0); + EXPECT_STREQ(t.wpz, L"-2000000.8765"); + tVariantDestroy(&t); +} + +TEST(testCase, tGetToken_Test) { + char* s = ".123 "; + uint32_t type = 0; + + int32_t len = tSQLGetToken(s, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s) - 1); + + char s1[] = "1.123e10 "; + len = tSQLGetToken(s1, &type); + EXPECT_EQ(type, TK_FLOAT); + EXPECT_EQ(len, strlen(s1) - 1); + + char s4[] = "0xff "; + len = tSQLGetToken(s4, &type); + EXPECT_EQ(type, TK_HEX); + EXPECT_EQ(len, strlen(s4) - 1); + + // invalid data type + char s2[] = "e10 "; + len = tSQLGetToken(s2, &type); + EXPECT_FALSE(type == TK_FLOAT); + + char s3[] = "1.1.1.1"; + len = tSQLGetToken(s3, &type); + EXPECT_EQ(type, TK_IPTOKEN); + EXPECT_EQ(len, strlen(s3)); + + char s5[] = "0x "; + len = tSQLGetToken(s5, &type); + EXPECT_FALSE(type == TK_HEX); +} + +static SSQLToken createStrToken(char* s) { + SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; + t.type = TK_STRING; + t.z = s; + t.n = strlen(s); + + return t; +} + +TEST(testCase, isValidNumber_test) { + SSQLToken t1 = createStrToken("123abc"); + + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("0xabc"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("0b11101"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken(".134abc"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1e1 "); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("1+2"); + EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); + + t1 = createStrToken("-0x123"); + EXPECT_EQ(isValidNumber(&t1), TK_HEX); + + t1 = createStrToken("-1"); + EXPECT_EQ(isValidNumber(&t1), TK_INTEGER); + + t1 = createStrToken("-0b1110"); + EXPECT_EQ(isValidNumber(&t1), TK_BIN); + + t1 = createStrToken("-.234"); + EXPECT_EQ(isValidNumber(&t1), TK_FLOAT); +} + +TEST(testCase, getTempFilePath_test) { + char path[4096] = {0}; + memset(path, 1, 4096); + + getTmpfilePath("new_tmp", path); + printf("%s\n", path); +} + diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 67ecf39f1b..0844146bd4 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -29,6 +29,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () ENDIF () + + ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) @@ -113,4 +115,3 @@ ENDIF() #ENDIF () - diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt new file mode 100644 index 0000000000..042c925165 --- /dev/null +++ b/src/util/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib) + +IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) + MESSAGE(STATUS "gTest library found, build unit test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + + ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(utilTest tutil gtest pthread) +ENDIF() \ No newline at end of file diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp new file mode 100644 index 0000000000..411c899cc0 --- /dev/null +++ b/src/util/tests/cacheTest.cpp @@ -0,0 +1,142 @@ +#include +#include +#include + +#include "taos.h" +//#include "tsdb.h" + +//#include "testCommon.h" +#include "tstoken.h" +#include "tutil.h" +#include "tcache.h" +#include "ttimer.h" +#include "ttime.h" + +namespace { +int32_t tsMaxMgmtConnections = 10000; +int32_t tsMaxMeterConnections = 200; +} +// test cache +TEST(testCase, client_cache_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC"); + SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char* key1 = "test1"; + char* data1 = "test11"; + + char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1); + sleep(REFRESH_TIME_IN_SEC+1); + + printf("obj is still valid: %s\n", cachedObj); + + char* data2 = "test22"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + + /* the object is cleared by cache clean operation */ + cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20); + printf("after updated: %s\n", cachedObj); + + printf("start to remove data from cache\n"); + taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false); + printf("end of removing data from cache\n"); + + getchar(); + + char* key3 = "test2"; + char* data3 = "kkkkkkk"; + + char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + sleep(3); + char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3); +// assert(d == NULL); + + char* key5 = "test5"; + char* data5 = "data5kkkkk"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20); + + char* data6= "new Data after updated"; + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false); + + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20); + printf("%s\n", cachedObj2); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + + char* data7 = "add call update procedure"; + cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20); + printf("%s\n=======================================\n\n", cachedObj2); + + char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5); + + taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true); + taosCacheRelease(tscCacheHandle, (void**) &cc, false); + + char* data8 = "ttft"; + char* key6 = "key6"; + + char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20); + taosCacheRelease(tscCacheHandle, (void**) &ft, false); + + /** + * 140ns + */ + uint64_t startTime = taosGetTimestampUs(); + printf("Cache Performance Test\nstart time:%lld\n", startTime); + for(int32_t i=0; i<1000; ++i) { + char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6); + if (dd != NULL) { +// printf("get the data\n"); + } else { + printf("data has been released\n"); + } + + taosCacheRelease(tscCacheHandle, (void**) &dd, false); + } + + uint64_t endTime = taosGetTimestampUs(); + int64_t el = endTime - startTime; + + printf("End of Test, %lld\nTotal Elapsed Time:%lld us.avg:%f us\n", endTime, el, el/1000.0); + + taosCacheCleanup(tscCacheHandle); +} + +TEST(testCase, cache_resize_test) { + const int32_t REFRESH_TIME_IN_SEC = 2; + void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC"); + + auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC); + + char key[256] = {0}; + char data[1024] = "abcdefghijk"; + int32_t len = strlen(data); + + uint64_t startTime = taosGetTimestampUs(); + int32_t num = 10000; + + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + taosCachePut(pCache, key, data, len, 3600); + } + uint64_t endTime = taosGetTimestampUs(); + + printf("add 10,000,000 object cost:%lld us, avg:%f us\n", endTime - startTime, (endTime-startTime)/(double)num); + + startTime = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "abc_%7d", i); + void* k = taosCacheAcquireByName(pCache, key); + assert(k != 0); + } + endTime = taosGetTimestampUs(); + printf("retrieve 10,000,000 object cost:%lld us,avg:%f\n", endTime - startTime, (endTime - startTime)/(double)num); + + taosCacheCleanup(pCache); + taosMsleep(20000); + getchar(); +} \ No newline at end of file diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp new file mode 100644 index 0000000000..93a1989741 --- /dev/null +++ b/src/util/tests/hashTest.cpp @@ -0,0 +1,156 @@ +#include +#include +#include +#include + +#include "hash.h" +#include "taos.h" +#include "ttime.h" + +namespace { +// the simple test code for basic operations +void simpleTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + // put 400 elements in the hash table + for(int32_t i = -200; i < 200; ++i) { + taosHashPut(hashTable, (const char*) &i, sizeof(int32_t), (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 200; ++i) { + char* p = (char*) taosHashGet(hashTable, (const char*) &i, sizeof(int32_t)); + ASSERT_TRUE(p != nullptr); + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 1000; i < 2000; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 400); + + for(int32_t i = 0; i < 100; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 300); + + for(int32_t i = 100; i < 150; ++i) { + taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 250); + taosHashCleanup(hashTable); +} + +void stringKeyTest() { + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + + // put 200 elements in the hash table + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1000); + + for(int32_t i = 0; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + for(int32_t i = 500; i < 1000; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 500); + + for(int32_t i = 0; i < 499; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + + taosHashRemove(hashTable, key, len); + } + + ASSERT_EQ(taosHashGetSize(hashTable), 1); + + taosHashCleanup(hashTable); +} + +void functionTest() { + +} + +/** + * evaluate the performance issue, by add 10million elements in to hash table in + * a single threads situation + */ +void noLockPerformanceTest() { + auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + char key[128] = {0}; + int32_t num = 5000000; + + int64_t st = taosGetTimestampUs(); + + // put 10M elements in the hash table + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t)); + } + + ASSERT_EQ(taosHashGetSize(hashTable), num); + + int64_t et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + st = taosGetTimestampUs(); + for(int32_t i = 0; i < num; ++i) { + int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); + char* p = (char*) taosHashGet(hashTable, key, len); + ASSERT_TRUE(p != nullptr); + + ASSERT_EQ(*reinterpret_cast(p), i); + } + + et = taosGetTimestampUs(); + printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num); + + printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable)); + taosHashCleanup(hashTable); +} + +void multithreadsTest() { + //todo +} + +// check the function robustness +void invalidOperationTest() { + +} + +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, hashTest) { + simpleTest(); + stringKeyTest(); + noLockPerformanceTest(); + multithreadsTest(); +} \ No newline at end of file diff --git a/src/util/tests/stringTest.cpp b/src/util/tests/stringTest.cpp new file mode 100644 index 0000000000..b1b06c7f49 --- /dev/null +++ b/src/util/tests/stringTest.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +#include "taos.h" +#include "tutil.h" + +TEST(testCase, string_dequote_test) { + char t1[] = "'abc'"; + int32_t len = strdequote(t1); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t2[] = "\"abc\""; + len = strdequote(t2); + + EXPECT_EQ(3, len); + EXPECT_STRCASEEQ(t1, "abc"); + + char t21[] = " abc "; + strtrim(t21); + + EXPECT_STREQ("abc", t21); + EXPECT_EQ(3, strlen(t21)); +} + +TEST(testCase, string_replace_test) { + char t3[] = "abc01abc02abc"; + char* ret = strreplace(t3, "abc", "7"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("7017027", ret); + free(ret); + + char t4[] = "a01a02b03c04d05"; + ret = strreplace(t4, "0", "9999999999"); + + EXPECT_EQ(strlen(ret), 5 * 10 + 10); + EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret); + free(ret); + + char t5[] = "abc"; + ret = strreplace(t5, "abc", "12345678901234567890"); + + EXPECT_EQ(strlen(ret), 20); + EXPECT_STREQ("12345678901234567890", ret); + free(ret); + + char t6[] = "abc"; + ret = strreplace(t6, "def", "abc"); + + EXPECT_EQ(strlen(ret), 3); + EXPECT_STREQ("abc", ret); + free(ret); + + char t7[] = "abcde000000000000001234"; + ret = strreplace(t7, "ab", "0000000"); + + EXPECT_EQ(strlen(ret), 28); + EXPECT_STREQ("0000000cde000000000000001234", ret); + free(ret); + + char t8[] = "abc\ndef"; + char t[] = {10, 0}; + + char f1[] = "\\n"; + int32_t fx = strlen(f1); + ret = strreplace(t8, "\n", "\\n"); + + EXPECT_EQ(strlen(ret), 8); + EXPECT_STREQ("abc\\ndef", ret); + free(ret); + + char t9[] = "abc\\ndef"; + ret = strreplace(t9, "\\n", "\n"); + + EXPECT_EQ(strlen(ret), 7); + EXPECT_STREQ("abc\ndef", ret); + free(ret); + + char t10[] = "abcdef"; + ret = strreplace(t10, "", "0"); + + EXPECT_EQ(strlen(ret), 6); + EXPECT_STREQ("abcdef", ret); + free(ret); +} + +TEST(testCase, string_tolower_test) { + char t[1024] = {1}; + memset(t, 1, tListLen(t)); + + const char* a1 = "ABC"; + strtolower(t, a1); + EXPECT_STREQ(t, "abc"); + + memset(t, 1, tListLen(t)); + const char* a2 = "ABC\'ABC\'D"; + strtolower(t, a2); + EXPECT_STREQ(t, "abc\'ABC\'d"); + + memset(t, 1, tListLen(t)); + const char* a3 = ""; + strtolower(t, a3); + EXPECT_STREQ(t, ""); + + memset(t, 1, tListLen(t)); + const char* a4 = "\"AbcDEF\""; + strtolower(t, a4); + EXPECT_STREQ(t, a4); + + memset(t, 1, tListLen(t)); + const char* a5 = "1234\"AbcDEF\"456"; + strtolower(t, a5); + EXPECT_STREQ(t, a5); + + memset(t, 1, tListLen(t)); + const char* a6 = "1234"; + strtolower(t, a6); + EXPECT_STREQ(t, a6); +} + +TEST(testCase, string_strnchr_test) { + char t[1024] = {0}; + memset(t, 1, tListLen(t)); + + char a1[] = "AB.C"; + EXPECT_TRUE(strnchr(a1, '.', strlen(a1), true) != NULL); + + char a2[] = "abc."; + EXPECT_TRUE(strnchr(a2, '.', strlen(a2), true) != NULL); + + char a8[] = "abc."; + EXPECT_TRUE(strnchr(a8, '.', 1, true) == NULL); + + char a3[] = ".abc"; + EXPECT_TRUE(strnchr(a3, '.', strlen(a3), true) != NULL); + + char a4[] = "'.abc'"; + EXPECT_TRUE(strnchr(a4, '.', strlen(a4), true) == NULL); + + char a5[] = "'.abc.'abc"; + EXPECT_TRUE(strnchr(a5, '.', strlen(a5), true) == NULL); + + char a6[] = "0123456789."; + EXPECT_TRUE(strnchr(a6, '.', strlen(a6), true) != NULL); + + char a7[] = "0123456789."; + EXPECT_TRUE(strnchr(a7, '.', 3, true) == NULL); + + char a9[] = "0123456789."; + EXPECT_TRUE(strnchr(a9, '.', 0, true) == NULL); + + char a10[] = "0123456789'.'"; + EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL); +} + +TEST(testCase, cache_resize_test) { + char a11[] = "abc'.'"; + EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL); + + char a12[] = "abc'-'"; + EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL); + + char a15[] = "abc'-'"; + EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL); + + char a13[] = "'-'"; + EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL); + + char a14[] = "'-'"; + EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL); + + char a16[] = "'-'."; + EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL); +} \ No newline at end of file diff --git a/src/vnode/tsdb/tests/CMakeLists.txt b/src/vnode/tsdb/tests/CMakeLists.txt index 51c15bce20..ee1aaba8cd 100644 --- a/src/vnode/tsdb/tests/CMakeLists.txt +++ b/src/vnode/tsdb/tests/CMakeLists.txt @@ -3,9 +3,4 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) add_executable(tsdbTests ${SOURCE_LIST}) target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb) -add_test( - NAME - unit - COMMAND - ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests -) \ No newline at end of file +add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests) \ No newline at end of file From ffb02d974dbbb3c3fc95ab101aa7b99b19c6aab3 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 29 Mar 2020 15:01:00 +0800 Subject: [PATCH 72/85] TD-34 --- src/util/src/tlist.c | 9 +++--- src/vnode/tsdb/inc/tsdbFile.h | 11 +++++++ src/vnode/tsdb/src/tsdbFile.c | 3 +- src/vnode/tsdb/src/tsdbMain.c | 59 ++++++++++++++++++++++++++--------- 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index badcb7802f..aaedc76726 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -138,11 +138,10 @@ SListNode *tdListPopNode(SList *list, SListNode *node) { // Move all node elements from src to dst, the dst is assumed as an empty list void tdListMove(SList *src, SList *dst) { // assert(dst->eleSize == src->eleSize); - dst->numOfEles = src->numOfEles; - dst->head = src->head; - dst->tail = src->tail; - src->numOfEles = 0; - src->head = src->tail = NULL; + SListNode *node = NULL; + while ((node = tdListPopHead(src)) != NULL) { + tdListAppendNode(dst, node); + } } void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); } diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 7e358cc0a2..8c106d1067 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -122,6 +122,15 @@ typedef struct { } SCompInfo; #define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx)) +#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size)\ +do {\ + if (pCompBlock->numOfSubBlocks > 1) {\ + pCompBlock = pCompInfo->blocks + pCompBlock->offset;\ + size = pCompBlock->numOfSubBlocks;\ + } else {\ + size = 1;\ + }\ +} while (0) // TODO: take pre-calculation into account typedef struct { @@ -147,6 +156,8 @@ int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf); int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf); int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData); +SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); + // TODO: need an API to merge all sub-block data into one void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 9a0db96bd0..5240a99a37 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -35,7 +35,6 @@ static int compFGroup(const void *arg1, const void *arg2); static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname); static int tsdbWriteFileHead(SFile *pFile); static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); -static SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * maxFiles); @@ -309,7 +308,7 @@ void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t file *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; } -static SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { +SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) return NULL; void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 984f2ca203..769fc23815 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -90,6 +90,8 @@ static void * tsdbCommitData(void *arg); static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SDataCols *pCols); static int tsdbHasDataInRange(SSkipListIterator *pIter, TSKEY minKey, TSKEY maxKey); static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey); +static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len, + int64_t uid); #define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] #define TSDB_GET_TABLE_BY_NAME(pRepo, name) @@ -330,13 +332,13 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; pRepo->tsdbCache->mem = NULL; pRepo->tsdbCache->curBlock = NULL; + tsdbUnLockRepo(repo); // TODO: here should set as detached or use join for memory leak pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo); - tsdbUnLockRepo(repo); return 0; } @@ -814,7 +816,9 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) } if (!tSkipListIterNext(iters[tid])) { - assert(false); + // No data in this iterator + tSkipListDestroyIter(iters[tid]); + iters[tid] = NULL; } } @@ -839,8 +843,8 @@ static void *tsdbCommitData(void *arg) { } // Create a data column buffer for commit - SDataCols *pCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock); - if (pCols == NULL) { + SDataCols *pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock); + if (pDataCols == NULL) { // TODO: deal with the error return NULL; } @@ -849,16 +853,15 @@ static void *tsdbCommitData(void *arg) { int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, pCols) < 0) { + if (tsdbCommitToFile(pRepo, fid, iters, pDataCols) < 0) { // TODO: deal with the error here // assert(0); } } - tdFreeDataCols(pCols); + tdFreeDataCols(pDataCols); tsdbDestroyTableIters(iters, pCfg->maxTables); - tsdbLockRepo(arg); tdListMove(pCache->imem->list, pCache->pool.memPool); free(pCache->imem); @@ -918,25 +921,52 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters if (tsdbLoadCompIdx(pGroup, (void *)pIndices, pCfg->maxTables) < 0) { /* TODO */ } + lseek(hFile.fd, TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables, SEEK_SET); + // Loop to commit data in each table for (int tid = 0; tid < pCfg->maxTables; tid++) { STable * pTable = pMeta->tables[tid]; SSkipListIterator *pIter = iters[tid]; SCompIdx * pIdx = &pIndices[tid]; + if (pTable == NULL || pIter == NULL) continue; + + /* If no new data to write for this table, just write the old data to new file + * if there are. + */ if (!tsdbHasDataInRange(pIter, minKey, maxKey)) { - // This table does not have data in this range, just copy its head part and last - // part (if neccessary) to new file - if (pIdx->offset > 0) { // has old data + // has old data + if (pIdx->offset > 0) { if (isNewLastFile && pIdx->hasLast) { - // Need to load SCompBlock part and copy to new file + // need to move the last block to new file if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */ } if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ } - // TODO: Copy the last block from old last file to new file - // tsdbCopyBlockData() + tdInitDataCols(pCols, pTable->schema); + + SCompBlock *pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); + int nBlocks = 0; + + TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pTBlock, nBlocks); + + SCompBlock tBlock; + int64_t toffset, tlen; + tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_LAST], pTBlock, nBlocks, pCols, &tBlock); + + tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, tlen, pTable->tableId.uid); + pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks); + pTBlock->offset = toffset; + pTBlock->len = tlen; + pTBlock->numOfPoints = pCols->numOfPoints; + pTBlock->numOfSubBlocks = 1; + + pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); + if (nBlocks > 1) { + pIdx->len -= (sizeof(SCompBlock) * nBlocks); + } + write(hFile.fd, (void *)pCompInfo, pIdx->len); } else { pIdx->offset = lseek(hFile.fd, 0, SEEK_CUR); sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); @@ -951,6 +981,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters if (pIdx->offset > 0) { if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) { // has last block || cache key overlap with commit key + pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len + sizeof(SCompBlock) * 100); if (tsdbLoadCompBlocks(pGroup, pIdx, (void *)pCompInfo) < 0) { /* TODO */ } if (pCompInfo->uid == pTable->tableId.uid) isCompBlockLoaded = 1; @@ -984,7 +1015,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters // // SCompBlock *pTBlock = NULL; // } // } - pointsWritten = pCols->numOfPoints; + // pointsWritten = pCols->numOfPoints; tdPopDataColsPoints(pCols, pointsWritten); maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints; } From ab22c7f60671a80014868bad81d46ac9b6bb1a8f Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 29 Mar 2020 16:41:25 +0800 Subject: [PATCH 73/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 72 ++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 769fc23815..6abee55582 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -900,6 +900,9 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); if (!hasDataToCommit) return 0; // No data to commit, just return + // TODO: make it more flexible + pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + sizeof(SCompBlock) * 1000); + // Create and open files for commit tsdbGetDataDirName(pRepo, dataDir); if (tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables) < 0) { /* TODO */ @@ -908,7 +911,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters if (pGroup == NULL) { /* TODO */ } tsdbCreateFile(dataDir, fid, ".h", pCfg->maxTables, &hFile, 1, 0); - if (1 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { + if (0 /*pGroup->files[TSDB_FILE_TYPE_LAST].size > TSDB_MAX_LAST_FILE_SIZE*/) { // TODO: make it not to write the last file every time tsdbCreateFile(dataDir, fid, ".l", pCfg->maxTables, &lFile, 0, 0); isNewLastFile = 1; @@ -929,6 +932,8 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters SSkipListIterator *pIter = iters[tid]; SCompIdx * pIdx = &pIndices[tid]; + int nNewBlocks = 0; + if (pTable == NULL || pIter == NULL) continue; /* If no new data to write for this table, just write the old data to new file @@ -936,8 +941,10 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters */ if (!tsdbHasDataInRange(pIter, minKey, maxKey)) { // has old data - if (pIdx->offset > 0) { - if (isNewLastFile && pIdx->hasLast) { + if (pIdx->len > 0) { + goto _table_over; + // if (isNewLastFile && pIdx->hasLast) { + if (0) { // need to move the last block to new file if ((pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len)) == NULL) { /* TODO */ } @@ -976,9 +983,14 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters continue; } + pCompInfo->delimiter = TSDB_FILE_DELIMITER; + pCompInfo->checksum = 0; + pCompInfo->uid = pTable->tableId.uid; + // Load SCompBlock part if neccessary int isCompBlockLoaded = 0; - if (pIdx->offset > 0) { + if (0) { + // if (pIdx->offset > 0) { if (pIdx->hasLast || tsdbHasDataInRange(pIter, minKey, pIdx->maxKey)) { // has last block || cache key overlap with commit key pCompInfo = (SCompInfo *)realloc((void *)pCompInfo, pIdx->len + sizeof(SCompBlock) * 100); @@ -998,34 +1010,50 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters tsdbReadRowsFromCache(pIter, maxKey, maxRowsToRead, pCols); if (pCols->numOfPoints == 0) break; - int pointsWritten = 0; - // { // TODO : try to write the block data to file - // if (!isCompBlockLoaded) { // Just append - // if (pCols->numOfPoints > pCfg->minRowsPerFileBlock) { // write directly to .data file - // lseek(pGroup->files[TSDB_FILE_TYPE_DATA], 0, SEEK_END); + int pointsWritten = pCols->numOfPoints; + // TODO: all write to the end of .data file + int64_t toffset = 0; + int32_t tlen = 0; + tsdbWriteBlockToFileImpl(&pGroup->files[TSDB_FILE_TYPE_DATA], pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid); - // } else { - // if (isNewLastFile) { // write directly to .l file + // Make the compBlock + SCompBlock *pTBlock = pCompInfo->blocks + nNewBlocks++; + pTBlock->offset = toffset; + pTBlock->len = tlen; + pTBlock->keyFirst = dataColsKeyFirst(pCols); + pTBlock->keyLast = dataColsKeyLast(pCols); + pTBlock->last = 0; + pTBlock->algorithm = 0; + pTBlock->numOfPoints = pCols->numOfPoints; + pTBlock->sversion = pTable->sversion; + pTBlock->numOfSubBlocks = 1; - // } else { // write directly to .last file + if (dataColsKeyLast(pCols) > pIdx->maxKey) pIdx->maxKey = dataColsKeyLast(pCols); - // } - // } - // } else { // Need to append - // // SCompBlock *pTBlock = NULL; - // } - // } - // pointsWritten = pCols->numOfPoints; tdPopDataColsPoints(pCols, pointsWritten); maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pCols->numOfPoints; } + +_table_over: // Write the SCompBlock part - if (isCompBlockLoaded) { - // merge the block into old and update pIdx + pIdx->offset = lseek(hFile.fd, 0, SEEK_END); + if (pIdx->len > 0) { + sendfile(pGroup->files[TSDB_FILE_TYPE_HEAD].fd, hFile.fd, NULL, pIdx->len); + if (nNewBlocks > 0) { + write(hFile.fd, (void *)(pCompInfo->blocks), sizeof(SCompBlock) * nNewBlocks); + pIdx->len += (sizeof(SCompBlock) * nNewBlocks); + } } else { - // sendfile the SCompBlock part and update the pIdx + if (nNewBlocks > 0) { + write(hFile.fd, (void *)pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks); + pIdx->len += sizeof(SCompInfo) + sizeof(SCompBlock) * nNewBlocks; + } } + + pIdx->checksum = 0; + pIdx->numOfSuperBlocks += nNewBlocks; + pIdx->hasLast = 0; } // Write the SCompIdx part From 1bf6d52355feffc8a5461de12f64204b5629417a Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 29 Mar 2020 16:56:49 +0800 Subject: [PATCH 74/85] [TD-32] fix bugs in check data in file block --- src/vnode/tsdb/inc/tsdbFile.h | 3 + src/vnode/tsdb/src/tsdbMain.c | 5 + src/vnode/tsdb/src/tsdbRead.c | 374 ++++++++++++++++++++++++++++++++-- 3 files changed, 369 insertions(+), 13 deletions(-) diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 8c106d1067..0c85c5ef46 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -20,6 +20,7 @@ #include "dataformat.h" #include "taosdef.h" #include "tglobalcfg.h" +#include "tsdb.h" #ifdef __cplusplus extern "C" { @@ -148,6 +149,8 @@ typedef struct { SCompCol cols[]; } SCompData; +STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo); + int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols); int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables); diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 769fc23815..2b1424f88a 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -551,6 +551,11 @@ STsdbMeta* tsdbGetMeta(tsdb_repo_t* pRepo) { return tsdb->tsdbMeta; } +STsdbFileH* tsdbGetFile(tsdb_repo_t* pRepo) { + STsdbRepo* tsdb = (STsdbRepo*) pRepo; + return tsdb->tsdbFileH; +} + // Check the configuration and set default options static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // Check precision diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 7b996c8f7a..3ed6e22ad8 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#include #include "os.h" + +#include "tlog.h" #include "tutil.h" #include "../../../query/inc/qast.h" @@ -28,6 +29,11 @@ #define QUERY_IS_ASC_QUERY(o) (o == TSQL_SO_ASC) #define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns)) +enum { + QUERY_RANGE_LESS_EQUAL = 0, + QUERY_RANGE_GREATER_EQUAL = 1, +}; + typedef struct SField { // todo need the definition } SField; @@ -36,12 +42,12 @@ typedef struct SHeaderFileInfo { int32_t fileId; } SHeaderFileInfo; -typedef struct SQueryHandlePos { - int32_t fileId; +typedef struct SQueryFilePos { + int32_t fid; int32_t slot; int32_t pos; - int32_t fileIndex; -} SQueryHandlePos; + int64_t lastKey; +} SQueryFilePos; typedef struct SDataBlockLoadInfo { int32_t fileListIndex; @@ -78,8 +84,12 @@ typedef struct STableCheckInfo { TSKEY lastKey; STable * pTableObj; int64_t offsetInHeaderFile; - int32_t numOfBlocks; +// int32_t numOfBlocks; int32_t start; + bool checkFirstFileBlock; + + SCompIdx* compIndex; + SCompBlock *pBlock; SSkipListIterator* iter; } STableCheckInfo; @@ -104,8 +114,8 @@ enum { typedef struct STsdbQueryHandle { struct STsdbRepo* pTsdb; int8_t model; // access model, single table model or multi-table model - SQueryHandlePos cur; // current position - SQueryHandlePos start; // the start position, used for secondary/third iteration + SQueryFilePos cur; // current position + SQueryFilePos start; // the start position, used for secondary/third iteration int32_t unzipBufSize; char *unzipBuffer; char *secondaryUnzipBuffer; @@ -342,6 +352,344 @@ static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) { return true; } +// todo dynamic get the daysperfile +static int32_t getFileIdFromKey(TSKEY key) { + return (int32_t)(key / 10); // set the starting fileId +} + +static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGroup) { + tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables + SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; + + if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + + } else { + tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pBlock); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t binarySearchForBlockImpl(SCompBlock *pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { + int32_t firstSlot = 0; + int32_t lastSlot = numOfBlocks - 1; + + int32_t midSlot = firstSlot; + + while (1) { + numOfBlocks = lastSlot - firstSlot + 1; + midSlot = (firstSlot + (numOfBlocks >> 1)); + + if (numOfBlocks == 1) break; + + if (skey > pBlock[midSlot].keyLast) { + if (numOfBlocks == 2) break; + if ((order == TSQL_SO_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + firstSlot = midSlot + 1; + } else if (skey < pBlock[midSlot].keyFirst) { + if ((order == TSQL_SO_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + lastSlot = midSlot - 1; + } else { + break; // got the slot + } + } + + return midSlot; +} + +static SDataBlockInfo getTrueBlockInfo(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { + SDataBlockInfo info = {{0}, 0}; + + SCompBlock *pDiskBlock = &pCheckInfo->pBlock[pHandle->cur.slot]; + + info.window.skey = pDiskBlock->keyFirst; + info.window.ekey = pDiskBlock->keyLast; + info.size = pDiskBlock->numOfPoints; + info.numOfCols = pDiskBlock->numOfCols; + + return info; +} + +bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { + SQueryFilePos *cur = &pQueryHandle->cur; + + if (pQueryHandle->cur.fid >= 0) { + int32_t fileIndex = -1; + + /* + * 1. ascending order. The last data block of data file + * 2. descending order. The first block of file + */ + if ((step == QUERY_ASC_FORWARD_STEP && (pQueryHandle->cur.slot == pQueryHandle->numOfBlocks - 1)) || + (step == QUERY_DESC_FORWARD_STEP && (pQueryHandle->cur.slot == 0))) { + // temporarily keep the position value, in case of no data qualified when move forwards(backwards) + SQueryFilePos save = pQueryHandle->cur; + +// fileIndex = getNextDataFileCompInfo_(pQueryHandle, &pQueryHandle->cur, &pQueryHandle->vnodeFileInfo, step); + + // first data block in the next file + if (fileIndex >= 0) { + cur->slot = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->numOfBlocks - 1; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pQueryHandle->pBlock[cur->slot].numOfPoints - 1; +// return loadQaulifiedData(pQueryHandle); + } else {// try data in cache + assert(cur->fid == -1); + + if (step == QUERY_ASC_FORWARD_STEP) { +// TSKEY nextTimestamp = +// getQueryStartPositionInCache_rv(pQueryHandle, &pQueryHandle->cur.slot, &pQueryHandle->cur.pos, true); +// if (nextTimestamp < 0) { +// pQueryHandle->cur = save; +// } + +// return (nextTimestamp > 0); + } + + // no data to check for desc order query, restore the saved position value + pQueryHandle->cur = save; + return false; + } + } + + // next block in the same file + int32_t fid = cur->fid; +// fileIndex = vnodeGetVnodeHeaderFileIndex(&fid, pQueryHandle->order, &pQueryHandle->vnodeFileInfo); + cur->slot += step; + + SCompBlock *pBlock = &pQueryHandle->pBlock[cur->slot]; + cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; +// return loadQaulifiedData(pQueryHandle); + } else { // data in cache +// todo continue; + } +} + + +int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { + int firstPos, lastPos, midPos = -1; + int numOfPoints; + TSKEY *keyList; + + if (num <= 0) return -1; + + keyList = (TSKEY *)pValue; + firstPos = 0; + lastPos = num - 1; + + if (order == 0) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfPoints = lastPos - firstPos + 1; + midPos = (numOfPoints >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { + // only return the qualified data to client in terms of query time window, data rows in the same block but do not + // be included in the query time window will be discarded + SQueryFilePos *cur = &pQueryHandle->cur; + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + SDataBlockInfo blockInfo = getTrueBlockInfo(pQueryHandle, pCheckInfo); + + int32_t endPos = cur->pos; + if (QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { + endPos = blockInfo.size - 1; + pQueryHandle->realNumOfRows = endPos - cur->pos + 1; + } else if (!QUERY_IS_ASC_QUERY(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) { + endPos = 0; + pQueryHandle->realNumOfRows = cur->pos + 1; + } else { +// endPos = vnodeBinarySearchKey(pQueryHandle->tsBuf->data, blockInfo.size, pQueryHandle->window.ekey, pQueryHandle->order); + + if (QUERY_IS_ASC_QUERY(pQueryHandle->order)) { + if (endPos < cur->pos) { + pQueryHandle->realNumOfRows = 0; + return; + } else { + pQueryHandle->realNumOfRows = endPos - cur->pos; + } + } else { + if (endPos > cur->pos) { + pQueryHandle->realNumOfRows = 0; + return; + } else { + pQueryHandle->realNumOfRows = cur->pos - endPos; + } + } + } + + int32_t start = MIN(cur->pos, endPos); + + // move the data block in the front to data block if needed + if (start != 0) { + int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + + for (int32_t i = 0; i < taosArrayGetSize(sa); ++i) { + int16_t colId = *(int16_t *)taosArrayGet(sa, i); + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoEx *pCol = taosArrayGet(pQueryHandle->pColumns, j); + + if (pCol->info.colId == colId) { + memmove(pCol->pData, ((char *)pCol->pData) + pCol->info.bytes * start, pQueryHandle->realNumOfRows * pCol->info.bytes); + break; + } + } + } + } + + + + assert(pQueryHandle->realNumOfRows <= blockInfo.size); + + // forward(backward) the position for cursor + cur->pos = endPos; +} + +static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInfo* pCheckInfo, int32_t type) { + STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); + int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); + + SFileGroup* fileGroup = tsdbSearchFGroup(pFileHandle, fid); + pCheckInfo->checkFirstFileBlock = true; + + SQueryFilePos* cur = &pQueryHandle->cur; + + TSKEY key = pCheckInfo->lastKey; + int32_t index = -1; + + // todo add iterator for filegroup + while (1) { + if ((fid = getFileCompInfo(pCheckInfo, fileGroup)) < 0) { + break; + } + + int32_t tid = pCheckInfo->tableId.tid; + index = binarySearchForBlockImpl(pCheckInfo->pBlock, pCheckInfo->compIndex[tid].numOfSuperBlocks, pQueryHandle->order, key); + + if (type == QUERY_RANGE_GREATER_EQUAL) { + if (key <= pCheckInfo->pBlock[index].keyLast) { + break; + } else { + index = -1; + } + } else { + if (key >= pCheckInfo->pBlock[index].keyFirst) { + break; + } else { + index = -1; + } + } + } + + // failed to find qualified point in file, abort + if (index == -1) { + return false; + } + + assert(index >= 0 && index < pQueryHandle->numOfBlocks); + + // load first data block into memory failed, caused by disk block error + bool blockLoaded = false; + SArray *sa = NULL; + + // todo no need to loaded at all + cur->slot = index; + +// sa = getDefaultLoadColumns(pQueryHandle, true); + if (tsdbLoadDataBlock(&fileGroup->files[2], &pCheckInfo->pBlock[cur->slot], 1, fid, sa) == 0) { + blockLoaded = true; + } + + // dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files", + // GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx); + + // failed to load data from disk, abort current query + if (blockLoaded == false) { + return false; + } + + // todo search qualified points in blk, according to primary key (timestamp) column +// cur->pos = binarySearchForBlockImpl(ptsBuf->data, pBlocks->numOfPoints, key, pQueryHandle->order); + assert(cur->pos >= 0 && cur->fid >= 0 && cur->slot >= 0); + + filterDataInDataBlock(pQueryHandle, sa); + return pQueryHandle->realNumOfRows > 0; +} + +static bool hasMoreDataInFileForSingleTableModel(STsdbQueryHandle* pHandle) { + assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); + SQueryFilePos* cur = &pHandle->cur; + + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); + + if (!pCheckInfo->checkFirstFileBlock && pFileHandle != NULL) { + int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); + SFileGroup* fileGroup = tsdbSearchFGroup(pFileHandle, fid); + pCheckInfo->checkFirstFileBlock = true; + + if (fileGroup != NULL) { + return getQualifiedDataBlock(pHandle, pCheckInfo, 1); + } else { // no data in file, try cache + return hasMoreDataInCacheForSingleModel(pHandle); + } + } else { + pCheckInfo->checkFirstFileBlock = true; + if (pFileHandle == NULL) { + cur->fid = -1; + } + + if (cur->fid == -1 || pFileHandle != NULL) { // try data in cache + return hasMoreDataInCacheForSingleModel(pHandle); + } else { + return true; + } + } + +} + static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo); assert(numOfTables > 0); @@ -372,7 +720,7 @@ static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; if (pHandle->model == SINGLE_TABLE_MODEL) { - return hasMoreDataInCacheForSingleModel(pHandle); + return hasMoreDataInFileForSingleTableModel(pHandle); } else { return hasMoreDataInCacheForMultiModel(pHandle); } @@ -704,8 +1052,6 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char buf[TSDB_MAX_TAGS_LEN] = {0}; - char* val = dataRowTuple(pTable->tagVal); // todo not only the first column int8_t type = pInfo->sch.type; @@ -765,9 +1111,11 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond // query according to the binary expression SSyntaxTreeFilterSupporter s = {.pTagSchema = stcol, .numOfTags = schemaNCols(pSTable->tagSchema)}; - SBinaryFilterSupp supp = {.fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, + SBinaryFilterSupp supp = { + .fp = (__result_filter_fn_t)tSkipListNodeFilterCallback, .setupInfoFn = (__do_filter_suppl_fn_t)filterPrepare, - .pExtInfo = &s}; + .pExtInfo = &s + }; tSQLBinaryExprTraverse(pExpr, pSTable->pIndex, pRes, &supp); tSQLBinaryExprDestroy(&pExpr, tSQLListTraverseDestroyInfo); From 695b1a0fdcd9fe57971437a5d93e0703021626d8 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 29 Mar 2020 17:00:23 +0800 Subject: [PATCH 75/85] [TD-32] merge 2.0 branch --- tests/examples/c/demo.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 6fcedb8123..b621781a3c 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -46,7 +46,8 @@ int main(int argc, char *argv[]) { } printf("success to connect to server\n"); - int32_t code = taos_query(taos, "select * from test.t1"); +// int32_t code = taos_query(taos, "insert into test.tm2 values(now, 1)(now+1m,2)(now+2m,3) (now+3m, 4) (now+4m, 5);"); + int32_t code = taos_query(taos, "insert into test.tm2 values(now, 99)"); if (code != 0) { printf("failed to execute query, reason:%s\n", taos_errstr(taos)); } From a944461d573de8dd7a34eefc6a33063da5eb77d7 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 29 Mar 2020 17:17:57 +0800 Subject: [PATCH 76/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 6abee55582..c45a8407cc 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -345,12 +345,12 @@ int32_t tsdbTriggerCommit(tsdb_repo_t *repo) { int32_t tsdbLockRepo(tsdb_repo_t *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; - return pthread_mutex_lock(repo); + return pthread_mutex_lock(&(pRepo->mutex)); } int32_t tsdbUnLockRepo(tsdb_repo_t *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; - return pthread_mutex_unlock(repo); + return pthread_mutex_unlock(&(pRepo->mutex)); } /** From 8339f9c864b5858ce58d96abeeffcfc234b6b9ca Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 29 Mar 2020 19:19:51 +0800 Subject: [PATCH 77/85] TD-34 --- src/util/src/tlist.c | 4 ++++ src/util/tests/stringTest.cpp | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index aaedc76726..bdb12a59f9 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -100,6 +100,8 @@ SListNode *tdListPopHead(SList *list) { list->head = node->next; } list->numOfEles--; + node->next = NULL; + node->prev = NULL; return node; } @@ -113,6 +115,7 @@ SListNode *tdListPopTail(SList *list) { list->tail = node->prev; } list->numOfEles--; + node->next = node->prev = NULL; return node; } @@ -131,6 +134,7 @@ SListNode *tdListPopNode(SList *list, SListNode *node) { node->next->prev = node->prev; } list->numOfEles--; + node->next = node->prev = NULL; return node; } diff --git a/src/util/tests/stringTest.cpp b/src/util/tests/stringTest.cpp index b1b06c7f49..ef8df90e3e 100644 --- a/src/util/tests/stringTest.cpp +++ b/src/util/tests/stringTest.cpp @@ -157,22 +157,22 @@ TEST(testCase, string_strnchr_test) { EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL); } -TEST(testCase, cache_resize_test) { - char a11[] = "abc'.'"; - EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL); +// TEST(testCase, cache_resize_test) { +// char a11[] = "abc'.'"; +// EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL); - char a12[] = "abc'-'"; - EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL); +// char a12[] = "abc'-'"; +// EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL); - char a15[] = "abc'-'"; - EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL); +// char a15[] = "abc'-'"; +// EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL); - char a13[] = "'-'"; - EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL); +// char a13[] = "'-'"; +// EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL); - char a14[] = "'-'"; - EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL); +// char a14[] = "'-'"; +// EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL); - char a16[] = "'-'."; - EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL); -} \ No newline at end of file +// char a16[] = "'-'."; +// EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL); +// } \ No newline at end of file From e29ee2a2115fb4c9e33dd70185372c27e8344528 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 29 Mar 2020 21:54:23 +0800 Subject: [PATCH 78/85] TD-34 --- src/vnode/tsdb/src/tsdbCache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vnode/tsdb/src/tsdbCache.c b/src/vnode/tsdb/src/tsdbCache.c index 3496f6a5c8..d64b98d49d 100644 --- a/src/vnode/tsdb/src/tsdbCache.c +++ b/src/vnode/tsdb/src/tsdbCache.c @@ -75,8 +75,10 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { if (pCache->curBlock !=NULL && (pCache->mem->list) >= pCache->totalCacheBlocks/2) { tsdbTriggerCommit(pCache->pRepo); } - if (tsdbAllocBlockFromPool(pCache) < 0) { + + while (tsdbAllocBlockFromPool(pCache) < 0) { // TODO: deal with the error + // printf("Failed to allocate from cache pool\n"); } } From c94331fc6bf3232a20471fa9a53b9dbb19342e36 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Mon, 30 Mar 2020 13:29:36 +0800 Subject: [PATCH 79/85] TD-34 --- src/vnode/tsdb/src/tsdbMain.c | 23 ++++++++++++++++++++++- src/vnode/tsdb/tests/tsdbTests.cpp | 11 ++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index c45a8407cc..0a681a8237 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -287,8 +287,29 @@ int32_t tsdbCloseRepo(tsdb_repo_t *repo) { if (pRepo == NULL) return 0; pRepo->state = TSDB_REPO_STATE_CLOSED; + tsdbLockRepo(repo); + if (pRepo->commit) { + tsdbUnLockRepo(repo); + return -1; + } + pRepo->commit = 1; + // Loop to move pData to iData + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable *pTable = pRepo->tsdbMeta->tables[i]; + if (pTable != NULL && pTable->mem != NULL) { + pTable->imem = pTable->mem; + pTable->mem = NULL; + } + } + // TODO: Loop to move mem to imem + pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; + pRepo->tsdbCache->mem = NULL; + pRepo->tsdbCache->curBlock = NULL; + tsdbUnLockRepo(repo); - tsdbFlushCache(pRepo); + tsdbCommitData((void *)repo); + + tsdbCloseFileH(pRepo->tsdbFileH); tsdbFreeMeta(pRepo->tsdbMeta); diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index bc6532984f..73caeb6700 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -78,7 +78,7 @@ TEST(TsdbTest, createRepo) { tsdbCreateTable(pRepo, &tCfg); // // 3. Loop to write some simple data - int nRows = 10000000; + int nRows = 1000000; int rowsPerSubmit = 10; int64_t start_time = 1584081000000; @@ -87,6 +87,7 @@ TEST(TsdbTest, createRepo) { double stime = getCurTime(); for (int k = 0; k < nRows/rowsPerSubmit; k++) { + memset((void *)pMsg, 0, sizeof(SSubmitMsg)); SSubmitBlk *pBlock = pMsg->blocks; pBlock->uid = 987607499877672L; pBlock->tid = 0; @@ -108,6 +109,9 @@ TEST(TsdbTest, createRepo) { } pBlock->len += dataRowLen(row); } + pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; + pMsg->numOfBlocks = 1; + pBlock->len = htonl(pBlock->len); pBlock->numOfRows = htonl(pBlock->numOfRows); pBlock->uid = htobe64(pBlock->uid); @@ -116,7 +120,6 @@ TEST(TsdbTest, createRepo) { pBlock->sversion = htonl(pBlock->sversion); pBlock->padding = htonl(pBlock->padding); - pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len; pMsg->length = htonl(pMsg->length); pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); pMsg->compressed = htonl(pMsg->numOfBlocks); @@ -128,9 +131,7 @@ TEST(TsdbTest, createRepo) { printf("Spent %f seconds to write %d records\n", etime - stime, nRows); - - - // tsdbTriggerCommit(pRepo); + tsdbCloseRepo(pRepo); } From 5b10eda69880bfc8272a97f2f2257db571c85a43 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Mon, 30 Mar 2020 14:26:38 +0800 Subject: [PATCH 80/85] TD-34 --- src/vnode/tsdb/src/tsdbFile.c | 36 ++++++++++++++++++++++++++++-- src/vnode/tsdb/src/tsdbMain.c | 11 +++++++++ src/vnode/tsdb/tests/tsdbTests.cpp | 11 ++++++--- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index 5240a99a37..bfdd998c22 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -35,6 +35,7 @@ static int compFGroup(const void *arg1, const void *arg2); static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname); static int tsdbWriteFileHead(SFile *pFile); static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables); +static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH) + sizeof(SFileGroup) * maxFiles); @@ -50,10 +51,17 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { return NULL; } - struct dirent *dp; + struct dirent *dp = NULL; + int fid = 0; + SFileGroup fGroup = {0}; while ((dp = readdir(dir)) != NULL) { if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 1) == 0) continue; - // TODO + int fid = 0; + sscanf(dp->d_name, "f%d", &fid); + if (tsdbOpenFGroup(pFileH, dataDir, fid) < 0) { + break; + // TODO + } } return pFileH; @@ -61,6 +69,30 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) { void tsdbCloseFileH(STsdbFileH *pFileH) { free(pFileH); } +static int tsdbInitFile(char *dataDir, int fid, char *suffix, SFile *pFile) { + tsdbGetFileName(dataDir, fid, suffix, pFile->fname); + if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1; + pFile->fd = -1; + // TODO: recover the file info + // pFile->info = {0}; + return 0; +} + +static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { + if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; + + char fname[128] = "\0"; + SFileGroup fGroup = {0}; + fGroup.fileId = fid; + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; + } + pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; + qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); + return 0; +} + int tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { if (pFileH->numOfFGroups >= pFileH->maxFGroups) return -1; diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index 0a681a8237..97c98efa05 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -237,6 +237,7 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo) { * @return a TSDB repository handle on success, NULL for failure and the error number is set */ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { + char dataDir[128] = "\0"; if (access(tsdbDir, F_OK | W_OK | R_OK) < 0) { return NULL; } @@ -265,6 +266,16 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) { return NULL; } + tsdbGetDataDirName(pRepo, dataDir); + pRepo->tsdbFileH = tsdbInitFileH(dataDir, pRepo->config.maxTables); + if (pRepo->tsdbFileH == NULL) { + tsdbFreeCache(pRepo->tsdbCache); + tsdbFreeMeta(pRepo->tsdbMeta); + free(pRepo->rootDir); + free(pRepo); + return NULL; + } + pRepo->state = TSDB_REPO_STATE_ACTIVE; return (tsdb_repo_t *)pRepo; diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 73caeb6700..12311db68e 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -49,7 +49,8 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) { ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0); } -TEST(TsdbTest, createRepo) { +TEST(TsdbTest, DISABLED_createRepo) { +// TEST(TsdbTest, createRepo) { STsdbCfg config; // 1. Create a tsdb repository @@ -78,7 +79,7 @@ TEST(TsdbTest, createRepo) { tsdbCreateTable(pRepo, &tCfg); // // 3. Loop to write some simple data - int nRows = 1000000; + int nRows = 10000000; int rowsPerSubmit = 10; int64_t start_time = 1584081000000; @@ -129,13 +130,17 @@ TEST(TsdbTest, createRepo) { double etime = getCurTime(); + void *ptr = malloc(150000); + free(ptr); + printf("Spent %f seconds to write %d records\n", etime - stime, nRows); tsdbCloseRepo(pRepo); } -TEST(TsdbTest, DISABLED_openRepo) { +// TEST(TsdbTest, DISABLED_openRepo) { +TEST(TsdbTest, openRepo) { tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0"); ASSERT_NE(pRepo, nullptr); } From 18a63bd3f10cf6cd319977f004f07ef350683b52 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 30 Mar 2020 14:43:01 +0800 Subject: [PATCH 81/85] [TD-32] merge 2.0 branch --- src/util/inc/tskiplist.h | 42 +++-- src/util/src/tskiplist.c | 102 ++--------- src/util/tests/cacheTest.cpp | 4 +- src/util/tests/hashTest.cpp | 2 +- src/util/tests/skiplistTest.cpp | 314 ++++++++++++++++++++++++++++++++ src/vnode/tsdb/src/tsdbMain.c | 6 +- src/vnode/tsdb/src/tsdbMeta.c | 4 +- src/vnode/tsdb/src/tsdbRead.c | 47 ++--- 8 files changed, 379 insertions(+), 142 deletions(-) create mode 100644 src/util/tests/skiplistTest.cpp diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 0ec1d0eab5..18404f89c2 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -51,7 +51,7 @@ typedef struct SSkipListNode { #define SL_GET_BACKWARD_POINTER(n, _l) \ ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)] -#define SL_GET_NODE_DATA(n) ((char*)(n) + SL_NODE_HEADER_SIZE((n)->level)) +#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level)) #define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n))) #define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) @@ -106,25 +106,25 @@ typedef struct tSkipListState { typedef struct SSkipListKeyInfo { uint8_t dupKey : 2; // if allow duplicated key in the skip list - uint8_t type : 6; // key type + uint8_t type : 4; // key type + uint8_t freeNode:2; // free node when destroy the skiplist uint8_t len; // maximum key length, used in case of string key } SSkipListKeyInfo; typedef struct SSkipList { - __compar_fn_t comparFn; - __sl_key_fn_t keyFn; - uint32_t size; - uint8_t maxLevel; - uint8_t level; - SSkipListKeyInfo keyInfo; - + __compar_fn_t comparFn; + __sl_key_fn_t keyFn; + uint32_t size; + uint8_t maxLevel; + uint8_t level; + SSkipListKeyInfo keyInfo; pthread_rwlock_t *lock; - SSkipListNode * pHead; - + SSkipListNode * pHead; // point to the first element + SSkipListNode * pTail; // point to the last element + void * lastKey; // last key in the skiplist #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif - } SSkipList; /* @@ -147,7 +147,7 @@ typedef struct SSkipListIterator { * @return */ SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe, - __sl_key_fn_t fn); + uint8_t freeNode, __sl_key_fn_t fn); /** * @@ -182,21 +182,28 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); * @param keyType * @return */ -SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType); +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType); /** * get the size of skip list * @param pSkipList * @return */ -size_t tSkipListGetSize(const SSkipList* pSkipList); +size_t tSkipListGetSize(const SSkipList *pSkipList); + +/** + * display skip list of the given level, for debug purpose only + * @param pSkipList + * @param nlevel + */ +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); /** * create skiplist iterator * @param pSkipList * @return */ -SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList); +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); /** * forward the skip list iterator @@ -217,7 +224,7 @@ SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); * @param iter * @return */ -void* tSkipListDestroyIter(SSkipListIterator* iter); +void *tSkipListDestroyIter(SSkipListIterator *iter); /* * remove only one node of the pKey value. @@ -234,7 +241,6 @@ bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey); */ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); - #ifdef __cplusplus } #endif diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index c46f45fd37..1760919b05 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -281,7 +281,7 @@ static __compar_fn_t getKeyComparator(int32_t keyType) { } SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock, - __sl_key_fn_t fn) { + uint8_t freeNode, __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); if (pSkipList == NULL) { return NULL; @@ -291,9 +291,8 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui maxLevel = MAX_SKIP_LIST_LEVEL; } - pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey}; + pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparator(keyType); pSkipList->maxLevel = maxLevel; pSkipList->level = 1; @@ -348,12 +347,15 @@ void *tSkipListDestroy(SSkipList *pSkipList) { pthread_rwlock_wrlock(pSkipList->lock); } - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); // pSkipList->pHead.pForward[0]; + SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); while (pNode) { SSkipListNode *pTemp = pNode; pNode = SL_GET_FORWARD_POINTER(pNode, 0); - tfree(pTemp); + + if (pSkipList->keyInfo.freeNode) { + tfree(pTemp); + } } tfree(pSkipList->pHead); @@ -435,7 +437,11 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { recordNodeEachLevel(pSkipList, level); #endif + // clear pointer area int32_t level = SL_GET_NODE_LEVEL(pNode); + memset(pNode, 0, SL_NODE_HEADER_SIZE(pNode->level)); + pNode->level = level; + tSkipListDoInsert(pSkipList, forward, level, pNode); atomic_add_fetch_32(&pSkipList->size, 1); @@ -691,89 +697,6 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // return NULL; //} // -// int32_t tSkipListIterateList(SSkipList *pSkipList, SSkipListNode ***pRes, bool (*fp)(SSkipListNode *, void *), -// void *param) { -// (*pRes) = (SSkipListNode **)calloc(1, POINTER_BYTES * pSkipList->nSize); -// if (NULL == *pRes) { -// pError("error skiplist %p, malloc failed", pSkipList); -// return -1; -// } -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// SSkipListNode *pStartNode = pSkipList->pHead.pForward[0]; -// int32_t num = 0; -// -// for (int32_t i = 0; i < pSkipList->nSize; ++i) { -// if (pStartNode == NULL) { -// pError("error skiplist %p, required length:%d, actual length:%d", pSkipList, pSkipList->nSize, i - 1); -//#ifdef _DEBUG_VIEW -// tSkipListPrint(pSkipList, 1); -//#endif -// break; -// } -// -// if (fp == NULL || (fp != NULL && fp(pStartNode, param) == true)) { -// (*pRes)[num++] = pStartNode; -// } -// -// pStartNode = pStartNode->pForward[0]; -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// -// if (num == 0) { -// free(*pRes); -// *pRes = NULL; -// } else if (num < pSkipList->nSize) { // free unused memory -// char *tmp = realloc((*pRes), num * POINTER_BYTES); -// assert(tmp != NULL); -// -// *pRes = (SSkipListNode **)tmp; -// } -// -// return num; -//} -// -// int32_t tSkipListIteratorReset(SSkipList *pSkipList, SSkipListIterator *iter) { -// if (pSkipList == NULL) { -// return -1; -// } -// -// iter->pSkipList = pSkipList; -// if (pSkipList->lock) { -// pthread_rwlock_rdlock(&pSkipList->lock); -// } -// iter->cur = NULL; // pSkipList->pHead.pForward[0]; -// iter->num = pSkipList->size; -// -// if (pSkipList->lock) { -// pthread_rwlock_unlock(&pSkipList->lock); -// } -// -// return 0; -//} -// -// bool tSkipListIteratorNext(SSkipListIterator *iter) { -// if (iter->num == 0 || iter->pSkipList == NULL) { -// return false; -// } -// -// SSkipList *pSkipList = iter->pSkipList; -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// if (iter->cur == NULL) { -// iter->cur = pSkipList->pHead.pForward[0]; -// } else { -// iter->cur = iter->cur->pForward[0]; -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// -// return iter->cur != NULL; -//} -// -// SSkipListNode *tSkipListIteratorGet(SSkipListIterator *iter) { return iter->cur; } -// // int32_t tSkipListRangeQuery(SSkipList *pSkipList, tSKipListQueryCond *pCond, SSkipListNode ***pRes) { // pSkipList->state.queryCount++; // SSkipListNode *pStart = tSkipListParQuery(pSkipList, &pCond->lowerBnd, pCond->lowerBndRelOptr); @@ -841,7 +764,8 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); - int32_t id = 1; + + int32_t id = 1; while (p) { char *key = SL_GET_NODE_KEY(pSkipList, p); diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 411c899cc0..1902fef4b6 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -125,7 +125,7 @@ TEST(testCase, cache_resize_test) { } uint64_t endTime = taosGetTimestampUs(); - printf("add 10,000,000 object cost:%lld us, avg:%f us\n", endTime - startTime, (endTime-startTime)/(double)num); + printf("add %d object cost:%lld us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num); startTime = taosGetTimestampUs(); for(int32_t i = 0; i < num; ++i) { @@ -134,7 +134,7 @@ TEST(testCase, cache_resize_test) { assert(k != 0); } endTime = taosGetTimestampUs(); - printf("retrieve 10,000,000 object cost:%lld us,avg:%f\n", endTime - startTime, (endTime - startTime)/(double)num); + printf("retrieve %d object cost:%lld us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num); taosCacheCleanup(pCache); taosMsleep(20000); diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp index 93a1989741..95357886d7 100644 --- a/src/util/tests/hashTest.cpp +++ b/src/util/tests/hashTest.cpp @@ -151,6 +151,6 @@ int main(int argc, char** argv) { TEST(testCase, hashTest) { simpleTest(); stringKeyTest(); - noLockPerformanceTest(); +// noLockPerformanceTest(); multithreadsTest(); } \ No newline at end of file diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp new file mode 100644 index 0000000000..7a3e87d786 --- /dev/null +++ b/src/util/tests/skiplistTest.cpp @@ -0,0 +1,314 @@ +#include +#include +#include +#include + +#include "taosmsg.h" +#include "tskiplist.h" +#include "ttime.h" +#include "tutil.h" + +namespace { + +char* getkey(const void* data) { return (char*)(data); } + +void doubleSkipListTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, true, getkey); + + double doubleVal[1000] = {0}; + int32_t size = 20000; + + printf("generated %d keys is: \n", size); + + for (int32_t i = 0; i < size; ++i) { + if (i < 1000) { + doubleVal[i] = i * 0.997; + } + + int32_t level = 0; + int32_t size = 0; + + tSkipListRandNodeInfo(pSkipList, &level, &size); + auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + d->level = level; + + double* key = (double*)SL_GET_NODE_KEY(pSkipList, d); + key[0] = i * 0.997; + key[1] = i * 0.997; + + tSkipListPut(pSkipList, d); + } + + printf("the first level of skip list is:\n"); + tSkipListPrint(pSkipList, 1); + +#if 0 + SSkipListNode **pNodes = NULL; + SSkipListKey sk; + for (int32_t i = 0; i < 100; ++i) { + sk.nType = TSDB_DATA_TYPE_DOUBLE; + int32_t idx = abs((i * rand()) % 1000); + + sk.dKey = doubleVal[idx]; + + int32_t size = tSkipListGets(pSkipList, &sk, &pNodes); + + printf("the query result size is: %d\n", size); + for (int32_t j = 0; j < size; ++j) { + printf("the result is: %lf\n", pNodes[j]->key.dKey); + } + + if (size > 0) { + tfree(pNodes); + } + } + +#endif + + printf("double test end...\n"); + tSkipListDestroy(pSkipList); +} + +void stringKeySkiplistTest() { + const int32_t max_key_size = 12; + + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, max_key_size, 0, false, true, getkey); + + int32_t level = 0; + int32_t headsize = 0; + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + auto pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode->level = level; + + char* d = SL_GET_NODE_DATA(pNode); + strncpy(d, "nyse", 5); + + *(double*)(d + max_key_size) = 12; + + tSkipListPut(pSkipList, pNode); + + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode->level = level; + + d = SL_GET_NODE_DATA(pNode); + strncpy(d, "beijing", 8); + + *(double*)(d + max_key_size) = 911; + + tSkipListPut(pSkipList, pNode); + +#if 0 + SSkipListNode **pRes = NULL; + int32_t ret = tSkipListGets(pSkipList, &key1, &pRes); + + assert(ret == 1); + assert(strcmp(pRes[0]->key.pz, "beijing") == 0); + assert(pRes[0]->key.nType == TSDB_DATA_TYPE_BINARY); + + tSkipListDestroyKey(&key1); + tSkipListDestroyKey(&key); + + tSkipListDestroy(pSkipList); + + free(pRes); +#endif + + tSkipListDestroy(pSkipList); + + int64_t s = taosGetTimestampUs(); + pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_BINARY, 20, 0, false, true, getkey); + char k[256] = {0}; + + int32_t total = 10000; + for (int32_t i = 0; i < total; ++i) { + int32_t n = sprintf(k, "abc_%d_%d", i, i); + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + auto pNode = (SSkipListNode*)calloc(1, headsize + 20 + sizeof(double)); + pNode->level = level; + + char* d = SL_GET_NODE_DATA(pNode); + strncpy(d, k, strlen(k)); + + tSkipListPut(pSkipList, pNode); + } + + int64_t e = taosGetTimestampUs(); + printf("elapsed time:%lld us to insert %d data, avg:%f us\n", (e - s), total, (double)(e - s) / total); + +#if 0 + SSkipListNode **pres = NULL; + + s = taosGetTimestampMs(); + for (int32_t j = 0; j < total; ++j) { + int32_t n = sprintf(k, "abc_%d_%d", j, j); + key = tSkipListCreateKey(TSDB_DATA_TYPE_BINARY, k, n); + + int32_t num = tSkipListGets(pSkipList, &key, &pres); + assert(num > 0); + + // tSkipListRemove(pSkipList, &key); + tSkipListRemoveNode(pSkipList, pres[0]); + + if (num > 0) { + tfree(pres); + } + } + + e = taosGetTimestampMs(); + printf("elapsed time:%lldms\n", e - s); +#endif + tSkipListDestroy(pSkipList); +} + +void skiplistPerformanceTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, false, getkey); + + int32_t size = 900000; + int64_t prev = taosGetTimestampMs(); + int64_t s = prev; + + int32_t level = 0; + int32_t headsize = 0; + + int32_t unit = MAX_SKIP_LIST_LEVEL * POINTER_BYTES * 2 + sizeof(double) * 2 + sizeof(int16_t); + + char* total = (char*)calloc(1, unit * size); + char* p = total; + + for (int32_t i = size; i > 0; --i) { + tSkipListRandNodeInfo(pSkipList, &level, &headsize); + + SSkipListNode* d = (SSkipListNode*)p; + p += headsize + sizeof(double) * 2; + + d->level = level; + double* v = (double*)SL_GET_NODE_DATA(d); + v[0] = i * 0.997; + v[1] = i * 0.997; + + tSkipListPut(pSkipList, d); + + if (i % 100000 == 0) { + int64_t cur = taosGetTimestampMs(); + + int64_t elapsed = cur - prev; + printf("add %d, elapsed time: %lld ms, avg elapsed:%f ms, total:%d\n", 100000, elapsed, elapsed / 100000.0, i); + prev = cur; + } + } + + int64_t e = taosGetTimestampMs(); + printf("total:%lld ms, avg:%f\n", e - s, (e - s) / (double)size); + printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level); + + assert(tSkipListGetSize(pSkipList) == size); + + printf("the level of skiplist is:\n"); + +// printf("level two------------------\n"); +// tSkipListPrint(pSkipList, 2); +// +// printf("level three------------------\n"); +// tSkipListPrint(pSkipList, 3); +// +// printf("level four------------------\n"); +// tSkipListPrint(pSkipList, 4); +// +// printf("level nine------------------\n"); +// tSkipListPrint(pSkipList, 10); + + int64_t st = taosGetTimestampMs(); +#if 0 + for (int32_t i = 0; i < 100000; i += 1) { + key.dKey = i * 0.997; + tSkipListRemove(pSkipList, &key); + } +#endif + + int64_t et = taosGetTimestampMs(); + printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st); + assert(tSkipListGetSize(pSkipList) == size); + + tSkipListDestroy(pSkipList); + tfree(total); +} + +// todo not support duplicated key yet +void duplicatedKeyTest() { +#if 0 + SSkipListKey key; + key.nType = TSDB_DATA_TYPE_INT; + + SSkipListNode **pNodes = NULL; + + SSkipList *pSkipList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_INT, sizeof(int)); + + for (int32_t i = 0; i < 10000; ++i) { + for (int32_t j = 0; j < 5; ++j) { + key.i64Key = i; + tSkipListPut(pSkipList, "", &key, 1); + } + } + + tSkipListPrint(pSkipList, 1); + + for (int32_t i = 0; i < 100; ++i) { + key.i64Key = rand() % 1000; + int32_t size = tSkipListGets(pSkipList, &key, &pNodes); + + assert(size == 5); + + tfree(pNodes); + } + + tSkipListDestroy(pSkipList); +#endif +} + +} // namespace + +TEST(testCase, skiplist_test) { + assert(sizeof(SSkipListKey) == 8); + srand(time(NULL)); + + // stringKeySkiplistTest(); + // doubleSkipListTest(); + skiplistPerformanceTest(); + // duplicatedKeyTest(); + + // tSKipListQueryCond q; + // q.upperBndRelOptr = true; + // q.lowerBndRelOptr = true; + // q.upperBnd.nType = TSDB_DATA_TYPE_DOUBLE; + // q.lowerBnd.nType = TSDB_DATA_TYPE_DOUBLE; + // q.lowerBnd.dKey = 120; + // q.upperBnd.dKey = 171.989; + /* + int32_t size = tSkipListQuery(pSkipList, &q, &pNodes); + for (int32_t i = 0; i < size; ++i) { + printf("-----%lf\n", pNodes[i]->key.dKey); + } + printf("the range query result size is: %d\n", size); + tfree(pNodes); + + SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); + for (int32_t i = 0; i < 8; i += 2) { + pKeys[i].dKey = i * 0.997; + pKeys[i].nType = TSDB_DATA_TYPE_DOUBLE; + printf("%lf ", pKeys[i].dKey); + } + + int32_t r = tSkipListPointQuery(pSkipList, pKeys, 8, EXCLUDE_POINT_QUERY, &pNodes); + printf("\nthe exclude query result is: %d\n", r); + for (int32_t i = 0; i < r; ++i) { + // printf("%lf ", pNodes[i]->key.dKey); + } + tfree(pNodes); + + free(pKeys);*/ + getchar(); +} diff --git a/src/vnode/tsdb/src/tsdbMain.c b/src/vnode/tsdb/src/tsdbMain.c index b1a531455d..bf870a0d05 100644 --- a/src/vnode/tsdb/src/tsdbMain.c +++ b/src/vnode/tsdb/src/tsdbMain.c @@ -719,7 +719,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable if (pTable->mem == NULL) { pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); pTable->mem->keyFirst = INT64_MAX; pTable->mem->keyLast = 0; } @@ -742,7 +742,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable if (pTable->mem == NULL) { pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, getTupleKey); + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); pTable->mem->keyFirst = INT64_MAX; pTable->mem->keyLast = 0; } @@ -1162,7 +1162,7 @@ static int compareKeyBlock(const void *arg1, const void *arg2) { return 0; } -static int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { +int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) { STsdbCfg * pCfg = &(pRepo->config); SCompData *pCompData = NULL; SFile * pFile = NULL; diff --git a/src/vnode/tsdb/src/tsdbMeta.c b/src/vnode/tsdb/src/tsdbMeta.c index 1dd1c3b29d..22680a839b 100644 --- a/src/vnode/tsdb/src/tsdbMeta.c +++ b/src/vnode/tsdb/src/tsdbMeta.c @@ -102,7 +102,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { if (pTable->type == TSDB_SUPER_TABLE) { pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, getTupleKey); + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, 0, 0, getTupleKey); } tsdbAddTableToMeta(pMeta, pTable, false); @@ -207,7 +207,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = tdDataRowDup(pCfg->tagValues); super->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, - 0, getTupleKey); // Allow duplicate key, no lock + 0, 0, getTupleKey); // Allow duplicate key, no lock if (super->pIndex == NULL) { tdFreeSchema(super->schema); diff --git a/src/vnode/tsdb/src/tsdbRead.c b/src/vnode/tsdb/src/tsdbRead.c index 3ed6e22ad8..2919b2cf9e 100644 --- a/src/vnode/tsdb/src/tsdbRead.c +++ b/src/vnode/tsdb/src/tsdbRead.c @@ -362,7 +362,7 @@ static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGrou SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid]; if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file - + } else { tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pBlock); } @@ -425,7 +425,7 @@ bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { // temporarily keep the position value, in case of no data qualified when move forwards(backwards) SQueryFilePos save = pQueryHandle->cur; -// fileIndex = getNextDataFileCompInfo_(pQueryHandle, &pQueryHandle->cur, &pQueryHandle->vnodeFileInfo, step); +// fileIndex = getNextDataFileCompInfo(pQueryHandle, &pQueryHandle->cur, &pQueryHandle->vnodeFileInfo, step); // first data block in the next file if (fileIndex >= 0) { @@ -441,7 +441,7 @@ bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { // if (nextTimestamp < 0) { // pQueryHandle->cur = save; // } - +// // return (nextTimestamp > 0); } @@ -460,11 +460,10 @@ bool moveToNextBlock(STsdbQueryHandle *pQueryHandle, int32_t step) { cur->pos = (step == QUERY_ASC_FORWARD_STEP) ? 0 : pBlock->numOfPoints - 1; // return loadQaulifiedData(pQueryHandle); } else { // data in cache -// todo continue; + return hasMoreDataInCacheForSingleModel(pQueryHandle); } } - int vnodeBinarySearchKey(char *pValue, int num, TSKEY key, int order) { int firstPos, lastPos, midPos = -1; int numOfPoints; @@ -579,8 +578,6 @@ static void filterDataInDataBlock(STsdbQueryHandle *pQueryHandle, SArray *sa) { } } - - assert(pQueryHandle->realNumOfRows <= blockInfo.size); // forward(backward) the position for cursor @@ -592,7 +589,9 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); SFileGroup* fileGroup = tsdbSearchFGroup(pFileHandle, fid); - pCheckInfo->checkFirstFileBlock = true; + if (fileGroup == NULL) { + return false; + } SQueryFilePos* cur = &pQueryHandle->cur; @@ -636,7 +635,7 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf // todo no need to loaded at all cur->slot = index; - + // sa = getDefaultLoadColumns(pQueryHandle, true); if (tsdbLoadDataBlock(&fileGroup->files[2], &pCheckInfo->pBlock[cur->slot], 1, fid, sa) == 0) { blockLoaded = true; @@ -660,34 +659,28 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf static bool hasMoreDataInFileForSingleTableModel(STsdbQueryHandle* pHandle) { assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1); + STsdbFileH* pFileHandle = tsdbGetFile(pHandle->pTsdb); SQueryFilePos* cur = &pHandle->cur; STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - if (!pCheckInfo->checkFirstFileBlock && pFileHandle != NULL) { - int32_t fid = getFileIdFromKey(pCheckInfo->lastKey); - SFileGroup* fileGroup = tsdbSearchFGroup(pFileHandle, fid); + if (!pCheckInfo->checkFirstFileBlock) { pCheckInfo->checkFirstFileBlock = true; - if (fileGroup != NULL) { - return getQualifiedDataBlock(pHandle, pCheckInfo, 1); - } else { // no data in file, try cache - return hasMoreDataInCacheForSingleModel(pHandle); - } - } else { - pCheckInfo->checkFirstFileBlock = true; - if (pFileHandle == NULL) { - cur->fid = -1; + if (pFileHandle != NULL) { + bool found = getQualifiedDataBlock(pHandle, pCheckInfo, 1); + if (found) { + return true; + } } - if (cur->fid == -1 || pFileHandle != NULL) { // try data in cache - return hasMoreDataInCacheForSingleModel(pHandle); - } else { - return true; - } + // no data in file, try cache + pHandle->cur.fid = -1; + return hasMoreDataInCacheForSingleModel(pHandle); + } else { // move to next data block in file or in cache + return moveToNextBlock(pHandle, 1); } - } static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) { From 818b04e51ea19cdf9f9605f9f2159e6f109a0fb2 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 30 Mar 2020 14:48:02 +0800 Subject: [PATCH 82/85] [TD-32] change the unit test cases. --- src/util/tests/hashTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp index 95357886d7..93a1989741 100644 --- a/src/util/tests/hashTest.cpp +++ b/src/util/tests/hashTest.cpp @@ -151,6 +151,6 @@ int main(int argc, char** argv) { TEST(testCase, hashTest) { simpleTest(); stringKeyTest(); -// noLockPerformanceTest(); + noLockPerformanceTest(); multithreadsTest(); } \ No newline at end of file From d8c7cdf1c9ed2b7cc407596d0ab10231632578aa Mon Sep 17 00:00:00 2001 From: hzcheng Date: Mon, 30 Mar 2020 16:44:56 +0800 Subject: [PATCH 83/85] TD-34 --- src/util/src/tskiplist.c | 2 +- src/vnode/tsdb/inc/tsdbFile.h | 13 +++++++++ src/vnode/tsdb/src/tsdbFile.c | 46 ++++++++++++++++++++++++++++++ src/vnode/tsdb/tests/tsdbTests.cpp | 4 +-- 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 1760919b05..77f643a2c7 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -472,7 +472,7 @@ void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t le SL_GET_FORWARD_POINTER(x, i) = pNode; } else { SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; - SL_GET_BACKWARD_POINTER(pSkipList->pHead, i) = (pSkipList->pHead); + // SL_GET_BACKWARD_POINTER(pSkipList->pHead, i) = (pSkipList->pHead); } } } diff --git a/src/vnode/tsdb/inc/tsdbFile.h b/src/vnode/tsdb/inc/tsdbFile.h index 0c85c5ef46..6c42d4aa15 100644 --- a/src/vnode/tsdb/inc/tsdbFile.h +++ b/src/vnode/tsdb/inc/tsdbFile.h @@ -82,6 +82,19 @@ int tsdbOpenFile(SFile *pFile, int oflag); int tsdbCloseFile(SFile *pFile); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); +#define TSDB_FGROUP_ITER_FORWARD 0 +#define TSDB_FGROUP_ITER_BACKWARD 1 +typedef struct { + int numOfFGroups; + SFileGroup *base; + SFileGroup *pFileGroup; + int direction; +} SFileGroupIter; + +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction); +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid); +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter); + typedef struct { int32_t len; int32_t offset; diff --git a/src/vnode/tsdb/src/tsdbFile.c b/src/vnode/tsdb/src/tsdbFile.c index bfdd998c22..d6964112e7 100644 --- a/src/vnode/tsdb/src/tsdbFile.c +++ b/src/vnode/tsdb/src/tsdbFile.c @@ -22,6 +22,7 @@ #include #include +#include "tutil.h" #include "tsdbFile.h" const char *tsdbFileSuffix[] = { @@ -133,6 +134,51 @@ int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { return 0; } +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { + pIter->direction = direction; + pIter->base = pFileH->fGroup; + pIter->numOfFGroups = pFileH->numOfFGroups; + if (pFileH->numOfFGroups == 0){ + pIter->pFileGroup = NULL; + } else { + if (direction == TSDB_FGROUP_ITER_FORWARD) { + pIter->pFileGroup = pFileH->fGroup; + } else { + pIter->pFileGroup = pFileH->fGroup + pFileH->numOfFGroups - 1; + } + } +} + +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { + int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; + void *ptr = taosbsearch(&fid, pIter->base, sizeof(SFileGroup), pIter->numOfFGroups, compFGroupKey, flags); + if (ptr == NULL) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup = (SFileGroup *)ptr; + } +} + +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { + SFileGroup *ret = pIter->pFileGroup; + if (ret == NULL) return NULL; + + if (pIter->direction = TSDB_FGROUP_ITER_FORWARD) { + if (pIter->pFileGroup + 1 == pIter->base + pIter->numOfFGroups) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup += 1; + } + } else { + if (pIter->pFileGroup - 1 == pIter->base) { + pIter->pFileGroup = NULL; + } else { + pIter->pFileGroup -= 1; + } + } + return ret; +} + int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) { SCompBlock *pBlock = pStartBlock; for (int i = 0; i < numOfBlocks; i++) { diff --git a/src/vnode/tsdb/tests/tsdbTests.cpp b/src/vnode/tsdb/tests/tsdbTests.cpp index 12311db68e..6cfe0e626d 100644 --- a/src/vnode/tsdb/tests/tsdbTests.cpp +++ b/src/vnode/tsdb/tests/tsdbTests.cpp @@ -49,8 +49,8 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) { ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0); } -TEST(TsdbTest, DISABLED_createRepo) { -// TEST(TsdbTest, createRepo) { +// TEST(TsdbTest, DISABLED_createRepo) { +TEST(TsdbTest, createRepo) { STsdbCfg config; // 1. Create a tsdb repository From f928765dc6a5001624fa3b232305617d3792d026 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 30 Mar 2020 17:18:53 +0800 Subject: [PATCH 84/85] [TD-32] opt skiplist insertion performance for ordered data --- src/util/src/tskiplist.c | 215 ++++++++++++++++++-------------- src/util/tests/hashTest.cpp | 8 +- src/util/tests/skiplistTest.cpp | 79 +++++++++--- 3 files changed, 183 insertions(+), 119 deletions(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 77f643a2c7..11e1e740e0 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -51,28 +51,27 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { level = 1; pSkipList->level = 1; } else { - if (level > pSkipList->level && pSkipList->level < pSkipList->maxLevel) { - level = (++pSkipList->level); + if (level > pSkipList->level) { + if (pSkipList->level < pSkipList->maxLevel) { + level = (++pSkipList->level); + } else { + level = pSkipList->level; + } } } + + assert(level <= pSkipList->maxLevel); return level; } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode); +#define DO_MEMSET_PTR_AREA(n) do {\ +int32_t _l = (n)->level;\ +memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ +(n)->level = _l;\ +} while(0) -void tSkipListDoRecordPut(SSkipList *pSkipList) { -#if SKIP_LIST_RECORD_PERFORMANCE - const int32_t MAX_RECORD_NUM = 1000; - - if (pSkipList->state.nInsertObjs == MAX_RECORD_NUM) { - pSkipList->state.nInsertObjs = 1; - pSkipList->state.nTotalStepsForInsert = 0; - pSkipList->state.nTotalElapsedTimeForInsert = 0; - } else { - pSkipList->state.nInsertObjs++; - } -#endif -} +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); +static SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode); int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); @@ -142,28 +141,12 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { } int32_t compareStrVal(const void *pLeft, const void *pRight) { - // SSkipListKey *pL = (SSkipListKey *)pLeft; - // SSkipListKey *pR = (SSkipListKey *)pRight; - // - // if (pL->nLen == 0 && pR->nLen == 0) { - // return 0; - // } - // - // // handle only one-side bound compare situation, there is only lower bound or only upper bound - // if (pL->nLen == -1) { - // return 1; // no lower bound, lower bound is minimum, always return -1; - // } else if (pR->nLen == -1) { - // return -1; // no upper bound, upper bound is maximum situation, always return 1; - // } - // - // int32_t ret = strcmp(((SSkipListKey *)pLeft)->pz, ((SSkipListKey *)pRight)->pz); - // - // if (ret == 0) { - // return 0; - // } else { - // return ret > 0 ? 1 : -1; - // } - return 0; + int32_t ret = strcmp(pLeft, pRight); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } } int32_t compareWStrVal(const void *pLeft, const void *pRight) { @@ -280,6 +263,28 @@ static __compar_fn_t getKeyComparator(int32_t keyType) { return comparFn; } +static bool initForwardBackwardPtr(SSkipList* pSkipList) { + uint32_t maxLevel = pSkipList->maxLevel; + + // head info + pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2); + if (pSkipList->pHead == NULL) { + return false; + } + + pSkipList->pHead->level = pSkipList->maxLevel; + + // tail info + pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel)); + pSkipList->pTail->level = pSkipList->maxLevel; + + for(int32_t i = 0; i < maxLevel; ++i) { + SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; + SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; + } + + return true; +} SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock, uint8_t freeNode, __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); @@ -291,21 +296,24 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui maxLevel = MAX_SKIP_LIST_LEVEL; } - pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; - pSkipList->keyFn = fn; + pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; + pSkipList->keyFn = fn; pSkipList->comparFn = getKeyComparator(keyType); pSkipList->maxLevel = maxLevel; - pSkipList->level = 1; - - pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pHead->level = pSkipList->maxLevel; - + pSkipList->level = 1; + + if (!initForwardBackwardPtr(pSkipList)) { + tfree(pSkipList); + return NULL; + } + if (lock) { pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t)); if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { tfree(pSkipList->pHead); tfree(pSkipList); + return NULL; } } @@ -349,7 +357,7 @@ void *tSkipListDestroy(SSkipList *pSkipList) { SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - while (pNode) { + while (pNode != pSkipList->pTail) { SSkipListNode *pTemp = pNode; pNode = SL_GET_FORWARD_POINTER(pNode, 0); @@ -358,8 +366,6 @@ void *tSkipListDestroy(SSkipList *pSkipList) { } } - tfree(pSkipList->pHead); - if (pSkipList->lock) { pthread_rwlock_unlock(pSkipList->lock); pthread_rwlock_destroy(pSkipList->lock); @@ -382,32 +388,34 @@ void tSkipListRandNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSi } SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { - if (pSkipList == NULL) { + if (pSkipList == NULL || pNode == NULL) { return NULL; } if (pSkipList->lock) { pthread_rwlock_wrlock(pSkipList->lock); } - - // record one node is put into skiplist - tSkipListDoRecordPut(pSkipList); - + + // the new key is greater than the last key of skiplist append it at last position + char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); + if (pSkipList->size == 0 || pSkipList->comparFn(pSkipList->lastKey, newDatakey) < 0) { + return tSkipListDoAppend(pSkipList, pNode); + } + + // find the appropriated position to insert data SSkipListNode *px = pSkipList->pHead; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; bool identical = false; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); - while (p != NULL) { + while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); - char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); // if the forward element is less than the specified key, forward one step int32_t ret = pSkipList->comparFn(key, newDatakey); if (ret < 0) { px = p; - p = SL_GET_FORWARD_POINTER(px, i); } else { if (identical == false) { @@ -417,10 +425,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { break; } } - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalStepsForInsert++; -#endif + forward[i] = px; } @@ -432,49 +437,59 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { return forward[0]; } - -#if SKIP_LIST_RECORD_PERFORMANCE - recordNodeEachLevel(pSkipList, level); -#endif - - // clear pointer area - int32_t level = SL_GET_NODE_LEVEL(pNode); - memset(pNode, 0, SL_NODE_HEADER_SIZE(pNode->level)); - pNode->level = level; - tSkipListDoInsert(pSkipList, forward, level, pNode); - - atomic_add_fetch_32(&pSkipList->size, 1); - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalMemSize += getOneNodeSize(pKey, level); -#endif - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - + tSkipListDoInsert(pSkipList, forward, pNode); return pNode; } -void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, int32_t level, SSkipListNode *pNode) { - for (int32_t i = 0; i < level; ++i) { +void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { + DO_MEMSET_PTR_AREA(pNode); + + for (int32_t i = 0; i < pNode->level; ++i) { SSkipListNode *x = forward[i]; - if (x != NULL) { + +// if (x != pSkipList->pTail) { SL_GET_BACKWARD_POINTER(pNode, i) = x; - SSkipListNode *pForward = SL_GET_FORWARD_POINTER(x, i); - if (pForward) { - SL_GET_BACKWARD_POINTER(pForward, i) = pNode; - } + SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); +// if (next) { + SL_GET_BACKWARD_POINTER(next, i) = pNode; +// } - SL_GET_FORWARD_POINTER(pNode, i) = SL_GET_FORWARD_POINTER(x, i); + SL_GET_FORWARD_POINTER(pNode, i) = next; SL_GET_FORWARD_POINTER(x, i) = pNode; - } else { - SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; - // SL_GET_BACKWARD_POINTER(pSkipList->pHead, i) = (pSkipList->pHead); - } +// } else { +// SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; +// } } + + atomic_add_fetch_32(&pSkipList->size, 1); + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); + } +} + +SSkipListNode* tSkipListDoAppend(SSkipList *pSkipList, SSkipListNode *pNode) { + // do clear pointer area + DO_MEMSET_PTR_AREA(pNode); + + for(int32_t i = 0; i < pNode->level; ++i) { + SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); + SL_GET_FORWARD_POINTER(prev, i) = pNode; + SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; + + SL_GET_BACKWARD_POINTER(pNode, i) = prev; + SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; + } + + pSkipList->lastKey = SL_GET_NODE_KEY(pSkipList, pNode); + + atomic_add_fetch_32(&pSkipList->size, 1); + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); + } + + return pNode; } SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey, int16_t keyType) { @@ -766,11 +781,18 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); int32_t id = 1; - - while (p) { + char* prev = NULL; + + while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); + if (prev != NULL) { + assert(pSkipList->comparFn(prev, key) < 0); + } + switch (pSkipList->keyInfo.type) { case TSDB_DATA_TYPE_INT: + fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); + break; case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BIGINT: @@ -786,7 +808,8 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { fprintf(stdout, "\n"); } + prev = SL_GET_NODE_KEY(pSkipList, p); + p = SL_GET_FORWARD_POINTER(p, nlevel - 1); - // p = p->pForward[nlevel - 1]; } } diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp index 93a1989741..b3baedb696 100644 --- a/src/util/tests/hashTest.cpp +++ b/src/util/tests/hashTest.cpp @@ -149,8 +149,8 @@ int main(int argc, char** argv) { } TEST(testCase, hashTest) { - simpleTest(); - stringKeyTest(); - noLockPerformanceTest(); - multithreadsTest(); +// simpleTest(); +// stringKeyTest(); +// noLockPerformanceTest(); +// multithreadsTest(); } \ No newline at end of file diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 7a3e87d786..c009231874 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -69,6 +69,41 @@ void doubleSkipListTest() { tSkipListDestroy(pSkipList); } +void randKeyTest() { + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), 0, false, true, getkey); + + int32_t size = 200000; + srand(time(NULL)); + + printf("generated %d keys is: \n", size); + + for (int32_t i = 0; i < size; ++i) { + int32_t level = 0; + int32_t s = 0; + + tSkipListRandNodeInfo(pSkipList, &level, &s); + auto d = (SSkipListNode*)calloc(1, s + sizeof(int32_t) * 2); + d->level = level; + + int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); + key[0] = rand() % 1000000000; + + key[1] = key[0]; + + tSkipListPut(pSkipList, d); + } + + printf("the first level of skip list is:\n"); + tSkipListPrint(pSkipList, 1); + + printf("the sec level of skip list is:\n"); + tSkipListPrint(pSkipList, 2); + + printf("the 5 level of skip list is:\n"); + tSkipListPrint(pSkipList, 5); + + tSkipListDestroy(pSkipList); +} void stringKeySkiplistTest() { const int32_t max_key_size = 12; @@ -100,6 +135,9 @@ void stringKeySkiplistTest() { tSkipListPut(pSkipList, pNode); + printf("level one------------------\n"); + tSkipListPrint(pSkipList, 1); + #if 0 SSkipListNode **pRes = NULL; int32_t ret = tSkipListGets(pSkipList, &key1, &pRes); @@ -139,6 +177,9 @@ void stringKeySkiplistTest() { int64_t e = taosGetTimestampUs(); printf("elapsed time:%lld us to insert %d data, avg:%f us\n", (e - s), total, (double)(e - s) / total); + printf("level two------------------\n"); + tSkipListPrint(pSkipList, 1); + #if 0 SSkipListNode **pres = NULL; @@ -167,7 +208,7 @@ void stringKeySkiplistTest() { void skiplistPerformanceTest() { SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, false, getkey); - int32_t size = 900000; + int32_t size = 1000000; int64_t prev = taosGetTimestampMs(); int64_t s = prev; @@ -179,7 +220,7 @@ void skiplistPerformanceTest() { char* total = (char*)calloc(1, unit * size); char* p = total; - for (int32_t i = size; i > 0; --i) { + for (int32_t i = 0; i < size; ++i) { tSkipListRandNodeInfo(pSkipList, &level, &headsize); SSkipListNode* d = (SSkipListNode*)p; @@ -207,19 +248,19 @@ void skiplistPerformanceTest() { assert(tSkipListGetSize(pSkipList) == size); - printf("the level of skiplist is:\n"); - -// printf("level two------------------\n"); -// tSkipListPrint(pSkipList, 2); -// -// printf("level three------------------\n"); -// tSkipListPrint(pSkipList, 3); -// -// printf("level four------------------\n"); -// tSkipListPrint(pSkipList, 4); -// -// printf("level nine------------------\n"); -// tSkipListPrint(pSkipList, 10); + // printf("the level of skiplist is:\n"); + // + // printf("level two------------------\n"); + // tSkipListPrint(pSkipList, 2); + // + // printf("level three------------------\n"); + // tSkipListPrint(pSkipList, 3); + // + // printf("level four------------------\n"); + // tSkipListPrint(pSkipList, 4); + // + // printf("level nine------------------\n"); + // tSkipListPrint(pSkipList, 10); int64_t st = taosGetTimestampMs(); #if 0 @@ -275,10 +316,11 @@ TEST(testCase, skiplist_test) { assert(sizeof(SSkipListKey) == 8); srand(time(NULL)); - // stringKeySkiplistTest(); - // doubleSkipListTest(); + stringKeySkiplistTest(); + doubleSkipListTest(); skiplistPerformanceTest(); - // duplicatedKeyTest(); + duplicatedKeyTest(); + randKeyTest(); // tSKipListQueryCond q; // q.upperBndRelOptr = true; @@ -310,5 +352,4 @@ TEST(testCase, skiplist_test) { tfree(pNodes); free(pKeys);*/ - getchar(); } From bf9c67405d00fe9ee962676e61ff627f35ae265b Mon Sep 17 00:00:00 2001 From: hzcheng Date: Mon, 30 Mar 2020 17:29:05 +0800 Subject: [PATCH 85/85] TD-34 --- src/rpc/src/rpcMain.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 1847475a97..78952e2209 100755 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -829,18 +829,19 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->port); } - if (terrno != TSDB_CODE_ALREADY_PROCESSED) { - if (terrno != 0) { // parsing error + int32_t code = terrno; + if (code != TSDB_CODE_ALREADY_PROCESSED) { + if (code != 0) { // parsing error if ( rpcIsReq(pHead->msgType) ) { - rpcSendErrorMsgToPeer(pRecv, terrno); - tTrace("%s %p, %s is sent with error code:%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], terrno); + rpcSendErrorMsgToPeer(pRecv, code); + tTrace("%s %p, %s is sent with error code:%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], code); } } else { // parsing OK rpcProcessIncomingMsg(pConn, pHead); } } - if (terrno) rpcFreeMsg(pRecv->msg); + if (code) rpcFreeMsg(pRecv->msg); return pConn; }