diff --git a/cmake/bdb_CMakeLists.txt.in.bak b/cmake/bdb_CMakeLists.txt.in similarity index 100% rename from cmake/bdb_CMakeLists.txt.in.bak rename to cmake/bdb_CMakeLists.txt.in diff --git a/cmake/cmake.define b/cmake/cmake.define index 0eb5206aab..4e27ff5f47 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -14,25 +14,6 @@ MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR}) MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH}) MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH}) -find_package(Git QUIET) -if(GIT_FOUND AND EXISTS "${TD_SOURCE_DIR}/.git") -# Update submodules as needed - option(GIT_SUBMODULE "Check submodules during build" ON) - if(GIT_SUBMODULE) - message(STATUS "Submodule update") - execute_process(COMMAND cd ${TD_SOURCE_DIR} && ${GIT_EXECUTABLE} submodule update --init --recursive - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - RESULT_VARIABLE GIT_SUBMOD_RESULT) - if(NOT GIT_SUBMOD_RESULT EQUAL "0") - message(WARNING "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") - endif() - endif() -endif() - -if(NOT EXISTS "${TD_SOURCE_DIR}/tools/taos-tools/CMakeLists.txt") - message(WARNING "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules manually if you need build them.") -endif() - if (NOT DEFINED TD_GRANT) SET(TD_GRANT FALSE) endif() diff --git a/cmake/cmake.options b/cmake/cmake.options index a60f5c7282..d83ab49fd5 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -78,6 +78,12 @@ option( OFF ) +option( + BUILD_WITH_BDB + "If build with BDB" + OFF +) + option( BUILD_WITH_LUCENE "If build with lucene" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 14a85ee4f6..97bfcfb8c0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -78,9 +78,9 @@ if(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV}) # bdb -#if(${BUILD_WITH_BDB}) - #cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -#endif(${BUILD_WITH_BDB}) +if(${BUILD_WITH_BDB}) + cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(${BUILD_WITH_BDB}) # sqlite if(${BUILD_WITH_SQLITE}) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 740488b39b..eacaeb9524 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) endif(${BUILD_WITH_LUCENE}) -#if(${BUILD_WITH_BDB}) - #add_subdirectory(bdb) -#endif(${BUILD_WITH_BDB}) +if(${BUILD_WITH_BDB}) + add_subdirectory(bdb) +endif(${BUILD_WITH_BDB}) if(${BUILD_WITH_SQLITE}) add_subdirectory(sqlite) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index e1f8832edf..72d2c142d2 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -37,7 +37,8 @@ typedef enum { TSDB_STREAM_TABLE = 4, // table created from stream computing TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_SYSTEM_TABLE = 6, - TSDB_TABLE_MAX = 7 + TSDB_TSMA_TABLE = 7, // time-range-wise sma + TSDB_TABLE_MAX = 8 } ETableType; typedef enum { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 72ec017980..544af9b6ee 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1670,6 +1670,7 @@ typedef struct SVDropStbReq { int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq); int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq); +// TDMT_VND_CREATE_TABLE ============== #define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { int32_t flags; @@ -1759,6 +1760,43 @@ typedef struct { int32_t tEncodeSVDropTbBatchRsp(SEncoder* pCoder, const SVDropTbBatchRsp* pRsp); int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); +// TDMT_VND_ALTER_TABLE ===================== +typedef struct { + const char* tbName; + int8_t action; + const char* colName; + // TSDB_ALTER_TABLE_ADD_COLUMN + int8_t type; + int8_t flags; + int32_t bytes; + // TSDB_ALTER_TABLE_DROP_COLUMN + // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES + int32_t colModBytes; + // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME + const char* colNewName; + // TSDB_ALTER_TABLE_UPDATE_TAG_VAL + const char* tagName; + int8_t isNull; + uint32_t nTagVal; + const uint8_t* pTagVal; + // TSDB_ALTER_TABLE_UPDATE_OPTIONS + int8_t updateTTL; + int32_t newTTL; + int8_t updateComment; + const char* newComment; +} SVAlterTbReq; + +int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); +int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq); + +typedef struct { + int32_t code; +} SVAlterTbRsp; + +int32_t tEncodeSVAlterTbRsp(SEncoder* pEncoder, const SVAlterTbRsp* pRsp); +int32_t tDecodeSVAlterTbRsp(SDecoder* pDecoder, SVAlterTbRsp* pRsp); +// ====================== + typedef struct { SMsgHead head; int64_t uid; @@ -2160,26 +2198,23 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - char* expr; // sma expression - char* tagsFilter; + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + const char* expr; // sma expression + const char* tagsFilter; } STSma; // Time-range-wise SMA -typedef struct { - int64_t ver; // use a general definition - STSma tSma; -} SVCreateTSmaReq; +typedef STSma SVCreateTSmaReq; typedef struct { int8_t type; // 0 status report, 1 update data @@ -2188,7 +2223,6 @@ typedef struct { } STSmaMsg; typedef struct { - int64_t ver; // use a general definition int64_t indexUid; char indexName[TSDB_INDEX_NAME_LEN]; } SVDropTSmaReq; @@ -2197,28 +2231,21 @@ typedef struct { int tmp; // TODO: to avoid compile error } SVCreateTSmaRsp, SVDropTSmaRsp; +#if 0 int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq); void* tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq); int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq); void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq); +#endif -// RSma: Rollup SMA -typedef struct { - int64_t interval; - int32_t retention; // unit: day - uint16_t days; // unit: day - int8_t intervalUnit; -} SSmaParams; +int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq); +int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq); +int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq); +int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq); typedef struct { - STSma tsma; - float xFilesFactor; - SArray* smaParams; // SSmaParams -} SRSma; - -typedef struct { - uint32_t number; - STSma* tSma; + int32_t number; + STSma* tSma; } STSmaWrapper; static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { @@ -2245,96 +2272,26 @@ static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) { return NULL; } -static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { - int32_t tlen = 0; +int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq); +int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq); - tlen += taosEncodeFixedI8(buf, pSma->version); - tlen += taosEncodeFixedI8(buf, pSma->intervalUnit); - tlen += taosEncodeFixedI8(buf, pSma->slidingUnit); - tlen += taosEncodeFixedI8(buf, pSma->timezoneInt); - tlen += taosEncodeString(buf, pSma->indexName); - tlen += taosEncodeFixedI32(buf, pSma->exprLen); - tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen); - tlen += taosEncodeFixedI64(buf, pSma->indexUid); - tlen += taosEncodeFixedI64(buf, pSma->tableUid); - tlen += taosEncodeFixedI64(buf, pSma->interval); - tlen += taosEncodeFixedI64(buf, pSma->offset); - tlen += taosEncodeFixedI64(buf, pSma->sliding); +int32_t tEncodeTSma(SEncoder* pCoder, const STSma* pSma); +int32_t tDecodeTSma(SDecoder* pCoder, STSma* pSma); - if (pSma->exprLen > 0) { - tlen += taosEncodeString(buf, pSma->expr); +static int32_t tEncodeTSmaWrapper(SEncoder* pEncoder, const STSmaWrapper* pReq) { + if (tEncodeI32(pEncoder, pReq->number) < 0) return -1; + for (int32_t i = 0; i < pReq->number; ++i) { + tEncodeTSma(pEncoder, pReq->tSma + i); } - - if (pSma->tagsFilterLen > 0) { - tlen += taosEncodeString(buf, pSma->tagsFilter); - } - - return tlen; + return 0; } -static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) { - int32_t tlen = 0; - - tlen += taosEncodeFixedU32(buf, pSW->number); - for (uint32_t i = 0; i < pSW->number; ++i) { - tlen += tEncodeTSma(buf, pSW->tSma + i); +static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { + if (tDecodeI32(pDecoder, &pReq->number) < 0) return -1; + for (int32_t i = 0; i < pReq->number; ++i) { + tDecodeTSma(pDecoder, pReq->tSma + i); } - return tlen; -} - -static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { - buf = taosDecodeFixedI8(buf, &pSma->version); - buf = taosDecodeFixedI8(buf, &pSma->intervalUnit); - buf = taosDecodeFixedI8(buf, &pSma->slidingUnit); - buf = taosDecodeFixedI8(buf, &pSma->timezoneInt); - buf = taosDecodeStringTo(buf, pSma->indexName); - buf = taosDecodeFixedI32(buf, &pSma->exprLen); - buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen); - buf = taosDecodeFixedI64(buf, &pSma->indexUid); - buf = taosDecodeFixedI64(buf, &pSma->tableUid); - buf = taosDecodeFixedI64(buf, &pSma->interval); - buf = taosDecodeFixedI64(buf, &pSma->offset); - buf = taosDecodeFixedI64(buf, &pSma->sliding); - - if (pSma->exprLen > 0) { - if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) { - tdDestroyTSma(pSma); - return NULL; - } - } else { - pSma->expr = NULL; - } - - if (pSma->tagsFilterLen > 0) { - if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) { - tdDestroyTSma(pSma); - return NULL; - } - } else { - pSma->tagsFilter = NULL; - } - - return buf; -} - -static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { - buf = taosDecodeFixedU32(buf, &pSW->number); - - pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma)); - if (pSW->tSma == NULL) { - return NULL; - } - - for (uint32_t i = 0; i < pSW->number; ++i) { - if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) { - for (uint32_t j = i; j >= 0; --i) { - tdDestroyTSma(pSW->tSma + j); - } - taosMemoryFree(pSW->tSma); - return NULL; - } - } - return buf; + return 0; } typedef struct { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 796fee4f89..4460327b88 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -40,6 +40,7 @@ enum { TASK_INPUT_STATUS__BLOCKED, TASK_INPUT_STATUS__RECOVER, TASK_INPUT_STATUS__STOP, + TASK_INPUT_STATUS__FAILED, }; enum { @@ -234,7 +235,9 @@ struct SStreamTask { int8_t outputStatus; STaosQueue* inputQ; + STaosQall* inputQAll; STaosQueue* outputQ; + STaosQall* outputQAll; // application storage void* ahandle; @@ -282,7 +285,7 @@ int32_t streamDequeueOutput(SStreamTask* pTask, void** output); int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); -int32_t streamTaskExecNew(SStreamTask* pTask); +int32_t streamTaskRun(SStreamTask* pTask); int32_t streamTaskHandleInput(SStreamTask* pTask, void* data); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ca10465ed4..551e0fc7b8 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -144,6 +144,7 @@ int32_t syncInit(); void syncCleanUp(); int64_t syncOpen(const SSyncInfo* pSyncInfo); void syncStart(int64_t rid); +void syncStartStandBy(int64_t rid); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); ESyncState syncGetMyRole(int64_t rid); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 03f445b498..32bda084ed 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -323,6 +323,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0516) #define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0517) #define TSDB_CODE_VND_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0518) +#define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0519) +#define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051a) +#define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051b) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -354,7 +357,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C) -#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x062D) +#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x061D) +#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x061E) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) diff --git a/include/util/tencode.h b/include/util/tencode.h index 2a43d7934f..e49429f865 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -18,7 +18,6 @@ #include "tcoding.h" #include "tlist.h" -// #include "tfreelist.h" #ifdef __cplusplus extern "C" { diff --git a/include/util/tfreelist.h b/include/util/tfreelist.h deleted file mode 100644 index e9b5ca5fca..0000000000 --- a/include/util/tfreelist.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_FREELIST_H_ -#define _TD_UTIL_FREELIST_H_ - -#include "tlist.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct SFreeListNode { - TD_SLIST_NODE(SFreeListNode); - char payload[]; -}; - -typedef TD_SLIST(SFreeListNode) SFreeList; - -#define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \ - do { \ - void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \ - if (ptr) { \ - TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ - ptr = ((struct SFreeListNode *)ptr)->payload; \ - (PTR) = (TYPE)(ptr); \ - }else{ \ - (PTR) = NULL; \ - } \ - }while(0); - -#define tFreeListInit(pFL) TD_SLIST_INIT(pFL) - -static FORCE_INLINE void tFreeListClear(SFreeList *pFL) { - struct SFreeListNode *pNode; - for (;;) { - pNode = TD_SLIST_HEAD(pFL); - if (pNode == NULL) break; - TD_SLIST_POP(pFL); - taosMemoryFree(pNode); - } -} - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_FREELIST_H_*/ \ No newline at end of file diff --git a/include/util/tlog.h b/include/util/tlog.h index dead25a4a8..be31aa8115 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -61,6 +61,7 @@ extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; extern int32_t metaDebugFlag; extern int32_t fnDebugFlag; +extern int32_t smaDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 666e9e0ffa..100ec42899 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -518,8 +518,9 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t if (pRequest->code != TSDB_CODE_SUCCESS) { const char* errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); - printf("failed to connect to server, reason: %s\n\n", errorMsg); + fprintf(stderr,"failed to connect to server, reason: %s\n\n", errorMsg); + terrno = pRequest->code; destroyRequest(pRequest); taos_close(pTscObj); pTscObj = NULL; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8ed4308ea7..8fce5bfb00 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1007,8 +1007,6 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump while(*sql != '\0') { // parse value if (*sql == SPACE) { - valueLen = sql - value; - sql++; break; } if (*sql == EQUAL) { @@ -1017,9 +1015,7 @@ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dump } sql++; } - if(valueLen == 0){ - valueLen = sql - value; - } + valueLen = sql - value; if(valueLen == 0){ smlBuildInvalidDataMsg(msg, "invalid value", value); @@ -1365,7 +1361,7 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta){ static void smlDestroyCols(SArray *cols) { if (!cols) return; for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGet(cols, i); + void *kv = taosArrayGetP(cols, i); taosMemoryFree(kv); } } @@ -2077,12 +2073,16 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) { if(ret != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" smlParseTelnetLine failed", info->id); smlDestroyTableInfo(info, tinfo); + smlDestroyCols(cols); taosArrayDestroy(cols); return ret; } if(taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS){ smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); + smlDestroyTableInfo(info, tinfo); + smlDestroyCols(cols); + taosArrayDestroy(cols); return TSDB_CODE_SML_INVALID_DATA; } taosHashClear(info->dumplicateKey); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 589c3d3a10..e567a0c3e8 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -516,8 +516,8 @@ TEST(testCase, smlProcess_influx_Test) { int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_EQ(ret, 0); - TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); - ASSERT_NE(res, nullptr); +// TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); +// ASSERT_NE(res, nullptr); // int fieldNum = taos_field_count(res); // ASSERT_EQ(fieldNum, 5); // int rowNum = taos_affected_rows(res); @@ -525,7 +525,7 @@ TEST(testCase, smlProcess_influx_Test) { // for (int i = 0; i < rowNum; ++i) { // TAOS_ROW rows = taos_fetch_row(res); // } - taos_free_result(res); +// taos_free_result(res); destroyRequest(request); smlDestroyInfo(info); } @@ -605,8 +605,8 @@ TEST(testCase, smlProcess_telnet_Test) { int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_EQ(ret, 0); - TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); - ASSERT_NE(res, nullptr); +// TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); +// ASSERT_NE(res, nullptr); // int fieldNum = taos_field_count(res); // ASSERT_EQ(fieldNum, 2); // int rowNum = taos_affected_rows(res); @@ -614,7 +614,7 @@ TEST(testCase, smlProcess_telnet_Test) { // for (int i = 0; i < rowNum; ++i) { // TAOS_ROW rows = taos_fetch_row(res); // } - taos_free_result(res); +// taos_free_result(res); // res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961"); // ASSERT_NE(res, nullptr); @@ -670,16 +670,16 @@ TEST(testCase, smlProcess_json1_Test) { int ret = smlProcess(info, (char **)(&sql), -1); ASSERT_EQ(ret, 0); - TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); - ASSERT_NE(res, nullptr); - int fieldNum = taos_field_count(res); - ASSERT_EQ(fieldNum, 2); +// TAOS_RES *res = taos_query(taos, "select * from t_cb27a7198d637b4f1c6464bd73f756a7"); +// ASSERT_NE(res, nullptr); +// int fieldNum = taos_field_count(res); +// ASSERT_EQ(fieldNum, 2); // int rowNum = taos_affected_rows(res); // ASSERT_EQ(rowNum, 1); // for (int i = 0; i < rowNum; ++i) { // TAOS_ROW rows = taos_fetch_row(res); // } - taos_free_result(res); +// taos_free_result(res); destroyRequest(request); smlDestroyInfo(info); } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 025ad0ca0d..aaad606feb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -299,6 +299,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1; return 0; } @@ -480,6 +481,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; + smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; } static int32_t taosSetClientCfg(SConfig *pCfg) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 021ee8455e..52edd79f3b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3562,39 +3562,92 @@ int tDecodeSVCreateTbBatchRsp(SDecoder *pCoder, SVCreateTbBatchRsp *pRsp) { return 0; } -int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) { - int32_t tlen = 0; - - tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += tEncodeTSma(buf, &pReq->tSma); - - return tlen; -} - -void *tDeserializeSVCreateTSmaReq(void *buf, SVCreateTSmaReq *pReq) { - buf = taosDecodeFixedI64(buf, &(pReq->ver)); - - if ((buf = tDecodeTSma(buf, &pReq->tSma)) == NULL) { - tdDestroyTSma(&pReq->tSma); +int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { + if (tEncodeI8(pCoder, pSma->version) < 0) return -1; + if (tEncodeI8(pCoder, pSma->intervalUnit) < 0) return -1; + if (tEncodeI8(pCoder, pSma->slidingUnit) < 0) return -1; + if (tEncodeI8(pCoder, pSma->timezoneInt) < 0) return -1; + if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1; + if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1; + if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1; + if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1; + if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1; + if (tEncodeI64(pCoder, pSma->interval) < 0) return -1; + if (tEncodeI64(pCoder, pSma->offset) < 0) return -1; + if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1; + if (pSma->exprLen > 0) { + if (tEncodeCStr(pCoder, pSma->expr) < 0) return -1; } - return buf; + if (pSma->tagsFilterLen > 0) { + if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; + } + + return 0; } -int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) { - int32_t tlen = 0; +int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { + if (tDecodeI8(pCoder, &pSma->version) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1; + if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1; + if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1; + if (pSma->exprLen > 0) { + if (tDecodeCStr(pCoder, &pSma->expr) < 0) return -1; + } else { + pSma->expr = NULL; + } + if (pSma->tagsFilterLen > 0) { + if (tDecodeCStr(pCoder, &pSma->tagsFilter) < 0) return -1; + } else { + pSma->tagsFilter = NULL; + } - tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += taosEncodeFixedI64(buf, pReq->indexUid); - tlen += taosEncodeString(buf, pReq->indexName); - - return tlen; + return 0; } -void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) { - buf = taosDecodeFixedI64(buf, &(pReq->ver)); - buf = taosDecodeFixedI64(buf, &(pReq->indexUid)); - buf = taosDecodeStringTo(buf, pReq->indexName); - return buf; +int32_t tEncodeSVCreateTSmaReq(SEncoder *pCoder, const SVCreateTSmaReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + tEncodeTSma(pCoder, pReq); + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVCreateTSmaReq(SDecoder *pCoder, SVCreateTSmaReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + tDecodeTSma(pCoder, pReq); + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->indexName) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; + if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1; + + tEndDecode(pCoder); + return 0; } int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) { @@ -4110,3 +4163,114 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) { taosMemoryFree(pRsp); } + +int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + + if (tEncodeCStr(pEncoder, pReq->tbName) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->action) < 0) return -1; + switch (pReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->type) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->flags) < 0) return -1; + if (tEncodeI32v(pEncoder, pReq->bytes) < 0) return -1; + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1; + if (tEncodeI32v(pEncoder, pReq->colModBytes) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1; + if (tEncodeCStr(pEncoder, pReq->colNewName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: + if (tEncodeCStr(pEncoder, pReq->tagName) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->isNull) < 0) return -1; + if (!pReq->isNull) { + if (tEncodeBinary(pEncoder, pReq->pTagVal, pReq->nTagVal) < 0) return -1; + } + break; + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + if (tEncodeI8(pEncoder, pReq->updateTTL) < 0) return -1; + if (pReq->updateTTL) { + if (tEncodeI32v(pEncoder, pReq->newTTL) < 0) return -1; + } + if (tEncodeI8(pEncoder, pReq->updateComment) < 0) return -1; + if (pReq->updateComment) { + if (tEncodeCStr(pEncoder, pReq->newComment) < 0) return -1; + } + break; + default: + break; + } + + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + + if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1; + if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; + switch (pReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->type) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->flags) < 0) return -1; + if (tDecodeI32v(pDecoder, &pReq->bytes) < 0) return -1; + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; + if (tDecodeI32v(pDecoder, &pReq->colModBytes) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; + if (tDecodeCStr(pDecoder, &pReq->colNewName) < 0) return -1; + break; + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: + if (tDecodeCStr(pDecoder, &pReq->tagName) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->isNull) < 0) return -1; + if (!pReq->isNull) { + if (tDecodeBinary(pDecoder, &pReq->pTagVal, &pReq->nTagVal) < 0) return -1; + } + break; + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + if (tDecodeI8(pDecoder, &pReq->updateTTL) < 0) return -1; + if (pReq->updateTTL) { + if (tDecodeI32v(pDecoder, &pReq->newTTL) < 0) return -1; + } + if (tDecodeI8(pDecoder, &pReq->updateComment) < 0) return -1; + if (pReq->updateComment) { + if (tDecodeCStr(pDecoder, &pReq->newComment) < 0) return -1; + } + break; + default: + break; + } + + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSVAlterTbRsp(SEncoder *pEncoder, const SVAlterTbRsp *pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->code) < 0) return -1; + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->code) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d6e99c0899..edc0090973 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -192,7 +192,8 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO rpcFreeCont(originalRpcMsg.pCont); // if leader, send response - if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { + //if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { + if (pMsg->rpcMsg.handle != NULL) { rsp.ahandle = pMsg->rpcMsg.ahandle; rsp.handle = pMsg->rpcMsg.handle; rsp.refId = pMsg->rpcMsg.refId; @@ -213,7 +214,23 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf // todo SRpcMsg *pRsp = NULL; - (void)vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + int32_t ret = vnodeProcessSyncReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + if (ret != 0) { + // if leader, send response + if (pMsg->rpcMsg.handle != NULL) { + SRpcMsg rsp; + rsp.pCont = NULL; + rsp.contLen = 0; + + rsp.code = terrno; + dTrace("vmProcessSyncQueue error, code:%d", terrno); + + rsp.ahandle = pMsg->rpcMsg.ahandle; + rsp.handle = pMsg->rpcMsg.handle; + rsp.refId = pMsg->rpcMsg.refId; + tmsgSendRsp(&rsp); + } + } rpcFreeCont(pMsg->rpcMsg.pCont); taosFreeQitem(pMsg); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 824f031004..22a5f37334 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -194,6 +194,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p // source pTask->sourceType = TASK_SOURCE__MERGE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // exec pTask->execType = TASK_EXEC__NONE; @@ -235,6 +236,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); // source pTask->sourceType = TASK_SOURCE__MERGE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // exec pTask->execType = TASK_EXEC__NONE; @@ -309,6 +311,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SStreamTask* pTask = tNewSStreamTask(pStream->uid); // source part pTask->sourceType = TASK_SOURCE__SCAN; + pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; // sink part if (level == 0) { @@ -372,6 +375,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // source part, currently only support multi source pTask->sourceType = TASK_SOURCE__PIPE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // sink part pTask->sinkType = TASK_SINK__NONE; @@ -459,6 +463,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // source part pTask->sourceType = TASK_SOURCE__MERGE; + pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; // sink part pTask->sinkType = TASK_SINK__NONE; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 02a8eaa832..869bf0c0a9 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -242,26 +242,35 @@ SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) { } static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { - SName name = {0}; + SEncoder encoder = {0}; + int32_t contLen = 0; + SName name = {0}; tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVCreateTSmaReq req = {0}; - req.tSma.version = 0; - req.tSma.intervalUnit = pSma->intervalUnit; - req.tSma.slidingUnit = pSma->slidingUnit; - req.tSma.timezoneInt = pSma->timezone; - tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); - req.tSma.exprLen = pSma->exprLen; - req.tSma.tagsFilterLen = pSma->tagsFilterLen; - req.tSma.indexUid = pSma->uid; - req.tSma.tableUid = pSma->stbUid; - req.tSma.interval = pSma->interval; - req.tSma.offset = pSma->offset; - req.tSma.sliding = pSma->sliding; - req.tSma.expr = pSma->expr; - req.tSma.tagsFilter = pSma->tagsFilter; + req.version = 0; + req.intervalUnit = pSma->intervalUnit; + req.slidingUnit = pSma->slidingUnit; + req.timezoneInt = pSma->timezone; + tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); + req.exprLen = pSma->exprLen; + req.tagsFilterLen = pSma->tagsFilterLen; + req.indexUid = pSma->uid; + req.tableUid = pSma->stbUid; + req.interval = pSma->interval; + req.offset = pSma->offset; + req.sliding = pSma->sliding; + req.expr = pSma->expr; + req.tagsFilter = pSma->tagsFilter; + + // get length + int32_t ret = 0; + tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret); + if (ret < 0) { + return NULL; + } + contLen += sizeof(SMsgHead); - int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -272,22 +281,38 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm pHead->vgId = htonl(pVgroup->vgId); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); - tSerializeSVCreateTSmaReq(&pBuf, &req); + tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); + if (tEncodeSVCreateTSmaReq(&encoder, &req) < 0) { + taosMemoryFreeClear(pHead); + tEncoderClear(&encoder); + return NULL; + } + + tEncoderClear(&encoder); *pContLen = contLen; return pHead; } static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { + SEncoder encoder = {0}; + int32_t contLen; SName name = {0}; tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVDropTSmaReq req = {0}; - req.ver = 0; req.indexUid = pSma->uid; tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); - int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead); + // get length + int32_t ret = 0; + tEncodeSize(tEncodeSVDropTSmaReq, &req, contLen, ret); + if (ret < 0) { + return NULL; + } + + contLen += sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -298,7 +323,14 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, pHead->vgId = htonl(pVgroup->vgId); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); - tDeserializeSVDropTSmaReq(&pBuf, &req); + tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); + + if (tEncodeSVDropTSmaReq(&encoder, &req) < 0) { + taosMemoryFreeClear(pHead); + tEncoderClear(&encoder); + return NULL; + } + tEncoderClear(&encoder); *pContLen = contLen; return pHead; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 12e89277f4..5a834f4d33 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -383,9 +383,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.suid = pStb->uid; req.rollup = pStb->ast1Len > 0 ? 1 : 0; req.schema.nCols = pStb->numOfColumns; - req.schema.sver = 0; + req.schema.sver = pStb->version; req.schema.pSchema = pStb->pColumns; req.schemaTag.nCols = pStb->numOfTags; + req.schemaTag.nCols = 0; req.schemaTag.pSchema = pStb->pTags; if (req.rollup) { @@ -425,6 +426,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); if (tEncodeSVCreateStbReq(&encoder, &req) < 0) { + taosMemoryFreeClear(pHead); + taosMemoryFreeClear(req.pRSmaParam.qmsg1); + taosMemoryFreeClear(req.pRSmaParam.qmsg2); + tEncoderClear(&encoder); return NULL; } tEncoderClear(&encoder); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 58e00ee34a..a8e3860ed1 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -18,12 +18,21 @@ target_sources( "src/meta/metaOpen.c" "src/meta/metaIdx.c" "src/meta/metaTable.c" + "src/meta/metaSma.c" "src/meta/metaQuery.c" "src/meta/metaCommit.c" "src/meta/metaEntry.c" + # sma + "src/sma/sma.c" + "src/sma/smaTDBImpl.c" + "src/sma/smaEnv.c" + "src/sma/smaOpen.c" + "src/sma/smaRollup.c" + "src/sma/smaTimeRange.c" + # tsdb - "src/tsdb/tsdbTDBImpl.c" + # "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit2.c" "src/tsdb/tsdbFile.c" @@ -33,7 +42,7 @@ target_sources( "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" - "src/tsdb/tsdbSma.c" + # "src/tsdb/tsdbSma.c" "src/tsdb/tsdbWrite.c" # tq diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 130cebf0b1..4431a2c48b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -189,8 +189,12 @@ struct SMetaEntry { struct { int64_t ctime; int32_t ttlDays; + int32_t ncid; // next column id SSchemaWrapper schema; } ntbEntry; + struct { + STSma *tsma; + } smaEntry; }; }; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 60b3a889ef..c5ca806829 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -73,6 +73,7 @@ struct SMeta { TDB* pCtbIdx; TDB* pTagIdx; TDB* pTtlIdx; + TDB* pSmaIdx; SMetaIdx* pIdx; }; @@ -108,6 +109,11 @@ typedef struct { tb_uid_t uid; } STtlIdxKey; +typedef struct { + tb_uid_t uid; + int64_t smaUid; +} SSmaIdxKey; + #if 1 SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); @@ -118,7 +124,7 @@ int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); // SMetaDB int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); -// int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); +int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h new file mode 100644 index 0000000000..76a30f58dd --- /dev/null +++ b/source/dnode/vnode/src/inc/sma.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_SMA_H_ +#define _TD_VNODE_SMA_H_ + +#include "vnodeInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// smaDebug ================ +// clang-format off +#define smaFatal(...) do { if (smaDebugFlag & DEBUG_FATAL) { taosPrintLog("SMA FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) +// clang-format on + +typedef struct SSmaEnv SSmaEnv; +typedef struct SSmaStat SSmaStat; +typedef struct SSmaStatItem SSmaStatItem; +typedef struct SSmaKey SSmaKey; +typedef struct SRSmaInfo SRSmaInfo; + +#define SMA_IVLD_FID INT_MIN + +struct SSmaEnv { + TdThreadRwlock lock; + int8_t type; + TXN txn; + void *pPool; // SPoolMem + SDiskID did; + TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + char *path; // relative path + SSmaStat *pStat; +}; + +#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_TYPE(env) ((env)->type) +#define SMA_ENV_DID(env) ((env)->did) +#define SMA_ENV_ENV(env) ((env)->dbEnv) +#define SMA_ENV_PATH(env) ((env)->path) +#define SMA_ENV_STAT(env) ((env)->pStat) +#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) + +struct SSmaStatItem { + /** + * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service. + * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from + * Streaming Module or TSDB local persistence. + * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, + * without information about its previous state. + * - TSDB_SMA_STAT_DROPPED: 1)sma dropped + * N.B. only applicable to tsma + */ + int8_t state; // ETsdbSmaStat + SHashObj *expiredWindows; // key: skey of time window, value: N/A + STSma *pTSma; // cache schema +}; + +struct SSmaStat { + union { + SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem for tsma + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + }; + T_REF_DECLARE() +}; +#define SMA_STAT_ITEMS(s) ((s)->smaStatItems) +#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) + +struct SSmaKey { + TSKEY skey; + int64_t groupId; +}; + +typedef struct SDBFile SDBFile; + +struct SDBFile { + int32_t fid; + TDB *pDB; + char *path; +}; + +int32_t tdSmaBeginCommit(SSmaEnv *pEnv); +int32_t tdSmaEndCommit(SSmaEnv *pEnv); + +int32_t smaOpenDBEnv(TENV **ppEnv, const char *path); +int32_t smaCloseDBEnv(TENV *pEnv); +int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF); +int32_t smaCloseDBF(SDBFile *pDBF); +int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); +void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); + +void tdDestroySmaEnv(SSmaEnv *pSmaEnv); +void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); +#if 0 +int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result); +int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin); +#endif + +static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { + int32_t len = 0; + len += taosEncodeFixedI64(pData, tsKey); + len += taosEncodeFixedI64(pData, groupId); + return len; +} + +int32_t tdInitSma(SSma *pSma); +int32_t tdDropTSma(SSma *pSma, char *pMsg); +int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid); +int32_t tdInsertRSmaData(SSma *pSma, char *msg); + +int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); +int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); + +int32_t tdLockSma(SSma *pSma); +int32_t tdUnLockSma(SSma *pSma); + +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); + +static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); } +static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); } + +static FORCE_INLINE int32_t tdRLockSmaEnv(SSmaEnv *pEnv) { + int code = taosThreadRwlockRdlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tdWLockSmaEnv(SSmaEnv *pEnv) { + int code = taosThreadRwlockWrlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) { + int code = taosThreadRwlockUnlock(&(pEnv->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) { + if (pStatItem) { + return atomic_load_8(&pStatItem->state); + } + return TSDB_SMA_STAT_UNKNOWN; +} + +static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { + if (!pStatItem) { + return false; + } + + if (state) { + *state = atomic_load_8(&pStatItem->state); + return *state == TSDB_SMA_STAT_OK; + } + return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; +} + +static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; +} + +static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) { + return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; +} + +static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); + } +} + +static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); + } +} + +static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) { + if (pStatItem) { + atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); + } +} + +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); +void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem); +static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); +static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did); +static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv); + +void *tdFreeRSmaInfo(SRSmaInfo *pInfo); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_SMA_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 102c40337d..93a25da0a8 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -60,31 +60,22 @@ typedef struct { TSKEY minKey; } SRtn; -struct SSmaEnvs { - int16_t nTSma; - int16_t nRSma; - SSmaEnv *pTSmaEnv; - SSmaEnv *pRSmaEnv; -}; - +#define TSDB_DATA_DIR_LEN 6 struct STsdb { char *path; SVnode *pVnode; TdThreadMutex mutex; + char dir[TSDB_DATA_DIR_LEN]; bool repoLocked; - int8_t level; // retention level STsdbKeepCfg keepCfg; STsdbMemTable *mem; STsdbMemTable *imem; SRtn rtn; STsdbFS *fs; - SSmaEnvs smaEnvs; }; #if 1 // ====================================== -typedef struct SSmaStat SSmaStat; - struct STable { uint64_t tid; uint64_t uid; @@ -95,10 +86,6 @@ struct STable { #define TABLE_UID(t) (t)->uid int tsdbPrepareCommit(STsdb *pTsdb); -int32_t tsdbInitSma(STsdb *pTsdb); -int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg); -int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); -int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); typedef enum { TSDB_FILE_HEAD = 0, // .head TSDB_FILE_DATA, // .data @@ -107,8 +94,6 @@ typedef enum { TSDB_FILE_SMAL, // .smal(Block-wise SMA) TSDB_FILE_MAX, // TSDB_FILE_META, // meta - TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA - TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA } E_TSDB_FILE_T; typedef struct { @@ -186,15 +171,10 @@ struct STsdbFS { #define REPO_ID(r) TD_VID((r)->pVnode) #define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) #define REPO_KEEP_CFG(r) (&(r)->keepCfg) -#define REPO_LEVEL(r) ((r)->level) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_TFS(r) ((r)->pVnode->pTfs) #define IS_REPO_LOCKED(r) ((r)->repoLocked) -#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma) -#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma) -#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv) -#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv) int tsdbLockRepo(STsdb *pTsdb); int tsdbUnlockRepo(STsdb *pTsdb); @@ -794,25 +774,6 @@ typedef struct { } SFSHeader; // ================== TSDB File System Meta - -/** - * @brief Directory structure of .tsma data files. - * - * /vnode2/tsdb $ tree tsma/ - * tsma/ - * ├── v2f100.index_name_1 - * ├── v2f101.index_name_1 - * ├── v2f102.index_name_1 - * ├── v2f1900.index_name_3 - * ├── v2f1901.index_name_3 - * ├── v2f1902.index_name_3 - * ├── v2f200.index_name_2 - * ├── v2f201.index_name_2 - * └── v2f202.index_name_2 - * - * 0 directories, 9 files - */ - #define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus) #define FS_NEW_STATUS(pfs) ((pfs)->nstatus) #define FS_IN_TXN(pfs) (pfs)->intxn @@ -874,43 +835,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } -typedef struct SSmaKey SSmaKey; - -struct SSmaKey { - TSKEY skey; - int64_t groupId; -}; - -typedef struct SDBFile SDBFile; - -struct SDBFile { - int32_t fid; - TDB *pDB; - char *path; -}; - -int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path); -int32_t tsdbCloseDBEnv(TENV *pEnv); -int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF); -int32_t tsdbCloseDBF(SDBFile *pDBF); -int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); -void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); - -void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); -void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); -#if 0 -int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); -int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); -#endif - -// internal func -static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { - int32_t len = 0; - len += taosEncodeFixedI64(pData, tsKey); - len += taosEncodeFixedI64(pData, groupId); - return len; -} - #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9a36fc6eae..2018f0a68c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -47,13 +47,15 @@ extern "C" { #endif -typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; -typedef struct STsdb STsdb; -typedef struct STQ STQ; -typedef struct SVState SVState; -typedef struct SVBufPool SVBufPool; -typedef struct SQWorker SQHandle; +typedef struct SVnodeInfo SVnodeInfo; +typedef struct SMeta SMeta; +typedef struct SSma SSma; +typedef struct STsdb STsdb; +typedef struct STQ STQ; +typedef struct SVState SVState; +typedef struct SVBufPool SVBufPool; +typedef struct SQWorker SQHandle; +typedef struct STsdbKeepCfg STsdbKeepCfg; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -77,9 +79,11 @@ int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta); int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); +int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -90,17 +94,14 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); -int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); +int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); // tsdb -int tsdbOpen(SVnode* pVnode, int8_t type); -int tsdbClose(STsdb* pTsdb); +int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); +int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); -int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); -int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); -int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, @@ -121,13 +122,31 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); // sma +int32_t smaOpen(SVnode* pVnode); +int32_t smaClose(SSma* pSma); +int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version); +int32_t tdProcessTSmaCreate(SSma* pSma, char* pMsg); +int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); + +int32_t tdProcessRSmaCreate(SSma* pSma, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb); +int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); +int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore); +void tdUidStoreDestory(STbUidStore* pStore); +void* tdUidStoreFree(STbUidStore* pStore); + +#if 0 +int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version); +int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); +int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb); int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); +#endif typedef struct { int8_t streamType; // sma or other @@ -164,13 +183,13 @@ typedef enum { TSDB_TYPE_RSMA_L2 = 4, // RSMA Level 2 } ETsdbType; -typedef struct { +struct STsdbKeepCfg { int8_t precision; // precision always be used with below keep cfgs int32_t days; int32_t keep0; int32_t keep1; int32_t keep2; -} STsdbKeepCfg; +}; struct SVnode { char* path; @@ -183,9 +202,8 @@ struct SVnode { SVBufPool* onCommit; SVBufPool* onRecycle; SMeta* pMeta; + SSma* pSma; STsdb* pTsdb; - STsdb* pRSma1; - STsdb* pRSma2; SWal* pWal; STQ* pTq; SSink* pSink; @@ -194,10 +212,12 @@ struct SVnode { SQHandle* pQuery; }; +#define TD_VID(PVNODE) (PVNODE)->config.vgId + #define VND_TSDB(vnd) ((vnd)->pTsdb) #define VND_RSMA0(vnd) ((vnd)->pTsdb) -#define VND_RSMA1(vnd) ((vnd)->pRSma1) -#define VND_RSMA2(vnd) ((vnd)->pRSma2) +#define VND_RSMA1(vnd) ((vnd)->pSma->pRSmaTsdb1) +#define VND_RSMA2(vnd) ((vnd)->pSma->pRSmaTsdb2) #define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions) struct STbUidStore { @@ -207,7 +227,29 @@ struct STbUidStore { SHashObj* uidHash; }; -#define TD_VID(PVNODE) (PVNODE)->config.vgId +struct SSma { + int16_t nTSma; + bool locked; + TdThreadMutex mutex; + SVnode* pVnode; + STsdb* pRSmaTsdb1; + STsdb* pRSmaTsdb2; + void* pTSmaEnv; + void* pRSmaEnv; +}; + +#define SMA_CFG(s) (&(s)->pVnode->config) +#define SMA_TSDB_CFG(s) (&(s)->pVnode->config.tsdbCfg) +#define SMA_LOCKED(s) ((s)->locked) +#define SMA_META(s) ((s)->pVnode->pMeta) +#define SMA_VID(s) TD_VID((s)->pVnode) +#define SMA_TFS(s) ((s)->pVnode->pTfs) +#define SMA_TSMA_NUM(s) ((s)->nTSma) +#define SMA_TSMA_ENV(s) ((s)->pTSmaEnv) +#define SMA_RSMA_ENV(s) ((s)->pRSmaEnv) +#define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb) +#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1) +#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2) static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 581b876e84..2bc0d7517d 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -34,7 +34,10 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { } else if (pME->type == TSDB_NORMAL_TABLE) { if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; + if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; + } else if (pME->type == TSDB_TSMA_TABLE) { + if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { ASSERT(0); } @@ -63,7 +66,10 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { } else if (pME->type == TSDB_NORMAL_TABLE) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; + if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; + } else if (pME->type == TSDB_TSMA_TABLE) { + if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 853b2ecefb..3f52071315 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -112,35 +112,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { #endif // TODO return 0; -} - -int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) { - // TODO: Validate the cfg - // The table uid should exists and be super table or common table. - // Check other cfg value - - // TODO: add atomicity - -#ifdef META_REFACT -#else - if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) { - // TODO: handle error - return -1; - } -#endif - return TSDB_CODE_SUCCESS; -} - -int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { - // TODO: Validate the cfg - // TODO: add atomicity - -#ifdef META_REFACT -#else - if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { - // TODO: handle error - return -1; - } -#endif - return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 07422e3193..3ce146904c 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -21,6 +21,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); } static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); } @@ -104,6 +105,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { goto _err; } + // open pSmaIdx + ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); + if (ret < 0) { + metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open index if (metaOpenIdx(pMeta) < 0) { metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); @@ -117,11 +125,12 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { _err: if (pMeta->pIdx) metaCloseIdx(pMeta); + if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); - if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); + if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); @@ -133,11 +142,12 @@ _err: int metaClose(SMeta *pMeta) { if (pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); + if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); - if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); + if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); @@ -295,3 +305,22 @@ static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return 0; } + +static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + SSmaIdxKey *pSmaIdxKey1 = (SSmaIdxKey *)pKey1; + SSmaIdxKey *pSmaIdxKey2 = (SSmaIdxKey *)pKey2; + + if (pSmaIdxKey1->uid > pSmaIdxKey2->uid) { + return 1; + } else if (pSmaIdxKey1->uid < pSmaIdxKey2->uid) { + return -1; + } + + if (pSmaIdxKey1->smaUid > pSmaIdxKey2->smaUid) { + return 1; + } else if (pSmaIdxKey1->smaUid < pSmaIdxKey2->smaUid) { + return -1; + } + + return 0; +} diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c new file mode 100644 index 0000000000..8ce7ea5895 --- /dev/null +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "meta.h" + +static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME); +static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME); + +int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { + // TODO: Validate the cfg + // The table uid should exists and be super table or normal table. + // Check other cfg value + + SMetaEntry me = {0}; + int kLen = 0; + int vLen = 0; + const void *pKey = NULL; + const void *pVal = NULL; + void *pBuf = NULL; + int32_t szBuf = 0; + void *p = NULL; + SMetaReader mr = {0}; + + // validate req + metaReaderInit(&mr, pMeta, 0); + if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) { +// TODO: just for pass case +#if 1 + terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST; + metaReaderClear(&mr); + return -1; +#else + metaReaderClear(&mr); + return 0; +#endif + } + metaReaderClear(&mr); + + // set structs + me.version = version; + me.type = TSDB_TSMA_TABLE; + me.uid = pCfg->indexUid; + me.name = pCfg->indexName; + me.smaEntry.tsma = pCfg; + + if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err; + + metaDebug("vgId:%d tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid); + + return 0; + +_err: + metaError("vgId:%d failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, + pCfg->indexUid, tstrerror(terrno)); + return -1; +} + +int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { + // TODO: Validate the cfg + // TODO: add atomicity + +#ifdef META_REFACT +#else + if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) { + // TODO: handle error + return -1; + } +#endif + return TSDB_CODE_SUCCESS; +} + +// static int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { +// int32_t ret = 0; +// void *pBuf = NULL, *qBuf = NULL; +// void *key = {0}, *val = {0}; + +// // save sma info +// int32_t len = tEncodeTSma(NULL, pSmaCfg); +// pBuf = taosMemoryCalloc(1, len); +// if (pBuf == NULL) { +// terrno = TSDB_CODE_OUT_OF_MEMORY; +// return -1; +// } + +// key = (void *)&pSmaCfg->indexUid; +// qBuf = pBuf; +// tEncodeTSma(&qBuf, pSmaCfg); +// val = pBuf; + +// int32_t kLen = sizeof(pSmaCfg->indexUid); +// int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); + +// ret = tdbDbInsert(pMeta->pTbDb, key, kLen, val, vLen, &pMeta->txn); +// if (ret < 0) { +// taosMemoryFreeClear(pBuf); +// return -1; +// } + +// // add sma idx +// SSmaIdxKey smaIdxKey; +// smaIdxKey.uid = pSmaCfg->tableUid; +// smaIdxKey.smaUid = pSmaCfg->indexUid; +// key = &smaIdxKey; +// kLen = sizeof(smaIdxKey); +// val = NULL; +// vLen = 0; + +// ret = tdbDbInsert(pMeta->pSmaIdx, key, kLen, val, vLen, &pMeta->txn); +// if (ret < 0) { +// taosMemoryFreeClear(pBuf); +// return -1; +// } + +// // release +// taosMemoryFreeClear(pBuf); + +// return 0; +// } + + +static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { + STbDbKey tbDbKey; + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + SEncoder coder = {0}; + + // set key and value + tbDbKey.version = pME->version; + tbDbKey.uid = pME->uid; + + pKey = &tbDbKey; + kLen = sizeof(tbDbKey); + + int32_t ret = 0; + tEncodeSize(metaEncodeEntry, pME, vLen, ret); + if (ret < 0) { + goto _err; + } + + pVal = taosMemoryMalloc(vLen); + if (pVal == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tEncoderInit(&coder, pVal, vLen); + + if (metaEncodeEntry(&coder, pME) < 0) { + goto _err; + } + + tEncoderClear(&coder); + + // write to table.db + if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + goto _err; + } + + taosMemoryFree(pVal); + return 0; + +_err: + taosMemoryFree(pVal); + return -1; +} + +static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); +} + +static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { + SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; + + return tdbDbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); +} + +static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { + metaWLock(pMeta); + + // save to table.db + if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err; + + // // update uid.idx + if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err; + + if (metaUpdateSmaIdx(pMeta, pME) < 0) goto _err; + + metaULock(pMeta); + return 0; + +_err: + metaULock(pMeta); + return -1; +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index d666bd22c1..e61064fe67 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -131,6 +131,75 @@ _err: return -1; } +int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + TDBC *pUidIdxc = NULL; + TDBC *pTbDbc = NULL; + const void *pData; + int nData; + int64_t oversion; + SDecoder dc = {0}; + int32_t ret; + int32_t c; + + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + if (ret < 0 || c) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + oversion = *(int64_t *)pData; + + tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); + ASSERT(ret == 0 && c == 0); + + ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ASSERT(ret == 0); + + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &oStbEntry); + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = pReq->suid; + nStbEntry.name = pReq->name; + nStbEntry.stbEntry.schema = pReq->schema; + nStbEntry.stbEntry.schemaTag = pReq->schemaTag; + + metaWLock(pMeta); + // compare two entry + if (oStbEntry.stbEntry.schema.sver != pReq->schema.sver) { + if (oStbEntry.stbEntry.schema.nCols != pReq->schema.nCols) { + metaSaveToSkmDb(pMeta, &nStbEntry); + } + } + + // if (oStbEntry.stbEntry.schemaTag.sver != pReq->schemaTag.sver) { + // // change tag schema + // } + + // update table.db + metaSaveToTbDb(pMeta, &nStbEntry); + + // update uid index + tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); + + metaULock(pMeta); + tDecoderClear(&dc); + tdbDbcClose(pTbDbc); + tdbDbcClose(pUidIdxc); + return 0; +} + int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { SMetaEntry me = {0}; SMetaReader mr = {0}; @@ -171,6 +240,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { me.ntbEntry.ctime = pReq->ctime; me.ntbEntry.ttlDays = pReq->ttl; me.ntbEntry.schema = pReq->ntb.schema; + me.ntbEntry.ncid = me.ntbEntry.schema.pSchema[me.ntbEntry.schema.nCols - 1].colId + 1; } if (metaHandleEntry(pMeta, &me) < 0) goto _err; @@ -305,6 +375,170 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { return 0; } +static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + void *pVal = NULL; + int nVal = 0; + const void *pData = NULL; + int nData = 0; + int ret = 0; + tb_uid_t uid; + int64_t oversion; + SSchema *pColumn = NULL; + SMetaEntry entry = {0}; + SSchemaWrapper *pSchema; + int c; + + // search name index + ret = tdbDbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + return -1; + } + + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + // search uid index + TDBC *pUidIdxc = NULL; + + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + ASSERT(c == 0); + + tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + oversion = *(int64_t *)pData; + + // search table.db + TDBC *pTbDbc = NULL; + + tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + ASSERT(c == 0); + tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + + // get table entry + SDecoder dc = {0}; + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &entry); + + if (entry.type != TSDB_NORMAL_TABLE) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + + // search the column to add/drop/update + pSchema = &entry.ntbEntry.schema; + int32_t iCol = 0; + for (;;) { + pColumn = NULL; + + if (iCol >= pSchema->nCols) break; + pColumn = &pSchema->pSchema[iCol]; + + if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; + iCol++; + } + + entry.version = version; + int tlen; + switch (pAlterTbReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + if (pColumn) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + pSchema->sver++; + pSchema->nCols++; + pSchema->pSchema = + taosMemoryRealloc(entry.ntbEntry.schema.pSchema, sizeof(SSchema) * entry.ntbEntry.schema.nCols); + pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes; + pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type; + pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags; + pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].colId = entry.ntbEntry.ncid++; + strcpy(pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].name, pAlterTbReq->colName); + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; + goto _err; + } + if (pColumn->colId == 0) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + pSchema->sver++; + pSchema->nCols--; + tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); + if (tlen) { + memmove(pColumn, pColumn + 1, tlen); + } + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; + goto _err; + } + if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes <= pAlterTbReq->bytes) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + pSchema->sver++; + pColumn->bytes = pAlterTbReq->bytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; + goto _err; + } + pSchema->sver++; + strcpy(pColumn->name, pAlterTbReq->colNewName); + break; + } + + entry.version = version; + + tDecoderClear(&dc); + tdbDbcClose(pTbDbc); + tdbDbcClose(pUidIdxc); + return 0; + +_err: + tDecoderClear(&dc); + tdbDbcClose(pTbDbc); + tdbDbcClose(pUidIdxc); + return -1; +} + +static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + // TODO + return 0; +} + +static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + // TODO + ASSERT(0); + return 0; +} + +int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + switch (pReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_DROP_COLUMN: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + return metaAlterTableColumn(pMeta, version, pReq); + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: + return metaUpdateTableTagVal(pMeta, version, pReq); + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + return metaUpdateTableOptions(pMeta, version, pReq); + default: + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + return -1; + break; + } +} + static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; void *pKey = NULL; diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/sma/sma.c similarity index 50% rename from source/dnode/vnode/src/inc/tsdbSma.h rename to source/dnode/vnode/src/sma/sma.c index 5215812ac5..2c54e10087 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/sma/sma.c @@ -13,35 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_VNODE_TSDB_SMA_H_ -#define _TD_VNODE_TSDB_SMA_H_ +#include "sma.h" -#include "tsdb.h" -#ifdef __cplusplus -extern "C" { -#endif +// TODO: Who is responsible for resource allocate and release? +int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) { + int32_t code = TSDB_CODE_SUCCESS; -// typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2); - -// struct STbDdlH { -// void *ahandle; -// void *result; -// __tb_ddl_fn_t fp; -// }; - -static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { - ASSERT(*pStore == NULL); - *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); - if (*pStore == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { + smaWarn("vgId:%d insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); } - return TSDB_CODE_SUCCESS; + // TODO: destroy SSDataBlocks(msg) + return code; } -#ifdef __cplusplus -} -#endif -#endif /*_TD_VNODE_TSDB_SMA_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c new file mode 100644 index 0000000000..c02276f5fe --- /dev/null +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -0,0 +1,463 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +typedef struct SSmaStat SSmaStat; + +static const char *TSDB_SMA_DNAME[] = { + "", // TSDB_SMA_TYPE_BLOCK + "tsma", // TSDB_SMA_TYPE_TIME_RANGE + "rsma", // TSDB_SMA_TYPE_ROLLUP +}; + +#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test +#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test +#define SMA_STATE_HASH_SLOT 4 + +#define RSMA_TASK_INFO_HASH_SLOT 8 + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +// declaration of static functions + +// insert data + +static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); + +// Pool Memory +static SPoolMem *openPool(); +static void clearPool(SPoolMem *pPool); +static void closePool(SPoolMem *pPool); +static void *poolMalloc(void *arg, size_t size); +static void poolFree(void *arg, void *ptr); + +// implementation + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + if (!pPool) return; + + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + if (pPool) { + clearPool(pPool); + taosMemoryFree(pPool); + } +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (!pMem) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +int32_t tdInitSma(SSma *pSma) { + // tSma + int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma), false)); + if (numOfTSma > 0) { + atomic_store_16(&SMA_TSMA_NUM(pSma), (int16_t)numOfTSma); + } + // TODO: rSma + return TSDB_CODE_SUCCESS; +} + +static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]); +} + +static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did) { + SSmaEnv *pEnv = NULL; + + pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); + if (!pEnv) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMA_ENV_TYPE(pEnv) = smaType; + + int code = taosThreadRwlockInit(&(pEnv->lock), NULL); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); + taosMemoryFree(pEnv); + return NULL; + } + + ASSERT(path && (strlen(path) > 0)); + SMA_ENV_PATH(pEnv) = strdup(path); + if (!SMA_ENV_PATH(pEnv)) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + SMA_ENV_DID(pEnv) = did; + + if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + char aname[TSDB_FILENAME_LEN] = {0}; + tfsAbsoluteName(SMA_TFS(pSma), did, path, aname); + if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + if (!(pEnv->pPool = openPool())) { + tdFreeSmaEnv(pEnv); + return NULL; + } + + return pEnv; +} + +static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) { + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + if (!(*pEnv)) { + if (!(*pEnv = tdNewSmaEnv(pSma, smaType, path, did))) { + return TSDB_CODE_FAILED; + } + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Release resources allocated for its member fields, not including itself. + * + * @param pSmaEnv + * @return int32_t + */ +void tdDestroySmaEnv(SSmaEnv *pSmaEnv) { + if (pSmaEnv) { + tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); + taosMemoryFreeClear(pSmaEnv->pStat); + taosMemoryFreeClear(pSmaEnv->path); + taosThreadRwlockDestroy(&(pSmaEnv->lock)); + smaCloseDBEnv(pSmaEnv->dbEnv); + closePool(pSmaEnv->pPool); + } +} + +void *tdFreeSmaEnv(SSmaEnv *pSmaEnv) { + tdDestroySmaEnv(pSmaEnv); + taosMemoryFreeClear(pSmaEnv); + return NULL; +} + +int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) { + if (!pStat) return 0; + + int ref = T_REF_INC(pStat); + smaDebug("vgId:%d ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); + return 0; +} + +int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { + if (!pStat) return 0; + + int ref = T_REF_DEC(pStat); + smaDebug("vgId:%d unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); + return 0; +} + +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { + ASSERT(pSmaStat != NULL); + + if (*pSmaStat) { // no lock + return TSDB_CODE_SUCCESS; + } + + /** + * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew). + * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if + * tdInitSmaStat invoked in other multithread environment later. + */ + if (!(*pSmaStat)) { + *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); + if (!(*pSmaStat)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + if (smaType == TSDB_SMA_TYPE_ROLLUP) { + SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + + if (!SMA_STAT_INFO_HASH(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + SMA_STAT_ITEMS(*pSmaStat) = + taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + if (!SMA_STAT_ITEMS(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + } else { + ASSERT(0); + } + } + return TSDB_CODE_SUCCESS; +} + +void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { + if (pSmaStatItem) { + tdDestroyTSma(pSmaStatItem->pTSma); + taosMemoryFreeClear(pSmaStatItem->pTSma); + taosHashCleanup(pSmaStatItem->expiredWindows); + taosMemoryFreeClear(pSmaStatItem); + } + return NULL; +} + +/** + * @brief Release resources allocated for its member fields, not including itself. + * + * @param pSmaStat + * @return int32_t + */ +int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { + if (pSmaStat) { + // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. + if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL); + while (item) { + SSmaStatItem *pItem = *(SSmaStatItem **)item; + tdFreeSmaStatItem(pItem); + item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item); + } + taosHashCleanup(SMA_STAT_ITEMS(pSmaStat)); + } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { + void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL); + while (infoHash) { + SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; + tdFreeRSmaInfo(pInfoHash); + infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash); + } + taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat)); + } else { + ASSERT(0); + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdLockSma(SSma *pSma) { + int code = taosThreadMutexLock(&pSma->mutex); + if (code != 0) { + smaError("vgId:%d failed to lock td since %s", SMA_VID(pSma), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pSma->locked = true; + return 0; +} + +int32_t tdUnLockSma(SSma *pSma) { + ASSERT(SMA_LOCKED(pSma)); + pSma->locked = false; + int code = taosThreadMutexUnlock(&pSma->mutex); + if (code != 0) { + smaError("vgId:%d failed to unlock td since %s", SMA_VID(pSma), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { + SSmaEnv *pEnv = NULL; + + // return if already init + switch (smaType) { + case TSDB_SMA_TYPE_TIME_RANGE: + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_TSMA_ENV(pSma)))) { + return TSDB_CODE_SUCCESS; + } + break; + case TSDB_SMA_TYPE_ROLLUP: + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_RSMA_ENV(pSma)))) { + return TSDB_CODE_SUCCESS; + } + break; + default: + TASSERT(0); + return TSDB_CODE_FAILED; + } + + // init sma env + tdLockSma(pSma); + pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&SMA_TSMA_ENV(pSma)) + : atomic_load_ptr(&SMA_RSMA_ENV(pSma)); + if (!pEnv) { + char rname[TSDB_FILENAME_LEN] = {0}; + + SDiskID did = {0}; + if (tfsAllocDisk(SMA_TFS(pSma), TFS_PRIMARY_LEVEL, &did) < 0) { + tdUnLockSma(pSma); + return TSDB_CODE_FAILED; + } + + if (did.level < 0 || did.id < 0) { + tdUnLockSma(pSma); + smaError("vgId:%d init sma env failed since invalid did(%d,%d)", SMA_VID(pSma), did.level, did.id); + return TSDB_CODE_FAILED; + } + + tdGetSmaDir(SMA_VID(pSma), smaType, rname); + + if (tfsMkdirRecurAt(SMA_TFS(pSma), rname, did) < 0) { + tdUnLockSma(pSma); + return TSDB_CODE_FAILED; + } + + if (tdInitSmaEnv(pSma, smaType, rname, did, &pEnv) < 0) { + tdUnLockSma(pSma); + return TSDB_CODE_FAILED; + } + + (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), pEnv) + : atomic_store_ptr(&SMA_RSMA_ENV(pSma), pEnv); + } + tdUnLockSma(pSma); + + return TSDB_CODE_SUCCESS; +}; + +int32_t tdSmaBeginCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + // start a new txn + tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (tdbBegin(pEnv->dbEnv, pTxn) != 0) { + smaWarn("tdSma tdb begin commit fail"); + return -1; + } + return 0; +} + +int32_t tdSmaEndCommit(SSmaEnv *pEnv) { + TXN *pTxn = &pEnv->txn; + + // Commit current txn + if (tdbCommit(pEnv->dbEnv, pTxn) != 0) { + smaWarn("tdSma tdb end commit fail"); + return -1; + } + tdbTxnClose(pTxn); + clearPool(pEnv->pPool); + return 0; +} + +#if 0 +/** + * @brief Get the start TS key of the last data block of one interval/sliding. + * + * @param pSma + * @param param + * @param result + * @return int32_t + * 1) Return 0 and fill the result if the check procedure is normal; + * 2) Return -1 if error occurs during the check procedure. + */ +int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) { + const char *procedure = ""; + if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) { + return -1; + } + // fill the result + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Remove the tSma data files related to param between pWin. + * + * @param pSma + * @param param + * @param pWin + * @return int32_t + */ +int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) { + // for ("tSmaFiles of param-interval-sliding between pWin") { + // // remove the tSmaFile + // } + return TSDB_CODE_SUCCESS; +} +#endif diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c new file mode 100644 index 0000000000..1c7db28e18 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +static int32_t smaEvalDays(SRetention *r, int8_t precision); +static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type); + +#define SMA_SET_KEEP_CFG(l) \ + do { \ + SRetention *r = &pCfg->retentions[l]; \ + pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ + pKeepCfg->keep0 = pKeepCfg->keep2; \ + pKeepCfg->keep1 = pKeepCfg->keep2; \ + pKeepCfg->days = smaEvalDays(r, pCfg->precision); \ + } while (0) + +#define SMA_OPEN_RSMA_IMPL(v, l) \ + do { \ + SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ + if (!RETENTION_VALID(r)) { \ + if (l == 0) { \ + goto _err; \ + } \ + break; \ + } \ + smaSetKeepCfg(&keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg) < 0) { \ + goto _err; \ + } \ + } while (0) + +#define RETENTION_DAYS_SPLIT_RATIO 10 +#define RETENTION_DAYS_SPLIT_MIN 1 +#define RETENTION_DAYS_SPLIT_MAX 30 + +static int32_t smaEvalDays(SRetention *r, int8_t precision) { + int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY); + int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY); + + int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO; + if (days <= RETENTION_DAYS_SPLIT_MIN) { + days = RETENTION_DAYS_SPLIT_MIN; + if (days < freqDays) { + days = freqDays + 1; + } + } else { + if (days > RETENTION_DAYS_SPLIT_MAX) { + days = RETENTION_DAYS_SPLIT_MAX; + } + if (days < freqDays) { + days = freqDays + 1; + } + } + return days * 1440; +} + +int smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) { + pKeepCfg->precision = pCfg->precision; + switch (type) { + case TSDB_TYPE_TSMA: + ASSERT(0); + break; + case TSDB_TYPE_RSMA_L0: + SMA_SET_KEEP_CFG(0); + break; + case TSDB_TYPE_RSMA_L1: + SMA_SET_KEEP_CFG(1); + break; + case TSDB_TYPE_RSMA_L2: + SMA_SET_KEEP_CFG(2); + break; + default: + ASSERT(0); + break; + } + return 0; +} + +int32_t smaOpen(SVnode *pVnode) { + STsdbCfg *pCfg = &pVnode->config.tsdbCfg; + + ASSERT(!pVnode->pSma); + + SSma *pSma = taosMemoryCalloc(1, sizeof(SSma)); + if (!pSma) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pSma->pVnode = pVnode; + taosThreadMutexInit(&pSma->mutex, NULL); + pSma->locked = false; + + if (vnodeIsRollup(pVnode)) { + STsdbKeepCfg keepCfg = {0}; + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + if (i == TSDB_RETENTION_L0) { + SMA_OPEN_RSMA_IMPL(pVnode, 0); + } else if (i == TSDB_RETENTION_L1) { + SMA_OPEN_RSMA_IMPL(pVnode, 1); + } else if (i == TSDB_RETENTION_L2) { + SMA_OPEN_RSMA_IMPL(pVnode, 2); + } else { + ASSERT(0); + } + } + } + + pVnode->pSma = pSma; + return 0; +_err: + taosMemoryFreeClear(pSma); + return -1; +} + +int32_t smaClose(SSma *pSma) { + if (pSma) { + taosThreadMutexDestroy(&pSma->mutex); + if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma)); + if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma)); + if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c new file mode 100644 index 0000000000..f9cb5a1a09 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); +static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo, + STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level); + +struct SRSmaInfo { + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t +}; + +static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) { + // Note: free/kill may in RC + qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); + if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + qDestroyTask(otaskHandle); + } +} + +void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { + if (pInfo->taskInfo[i]) { + tdFreeTaskHandle(pInfo->taskInfo[i]); + } + } + return NULL; +} + +static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { + ASSERT(*pStore == NULL); + *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); + if (*pStore == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + if (!suid || !tbUids) { + terrno = TSDB_CODE_INVALID_PTR; + smaError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaError("vgId:%d failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + return TSDB_CODE_FAILED; + } + + if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) { + smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } + + if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) { + smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) { + if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { + return TSDB_CODE_SUCCESS; + } + + if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } + + void *pIter = taosHashIterate(pStore->uidHash, NULL); + while (pIter) { + tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + SArray *pTbUids = *(SArray **)pIter; + + if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) { + taosHashCancelIterate(pStore->uidHash, pIter); + return TSDB_CODE_FAILED; + } + + pIter = taosHashIterate(pStore->uidHash, pIter); + } + return TSDB_CODE_SUCCESS; +} + +/** + * @brief fetch suid/uids when create child tables of rollup SMA + * + * @param pTsdb + * @param ppStore + * @param suid + * @param uid + * @return int32_t + */ +int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + + // only applicable to rollup SMA ctables + if (!pEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SHashObj *infoHash = NULL; + if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + return TSDB_CODE_FAILED; + } + + // info cached when create rsma stable and return directly for non-rsma ctables + if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) { + return TSDB_CODE_SUCCESS; + } + + ASSERT(ppStore != NULL); + + if (!(*ppStore)) { + if (tdUidStoreInit(ppStore) != 0) { + return TSDB_CODE_FAILED; + } + } + + if (tdUidStorePut(*ppStore, suid, &uid) != 0) { + *ppStore = tdUidStoreFree(*ppStore); + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * + * @param pTsdb + * @param pMeta + * @param pReq + * @return int32_t + */ +int32_t tdProcessRSmaCreate(SSma *pSma, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) { + if (!pReq->rollup) { + smaTrace("vgId:%d return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + SRSmaParam *param = &pReq->pRSmaParam; + + if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { + smaWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + if (pRSmaInfo) { + smaWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + return TSDB_CODE_SUCCESS; + } + + pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo)); + if (!pRSmaInfo) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta); + if (!pReadHandle) { + taosMemoryFree(pRSmaInfo); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + SReadHandle handle = { + .reader = pReadHandle, + .meta = pMeta, + .pMsgCb = pMsgCb, + }; + + if (param->qmsg1) { + pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle); + if (!pRSmaInfo->taskInfo[0]) { + taosMemoryFree(pRSmaInfo); + taosMemoryFree(pReadHandle); + return TSDB_CODE_FAILED; + } + } + + if (param->qmsg2) { + pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle); + if (!pRSmaInfo->taskInfo[1]) { + taosMemoryFree(pRSmaInfo); + taosMemoryFree(pReadHandle); + return TSDB_CODE_FAILED; + } + } + + if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != + TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief store suid/[uids], prefer to use array and then hash + * + * @param pStore + * @param suid + * @param uid + * @return int32_t + */ +static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) { + // prefer to store suid/uids in array + if ((suid == pStore->suid) || (pStore->suid == 0)) { + if (pStore->suid == 0) { + pStore->suid = suid; + } + if (uid) { + if (!pStore->tbUids) { + if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + if (!taosArrayPush(pStore->tbUids, uid)) { + return TSDB_CODE_FAILED; + } + } + } else { + // store other suid/uids in hash when multiple stable/table included in 1 batch of request + if (!pStore->uidHash) { + pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + if (!pStore->uidHash) { + return TSDB_CODE_FAILED; + } + } + if (uid) { + SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t)); + if (uidArray && ((uidArray = *(SArray **)uidArray))) { + taosArrayPush(uidArray, uid); + } else { + SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t)); + if (!pUidArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + if (!taosArrayPush(pUidArray, uid)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) { + return TSDB_CODE_FAILED; + } + } + } else { + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) { + return TSDB_CODE_FAILED; + } + } + } + return TSDB_CODE_SUCCESS; +} + +void tdUidStoreDestory(STbUidStore *pStore) { + if (pStore) { + if (pStore->uidHash) { + if (pStore->tbUids) { + // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys. + void *pIter = taosHashIterate(pStore->uidHash, NULL); + while (pIter) { + SArray *arr = *(SArray **)pIter; + taosArrayDestroy(arr); + pIter = taosHashIterate(pStore->uidHash, pIter); + } + } + taosHashCleanup(pStore->uidHash); + } + taosArrayDestroy(pStore->tbUids); + } +} + +void *tdUidStoreFree(STbUidStore *pStore) { + if (pStore) { + tdUidStoreDestory(pStore); + taosMemoryFree(pStore); + } + return NULL; +} + +static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if (!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { + ASSERT(pMsg != NULL); + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + + terrno = TSDB_CODE_SUCCESS; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + + if (!pBlock) break; + tdUidStorePut(pStore, msgIter.suid, NULL); + pStore->uid = msgIter.uid; // TODO: remove, just for debugging + } + + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; +} + +static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo, + STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level) { + SArray *pResult = NULL; + + if (!taskInfo) { + smaDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); + return TSDB_CODE_SUCCESS; + } + + smaDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, taskInfo, suid); + + qSetStreamInput(taskInfo, pMsg, inputType); + while (1) { + SSDataBlock *output = NULL; + uint64_t ts; + if (qExecTask(taskInfo, &output, &ts) < 0) { + ASSERT(false); + } + if (!output) { + break; + } + if (!pResult) { + pResult = taosArrayInit(0, sizeof(SSDataBlock)); + if (!pResult) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + + taosArrayPush(pResult, output); + } + + if (taosArrayGetSize(pResult) > 0) { + blockDebugShowData(pResult); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); + SSubmitReq *pReq = NULL; + if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), uid, suid) != 0) { + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; + } + if (tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { + taosArrayDestroy(pResult); + taosMemoryFreeClear(pReq); + return TSDB_CODE_FAILED; + } + taosMemoryFreeClear(pReq); + } else { + smaWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno)); + } + + taosArrayDestroy(pResult); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { + // only applicable when rsma env exists + return TSDB_CODE_SUCCESS; + } + + ASSERT(uid != 0); // TODO: remove later + + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaDebug("vgId:%d no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); + return TSDB_CODE_SUCCESS; + } + if (!pRSmaInfo->taskInfo[0]) { + smaDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); + return TSDB_CODE_SUCCESS; + } + + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + // TODO: use the proper schema instead of 0, and cache STSchema in cache + STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 0); + if (!pTSchema) { + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + return TSDB_CODE_FAILED; + } + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); + taosMemoryFree(pTSchema); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { + // only applicable when rsma env exists + return TSDB_CODE_SUCCESS; + } + + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + STbUidStore uidStore = {0}; + tdFetchSubmitReqSuids(pMsg, &uidStore); + + if (uidStore.suid != 0) { + tdExecuteRSma(pSma, pMsg, inputType, uidStore.suid, uidStore.uid); + + void *pIter = taosHashIterate(uidStore.uidHash, NULL); + while (pIter) { + tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + tdExecuteRSma(pSma, pMsg, inputType, *pTbSuid, 0); + pIter = taosHashIterate(uidStore.uidHash, pIter); + } + + tdUidStoreDestory(&uidStore); + } + } + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c new file mode 100644 index 0000000000..821ec44aa5 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTDBImpl.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include "sma.h" + +int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { + int ret = 0; + + if (path == NULL) return -1; + + ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + + if (ret != 0) { + smaError("failed to create tsdb db env, ret = %d", ret); + return -1; + } + + return 0; +} + +int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } + +static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { + const SSmaKey *pKey1 = (const SSmaKey *)arg1; + const SSmaKey *pKey2 = (const SSmaKey *)arg2; + + ASSERT(len1 == len2 && len1 == sizeof(SSmaKey)); + + if (pKey1->skey < pKey2->skey) { + return -1; + } else if (pKey1->skey > pKey2->skey) { + return 1; + } + if (pKey1->groupId < pKey2->groupId) { + return -1; + } else if (pKey1->groupId > pKey2->groupId) { + return 1; + } + + return 0; +} + +static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { + int ret; + tdb_cmpr_fn_t compFunc; + + // Create a database + compFunc = tdSmaKeyCmpr; + ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); + + return 0; +} + +static int32_t smaCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); } + +int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { + // TEnv is shared by a group of SDBFile + if (!pEnv || !pDBF) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } + + // Open DBF + if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + smaCloseDBDb(pDBF->pDB); + return -1; + } + + return 0; +} + +int32_t smaCloseDBF(SDBFile *pDBF) { + int32_t ret = 0; + if (pDBF->pDB) { + ret = smaCloseDBDb(pDBF->pDB); + pDBF->pDB = NULL; + } + taosMemoryFreeClear(pDBF->path); + return ret; +} + +int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { + int32_t ret; + + ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + if (ret < 0) { + smaError("failed to create insert sma data into db, ret = %d", ret); + return -1; + } + + return 0; +} + +void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) { + void *pVal = NULL; + int ret; + + ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + + if (ret < 0) { + smaError("failed to get sma data from db, ret = %d", ret); + return NULL; + } + + ASSERT(*valLen >= 0); + + // TODO: lock? + // TODO: Would the key/value be destoryed during return the data? + // TODO: How about the key is updated while value length is changed? The original value buffer would be freed + // automatically? + + return pVal; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c new file mode 100644 index 0000000000..b04885c5f0 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -0,0 +1,1103 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +#undef _TEST_SMA_PRINT_DEBUG_LOG_ +#define SMA_STORAGE_TSDB_DAYS 30 +#define SMA_STORAGE_TSDB_TIMES 10 +#define SMA_STORAGE_SPLIT_HOURS 24 +#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 +#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds + +#define SMA_STATE_ITEM_HASH_SLOT 32 + + +typedef struct { + SSma *pSma; + SDBFile dFile; + const SArray *pDataBlocks; // sma data + int32_t interval; // interval with the precision of DB +} STSmaWriteH; + +typedef struct { + int32_t iter; + int32_t fid; +} SmaFsIter; + +typedef struct { + STsdb *pTsdb; + SSma *pSma; + SDBFile dFile; + int32_t interval; // interval with the precision of DB + int32_t blockSize; // size of SMA block item + int8_t storageLevel; + int8_t days; + SmaFsIter smaFsIter; +} STSmaReadH; + +typedef enum { + SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma + SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma +} ESmaStorageLevel; + + +// static func + +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); +static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit); +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn); +// expired window +static int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version); +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, + int64_t version); +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); + +// read data +// TODO: This is the basic params, and should wrap the params to a queryHandle. +static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); + +// implementation + +/** + * @brief + * + * @param pSmaH + * @param pSma + * @param interval + * @param intervalUnit + * @return int32_t + */ +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->storageLevel = tdGetSmaStorageLevel(interval, intervalUnit); + pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Init of tSma FS + * + * @param pReadH + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { + SSma *pSma = pSmaH->pSma; + + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + pSmaH->smaFsIter.iter = 0; + pSmaH->smaFsIter.fid = fid; + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Set and open tSma file if it has key locates in queryWin. + * + * @param pReadH + * @param param + * @param queryWin + * @return true + * @return false + */ +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { + // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; + // int32_t nSmaFs = taosArrayGetSize(smaFs); + + smaCloseDBF(&pReadH->dFile); + +#if 0 + while (pReadH->smaFsIter.iter < nSmaFs) { + void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); + if (pSmaFile) { // match(indexName, queryWindow) + // TODO: select the file by index_name ... + pReadH->dFile = pSmaFile; + ++pReadH->smaFsIter.iter; + break; + } + ++pReadH->smaFsIter.iter; + } + + if (pReadH->pDFile) { + tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); + return true; + } +#endif + + return false; +} + + +/** + * @brief Approximate value for week/month/year. + * + * @param interval + * @param intervalUnit + * @param precision + * @param adjusted Interval already adjusted according to DB precision + * @return int64_t + */ +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { + if (adjusted) { + return interval; + } + + switch (intervalUnit) { + case TIME_UNIT_YEAR: // approximate value + interval *= 365 * 86400 * 1e3; + break; + case TIME_UNIT_MONTH: // approximate value + interval *= 30 * 86400 * 1e3; + break; + case TIME_UNIT_WEEK: // approximate value + interval *= 7 * 86400 * 1e3; + break; + case TIME_UNIT_DAY: // the interval for tSma calculation must <= day + interval *= 86400 * 1e3; + break; + case TIME_UNIT_HOUR: + interval *= 3600 * 1e3; + break; + case TIME_UNIT_MINUTE: + interval *= 60 * 1e3; + break; + case TIME_UNIT_SECOND: + interval *= 1e3; + break; + default: + break; + } + + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second + return interval / 1e6; + } else { // ms + return interval; + } + break; + case TSDB_TIME_PRECISION_MICRO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e3; + } else { // ms + return interval * 1e3; + } + break; + case TSDB_TIME_PRECISION_NANO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval * 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval; + } else { // ms + return interval * 1e6; + } + break; + default: // ms + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e6; + } else { // ms + return interval; + } + break; + } + return interval; +} + + +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit) { + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->pDataBlocks = pDataBlocks; + pSmaH->dFile.fid = SMA_IVLD_FID; + return TSDB_CODE_SUCCESS; +} + +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { + if (pSmaH) { + smaCloseDBF(&pSmaH->dFile); + } +} + +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { + SSma *pSma = pSmaH->pSma; + ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); + + pSmaH->dFile.fid = fid; + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma + * @param interval Interval calculated by DB's precision + * @param storageLevel + * @return int32_t + */ +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + int32_t daysPerFile = pCfg->days; + + if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { + int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); + daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; + } + + return daysPerFile; +} + +/** + * @brief Judge the tSma storage level + * + * @param interval + * @param intervalUnit + * @return int32_t + */ +static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { + // TODO: configurable for SMA_STORAGE_SPLIT_HOURS? + switch (intervalUnit) { + case TIME_UNIT_HOUR: + if (interval < SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_MINUTE: + if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_SECOND: + if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_MILLISECOND: + if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_MICROSECOND: + if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + case TIME_UNIT_NANOSECOND: + if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) { + return SMA_STORAGE_LEVEL_DFILESET; + } + break; + default: + break; + } + return SMA_STORAGE_LEVEL_TSDB; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. + * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. + * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The + * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). + * - The destination file of one data block for some interval is determined by its start TS key. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + const SArray *pDataBlocks = (const SArray *)msg; + + // TODO: destroy SSDataBlocks(msg) + + // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus + // the sma data would arrive ahead of the update-expired-window msg. + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + if (!pDataBlocks) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + + if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + + STSma *pTSma = pItem->pTSma; + STSmaWriteH tSmaH = {0}; + + if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { + return TSDB_CODE_FAILED; + } + + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } + + // Step 1: Judge the storage level and days + int32_t storageLevel = tdGetSmaStorageLevel(pTSma->interval, pTSma->intervalUnit); + int32_t daysPerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); + + char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId + char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? + void *pDataBuf = NULL; + int32_t sz = taosArrayGetSize(pDataBlocks); + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + for (int32_t j = 0; j < rows; ++j) { + printf("|"); + TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval + void *pSmaKey = &smaKey; + bool isStartKey = false; + + int32_t tlen = 0; // reset the len + pDataBuf = &dataBuf; // reset the buf + for (int32_t k = 0; k < colNum; ++k) { + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + skey = *(TSKEY *)var; + printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); + tdEncodeTSmaKey(groupId, skey, &pSmaKey); + } else { + printf(" %" PRIi64 " |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15d |", *(uint8_t *)var); + tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15d |", *(int8_t *)var); + tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15d |", *(int16_t *)var); + tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15d |", *(uint16_t *)var); + tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); + break; + case TSDB_DATA_TYPE_INT: + printf(" %15d |", *(int32_t *)var); + tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); + break; + case TSDB_DATA_TYPE_FLOAT: + printf(" %15f |", *(float *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); + break; + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t *)var); + tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); + break; + case TSDB_DATA_TYPE_BIGINT: + printf(" %15ld |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf(" %15lf |", *(double *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15lu |", *(uint64_t *)var); + tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); + break; + case TSDB_DATA_TYPE_NCHAR: { + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + // TODO: add binary/varbinary + TASSERT(0); + default: + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + break; + } + } + // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { + if (tlen > 0) { + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision)); + + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index + // file + // - Set and open the DFile or the B+Tree file + // TODO: tsdbStartTSmaCommit(); + if (fid != tSmaH.dFile.fid) { + if (tSmaH.dFile.fid != SMA_IVLD_FID) { + tdSmaEndCommit(pEnv); + smaCloseDBF(&tSmaH.dFile); + } + tdSetTSmaDataFile(&tSmaH, indexUid, fid); + if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { + smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + tdSmaBeginCommit(pEnv); + } + + if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { + smaWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 + " since %s", + SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); + tdSmaEndCommit(pEnv); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, + SMA_VID(pSma), indexUid, skey, groupId); + // TODO:tsdbEndTSmaCommit(); + + // Step 3: reset the SSmaStat + tdResetExpiredWindow(pSma, pStat, indexUid, skey); + } else { + smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, + SMA_VID(pSma), skey, tlen, indexUid); + } + + printf("\n"); + } + } + tdSmaEndCommit(pEnv); // TODO: not commit for every insert + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} + +int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { + smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +/** + * @brief Insert TSma data blocks to DB File build by B+Tree + * + * @param pSmaH + * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) + * @param keyLen + * @param pData + * @param dataLen + * @return int32_t + */ +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn) { + SDBFile *pDBFile = &pSmaH->dFile; + + // TODO: insert sma data blocks into B+Tree(TDB) + if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { + smaWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + uint32_t valueSize = 0; + void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); + ASSERT(data != NULL); + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); + } +#endif + return TSDB_CODE_SUCCESS; +} + +/** + * @brief When sma data received from stream computing, make the relative expired window valid. + * + * @param pSma + * @param pStat + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { + // pItem resides in hash buffer all the time unless drop sma index + // TODO: multithread protect + if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), + skey, indexUid); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), + skey, indexUid); + // TODO: use a standalone interface to received state upate notification from stream computing module. + /** + * @brief state + * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. + * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. + * when batch data caculation not finised) + * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. + */ + pItem->state = TSDB_SMA_STAT_OK; + } else { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); + return TSDB_CODE_FAILED; + } + + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Drop tSma data and local cache + * - insert/query reference + * @param pSma + * @param msg + * @return int32_t + */ +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + + // clear local cache + if (pEnv) { + smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); + + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { + if (tdSmaStatIsDropped(pItem)) { + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + + tdWLockSmaEnv(pEnv); + if (tdSmaStatIsDropped(pItem)) { + tdUnLockSmaEnv(pEnv); + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + tdSmaStatSetDropped(pItem); + tdUnLockSmaEnv(pEnv); + + int32_t nSleep = 0; + int32_t refVal = INT32_MAX; + while (true) { + if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { + smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + break; + } + smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + taosSsleep(1); + if (++nSleep > SMA_DROP_EXPIRED_TIME) { + smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, + refVal); + break; + }; + } + + tdFreeSmaStatItem(pItem); + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); + } + } + // clear sma data files + // TODO: + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma Return the data between queryWin and fill the pData. + * @param pData + * @param indexUid + * @param pQuerySKey + * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. + * @return int32_t + */ +static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + SSmaStat *pStat = NULL; + + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + pStat = SMA_ENV_STAT(pEnv); + + tdRefSmaStat(pSma, pStat); + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { + // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if + // it's NULL. + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_FAILED; + } + +#if 0 + int32_t nQueryWin = taosArrayGetSize(pQuerySKey); + for (int32_t n = 0; n < nQueryWin; ++n) { + TSKEY skey = taosArrayGet(pQuerySKey, n); + if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + } + } +#endif + +#if 1 + int8_t smaStat = 0; + if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, + tstrerror(terrno), smaStat); + return TSDB_CODE_FAILED; + } + + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), + querySKey, indexUid); + } else { + smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); + } + + STSma *pTSma = pItem->pTSma; +#endif + +#if 1 + STSmaReadH tReadH = {0}; + tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); + smaCloseDBF(&tReadH.dFile); + + tdUnRefSmaStat(pSma, pStat); + + tdInitTSmaFile(&tReadH, indexUid, querySKey); + if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { + smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + int64_t queryGroupId = 1; + tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); + + smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), + tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); + + void *result = NULL; + int32_t valueSize = 0; + if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { + smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", + SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_FAILED; + } + #endif + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); + } +#endif + taosMemoryFreeClear(result); // TODO: fill the result to output + +#if 0 + int32_t nResult = 0; + int64_t lastKey = 0; + + while (true) { + if (nResult >= nMaxResult) { + break; + } + + // set and open the file according to the STSma param + if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { + char bTree[100] = "\0"; + while (strncmp(bTree, "has more nodes", 100) == 0) { + if (nResult >= nMaxResult) { + break; + } + // tdGetDataFromBTree(bTree, queryWin, lastKey) + // fill the pData + ++nResult; + } + } + } +#endif + // read data from file and fill the result + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_SUCCESS; +} + +int32_t tdProcessTSmaCreate(SSma *pSma, char *pMsg) { + #if 0 + SSmaCfg vCreateSmaReq = {0}; + if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + smaWarn("vgId:%d tsma create msg received but deserialize failed since %s", SMA_VID(pSma), terrstr(terrno)); + return -1; + } + + smaDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", SMA_VID(pSma), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); + + // record current timezone of server side + vCreateSmaReq.tSma.timezoneInt = tsTimezone; + + if (metaCreateTSma(SMA_META(pSma), &vCreateSmaReq) < 0) { + // TODO: handle error + smaWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", SMA_VID(pSma), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); + tdDestroyTSma(&vCreateSmaReq.tSma); + return -1; + } + + tdTSmaAdd(pSma, 1); + + tdDestroyTSma(&vCreateSmaReq.tSma); + // TODO: return directly or go on follow steps? +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdDropTSma(SSma *pSma, char *pMsg) { +#if 0 + SVDropTSmaReq vDropSmaReq = {0}; + if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // TODO: send msg to stream computing to drop tSma + // if ((send msg to stream computing) < 0) { + // tdDestroyTSma(&vCreateSmaReq); + // return -1; + // } + // + + if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + tdTSmaSub(pSma, 1); +#endif + + // TODO: return directly or go on follow steps? + return TSDB_CODE_SUCCESS; +} + +static SSmaStatItem *tdNewSmaStatItem(int8_t state) { + SSmaStatItem *pItem = NULL; + + pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); + if (!pItem) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pItem->state = state; + pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), + true, HASH_ENTRY_LOCK); + if (!pItem->expiredWindows) { + taosMemoryFreeClear(pItem); + return NULL; + } + + return pItem; +} + +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, + int64_t version) { + SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + if (!pItem) { + // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later + pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state + if (!pItem) { + // Response to stream computing: OOM + // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid, true); + if (!pTSma) { + terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + smaWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, + tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + + if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + return TSDB_CODE_FAILED; + } + } else if (!(pItem = *(SSmaStatItem **)pItem)) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { + // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. + taosHashCleanup(pItem->expiredWindows); + taosMemoryFreeClear(pItem->pTSma); + taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); + smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_FAILED; + } + + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_SUCCESS; +} + + + +/** + * @brief Update expired window according to msg from stream computing module. + * + * @param pSma + * @param msg SSubmitReq + * @return int32_t + */ +int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) { + // no time-range-sma, just return success + if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { + smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); + return TSDB_CODE_SUCCESS; + } + + if (!SMA_META(pSma)) { + terrno = TSDB_CODE_INVALID_PTR; + smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { + smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + // Firstly, assume that tSma can only be created on super table/normal table. + // getActiveTimeWindow + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); + + TASSERT(pEnv && pStat && pItemsHash); + + // basic procedure + // TODO: optimization + tdRefSmaStat(pSma, pStat); + + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SInterval interval = {0}; + TSKEY lastWinSKey = INT64_MIN; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return TSDB_CODE_FAILED; + } + + while (true) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (!pBlock) break; + + STSmaWrapper *pSW = NULL; + STSma *pTSma = NULL; + + SSubmitBlkIter blkIter = {0}; + if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { + pSW = tdFreeTSmaWrapper(pSW); + break; + } + + while (true) { + STSRow *row = tGetSubmitBlkNext(&blkIter); + if (!row) { + tdFreeTSmaWrapper(pSW); + break; + } + if (!pSW || (pTSma->tableUid != pBlock->suid)) { + if (pSW) { + pSW = tdFreeTSmaWrapper(pSW); + } + if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), pBlock->suid))) { + break; + } + if ((pSW->number) <= 0 || !pSW->tSma) { + pSW = tdFreeTSmaWrapper(pSW); + break; + } + + pTSma = pSW->tSma; + + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = SMA_TSDB_CFG(pSma)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + } + + TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); + + if (lastWinSKey != winSKey) { + lastWinSKey = winSKey; + if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } else { + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", + SMA_VID(pSma), pTSma->indexUid, winSKey); + } + } + } + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tdUpdateExpireWindow(SSma *pSma, SSubmitReq *pMsg, int64_t version) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { + smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +int32_t tdGetTSmaData(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) { + smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + + diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac8a271a4f..873db62dd8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -14,6 +14,7 @@ */ #include "tq.h" +#include "tqueue.h" int32_t tqInit() { // @@ -234,7 +235,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (msgType != TDMT_VND_SUBMIT) return 0; // make sure msgType == TDMT_VND_SUBMIT - if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) { + if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { return -1; } @@ -1032,6 +1033,59 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) return 0; } +int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) { + void* pIter = NULL; + bool failed = false; + + SStreamDataSubmit* pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); + if (pSubmit == NULL) { + failed = true; + } + pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t)); + if (pSubmit->dataRef == NULL) { + failed = true; + } + + pSubmit->type = STREAM_DATA_TYPE_SUBMIT_BLOCK; + pSubmit->sourceVer = ver; + pSubmit->sourceVg = pTq->pVnode->config.vgId; + pSubmit->data = pReq; + *pSubmit->dataRef = 1; + + while (1) { + pIter = taosHashIterate(pTq->pStreamTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = (SStreamTask*)pIter; + if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue; + + int8_t inputStatus = atomic_load_8(&pTask->inputStatus); + if (inputStatus == TASK_INPUT_STATUS__NORMAL) { + if (failed) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + continue; + } + + streamDataSubmitRefInc(pSubmit); + taosWriteQitem(pTask->inputQ, pSubmit); + + int8_t execStatus = atomic_load_8(&pTask->status); + if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { + // TODO dispatch task launch msg to fetch queue + } + + } else { + // blocked or stopped, do nothing + } + } + + if (!failed) { + streamDataSubmitRefDec(pSubmit); + return 0; + } else { + return -1; + } +} + int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) { SStreamTaskExecReq req = {0}; tDecodeSStreamTaskExecReq(msg, &req); @@ -1051,7 +1105,7 @@ int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) { // try exec int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); if (execStatus == TASK_STATUS__IDLE) { - if (streamTaskExecNew(pTask) < 0) { + if (streamTaskRun(pTask) < 0) { atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); goto FAIL; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 996d789e24..8fbd1e24e1 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -91,7 +91,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p // TODO set to real sversion *pUid = 0; - int32_t sversion = 0; + int32_t sversion = 1; if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d180799e58..76d5c3cb3a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -465,7 +465,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pTbData = (STbData *)pNode->pData; pCommitIter = pCommith->iters + i; - pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); // TODO: schema version + pTSchema = metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 1); // TODO: schema version if (pTSchema) { pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); @@ -912,7 +912,7 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) { while (bidx < nBlocks) { if (!pTSchema && !tsdbCommitIsSameFile(pCommith, bidx)) { // Set commit table - pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 0); // TODO: schema version + pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 1); // TODO: schema version if (!pTSchema) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 585ef63531..844cfc094b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -16,6 +16,8 @@ #include "tsdb.h" int tsdbBegin(STsdb *pTsdb) { + if (!pTsdb) return 0; + STsdbMemTable *pMem; if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 52b466d0f6..6dfd73158e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -37,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]); +static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir); } -static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]); +static void tsdbGetDataDir(int repoid, const char* dir, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir); } // For backward compatibility @@ -591,7 +591,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), - TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]); + pRepo->dir, tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -721,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir); + tsdbGetRootDir(REPO_ID(pRepo), pRepo->dir, rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); @@ -755,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); @@ -803,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { regex_t regex; STsdbFS *pfs = REPO_FS(pRepo); - tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir); // Resource allocation and init regcomp(®ex, pattern, REG_EXTENDED); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 7f024786de..04be2a48de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -23,14 +23,6 @@ static const char *TSDB_FNAME_SUFFIX[] = { "smal", // TSDB_FILE_SMAL "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META - "tsma", // TSDB_FILE_TSMA - "rsma", // TSDB_FILE_RSMA -}; - -const char *TSDB_LEVEL_DNAME[] = { - "tsdb", - "rsma1", - "rsma2", }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); @@ -51,7 +43,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, pRepo->dir, fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 807ee95b03..8e689fc185 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -15,100 +15,17 @@ #include "tsdb.h" -#define TSDB_OPEN_RSMA_IMPL(v, l) \ - do { \ - SRetention *r = VND_RETENTIONS(v)[0]; \ - if (RETENTION_VALID(r)) { \ - return tsdbOpenImpl((v), type, &VND_RSMA##l(v), VNODE_RSMA##l##_DIR, TSDB_RETENTION_L##l); \ - } \ - } while (0) +static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg); -#define TSDB_SET_KEEP_CFG(l) \ - do { \ - SRetention *r = &pCfg->retentions[l]; \ - pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ - pKeepCfg->keep0 = pKeepCfg->keep2; \ - pKeepCfg->keep1 = pKeepCfg->keep2; \ - pKeepCfg->days = tsdbEvalDays(r, pCfg->precision); \ - } while (0) -#define RETENTION_DAYS_SPLIT_RATIO 10 -#define RETENTION_DAYS_SPLIT_MIN 1 -#define RETENTION_DAYS_SPLIT_MAX 30 +// implementation -static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type); -static int32_t tsdbEvalDays(SRetention *r, int8_t precision); -static int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level); - -int tsdbOpen(SVnode *pVnode, int8_t type) { - switch (type) { - case TSDB_TYPE_TSDB: - return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); - case TSDB_TYPE_TSMA: - ASSERT(0); - break; - case TSDB_TYPE_RSMA_L0: - TSDB_OPEN_RSMA_IMPL(pVnode, 0); - break; - case TSDB_TYPE_RSMA_L1: - TSDB_OPEN_RSMA_IMPL(pVnode, 1); - break; - case TSDB_TYPE_RSMA_L2: - TSDB_OPEN_RSMA_IMPL(pVnode, 2); - break; - default: - ASSERT(0); - break; - } - return 0; -} - -static int32_t tsdbEvalDays(SRetention *r, int8_t precision) { - int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY); - int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY); - - int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO; - if (days <= RETENTION_DAYS_SPLIT_MIN) { - days = RETENTION_DAYS_SPLIT_MIN; - if (days < freqDays) { - days = freqDays + 1; - } - } else { - if (days > RETENTION_DAYS_SPLIT_MAX) { - days = RETENTION_DAYS_SPLIT_MAX; - } - if (days < freqDays) { - days = freqDays + 1; - } - } - return days * 1440; -} - -static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type) { +static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg) { pKeepCfg->precision = pCfg->precision; - switch (type) { - case TSDB_TYPE_TSDB: - pKeepCfg->days = pCfg->days; - pKeepCfg->keep0 = pCfg->keep0; - pKeepCfg->keep1 = pCfg->keep1; - pKeepCfg->keep2 = pCfg->keep2; - break; - case TSDB_TYPE_TSMA: - ASSERT(0); - break; - case TSDB_TYPE_RSMA_L0: - TSDB_SET_KEEP_CFG(0); - break; - case TSDB_TYPE_RSMA_L1: - TSDB_SET_KEEP_CFG(1); - break; - case TSDB_TYPE_RSMA_L2: - TSDB_SET_KEEP_CFG(2); - break; - default: - ASSERT(0); - break; - } + pKeepCfg->days = pCfg->days; + pKeepCfg->keep0 = pCfg->keep0; + pKeepCfg->keep1 = pCfg->keep1; + pKeepCfg->keep2 = pCfg->keep2; return 0; } @@ -116,18 +33,16 @@ static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t typ * @brief * * @param pVnode - * @param type * @param ppTsdb * @param dir - * @param level retention level * @return int */ -int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) { +int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg) { STsdb *pTsdb = NULL; int slen = 0; *ppTsdb = NULL; - slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3; + slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + TSDB_DATA_DIR_LEN + 3; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -136,13 +51,18 @@ int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *di return -1; } + ASSERT(strlen(dir) < TSDB_DATA_DIR_LEN); + memcpy(pTsdb->dir, dir, strlen(dir)); pTsdb->path = (char *)&pTsdb[1]; sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); pTsdb->pVnode = pVnode; - pTsdb->level = level; pTsdb->repoLocked = false; taosThreadMutexInit(&pTsdb->mutex, NULL); - tsdbSetKeepCfg(REPO_KEEP_CFG(pTsdb), REPO_CFG(pTsdb), type); + if (!pKeepCfg) { + tsdbSetKeepCfg(&pTsdb->keepCfg, &pVnode->config.tsdbCfg); + } else { + memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg)); + } pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb)); // create dir (TODO: use tfsMkdir) @@ -163,12 +83,13 @@ _err: return -1; } -int tsdbClose(STsdb *pTsdb) { - if (pTsdb) { +int tsdbClose(STsdb **pTsdb) { + if (*pTsdb) { // TODO: destroy mem/imem - tsdbCloseFS(pTsdb); - tsdbFreeFS(pTsdb->fs); - taosMemoryFree(pTsdb); + taosThreadMutexDestroy(&(*pTsdb)->mutex); + tsdbCloseFS(*pTsdb); + tsdbFreeFS((*pTsdb)->fs); + taosMemoryFreeClear(*pTsdb); } return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 927babc26c..55fe8a3945 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -490,7 +490,7 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); - pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0); + pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 1); int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; @@ -1271,7 +1271,6 @@ _error: static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo); static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); -static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols); static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle); static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); @@ -1301,7 +1300,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer - int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; + int32_t step = ascScan ? 1 : -1; TSKEY maxKey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? (binfo.window.skey - step) : (binfo.window.ekey - step); @@ -1618,7 +1617,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (pSchema1 == NULL) { // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); // TODO: use the real schemaVersion - pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); + pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1); } #ifdef TD_DEBUG_PRINT_ROW @@ -1637,7 +1636,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (pSchema2 == NULL) { // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); // TODO: use the real schemaVersion - pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); + pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 1); } if (isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1790,22 +1789,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa #endif } -static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) { - if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - return; - } - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (numOfRows < pTsdbReadHandle->outputCapacity) { - int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, - numOfRows * pColInfo->info.bytes); - } - } -} - static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, int32_t* start, int32_t* end) { *start = -1; @@ -1891,9 +1874,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa cur->lastKey = tsArray[endPos] + step; cur->blockCompleted = true; - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); - // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. pos = endPos + step; updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); @@ -1944,18 +1924,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows - 1] == pBlock->keyLast); + bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); + int32_t step = ascScan ? 1 : -1; + // for search the endPos, so the order needs to reverse - int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int32_t order = ascScan ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); - - STable* pTable = NULL; int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); + STimeWindow* pWin = &blockInfo.window; tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 - " rows:%d, start:%d, end:%d, %s", - pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, + " rows:%d, start:%d, end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, pWin->skey, pWin->ekey, blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->idStr); // compared with the data from in-memory buffer, to generate the correct timestamp array list @@ -1986,20 +1966,16 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } TSKEY key = TD_ROW_KEY(row1); - if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + if ((key > pTsdbReadHandle->window.ekey && ascScan) || (key < pTsdbReadHandle->window.ekey && !ascScan)) { break; } - if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && - ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && - !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) || + ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) { break; } - if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + if ((key < tsArray[pos] && ascScan) || (key > tsArray[pos] && !ascScan)) { if (rv1 != TD_ROW_SVER(row1)) { // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = TD_ROW_SVER(row1); @@ -2054,23 +2030,19 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } #endif if (TD_SUPPORT_UPDATE(pCfg->update)) { - if (lastKeyAppend != key) { - lastKeyAppend = key; - ++curRow; - } numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); + lastKeyAppend = key; if (rv1 != TD_ROW_SVER(row1)) { - // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = TD_ROW_SVER(row1); } if (row2 && rv2 != TD_ROW_SVER(row2)) { - // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = TD_ROW_SVER(row2); } - numOfRows += - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend); + + // still assign data into current row + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend); if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2081,12 +2053,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->mixBlock = true; moveToNextRowInMem(pCheckInfo); + ++curRow; + pos += step; } else { moveToNextRowInMem(pCheckInfo); } - } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) { if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } @@ -2112,17 +2085,17 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf int32_t qstart = 0, qend = 0; getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend); - if ((lastKeyAppend != TSKEY_INITIAL_VAL) && - (lastKeyAppend != (ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qstart] : tsArray[qend]))) { + if ((lastKeyAppend != TSKEY_INITIAL_VAL) && (lastKeyAppend != (ascScan ? tsArray[qstart] : tsArray[qend]))) { ++curRow; } + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, qstart, qend); pos += (qend - qstart + 1) * step; if (numOfRows > 0) { curRow = numOfRows - 1; } - cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[qend] : tsArray[qstart]; + cur->win.ekey = ascScan ? tsArray[qend] : tsArray[qstart]; cur->lastKey = cur->win.ekey + step; lastKeyAppend = cur->win.ekey; } @@ -2134,10 +2107,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && - ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && - !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && ascScan) || + ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && !ascScan)) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; @@ -2149,22 +2120,20 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? tsArray[end] : tsArray[start]; + cur->win.ekey = ascScan ? tsArray[end] : tsArray[start]; cur->lastKey = cur->win.ekey + step; cur->mixBlock = true; } } } - cur->blockCompleted = - (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || - ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); + cur->blockCompleted = (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ascScan) || + ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ascScan)); - if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (!ascScan) { TSWAP(cur->win.skey, cur->win.ekey); } - moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pTsdbReadHandle); @@ -2755,7 +2724,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != TD_ROW_SVER(row)) { - pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); + pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 1); rv = TD_ROW_SVER(row); } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, @@ -3877,7 +3846,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch // NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true); + SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 1, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index e878668654..1589513110 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -2084,7 +2084,7 @@ static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { // TODO: use the proper schema instead of 0, and cache STSchema in cache - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0); + STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index e7bee3342a..b4fbd01c63 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -47,7 +47,7 @@ int vnodeBegin(SVnode *pVnode) { } // begin tsdb - if (vnodeIsRollup(pVnode)) { + if (pVnode->pSma) { if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 7476da2a0f..d44e30988d 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -96,26 +96,15 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { } // open tsdb - if (vnodeIsRollup(pVnode)) { - if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) { - vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_TYPE_TSDB) < 0) { + vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } - if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) { - vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } - - if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) { - vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } - } else { - if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) { - vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + // open sma + if (smaOpen(pVnode)) { + vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; } // open wal @@ -161,10 +150,10 @@ _err: if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pWal) walClose(pVnode->pWal); - if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb); + if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); - tsdbClose(VND_RSMA1(pVnode)); - tsdbClose(VND_RSMA2(pVnode)); + if (pVnode->pSma) smaClose(pVnode->pSma); + tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); return NULL; @@ -177,9 +166,8 @@ void vnodeClose(SVnode *pVnode) { vnodeQueryClose(pVnode); walClose(pVnode->pWal); tqClose(pVnode->pTq); - tsdbClose(VND_TSDB(pVnode)); - tsdbClose(VND_RSMA1(pVnode)); - tsdbClose(VND_RSMA2(pVnode)); + if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); + smaClose(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4f76bc5386..43d68fc9fd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -16,12 +16,13 @@ #include "vnd.h" static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { #if 0 @@ -72,7 +73,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_STB: - if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_STB: if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -81,15 +82,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_ALTER_TABLE: - if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err; + if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DROP_TABLE: if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; - case TDMT_VND_CREATE_SMA: { // timeRangeSMA - if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { - // TODO - } + case TDMT_VND_CREATE_SMA: { + if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; } break; /* TSDB */ case TDMT_VND_SUBMIT: @@ -195,10 +194,12 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO // blockDebugShowData(data); - tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); + tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + if (syncEnvIsStart()) { SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); assert(pSyncNode != NULL); @@ -220,67 +221,70 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnPingCb(pSyncNode, pSyncMsg); + ret = syncNodeOnPingCb(pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); syncClientRequestDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); + ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); - syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); + ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); } else { vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); + ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; } syncNodeRelease(pSyncNode); } else { vError("==vnodeProcessSyncReq== error syncEnv stop"); + ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; } - return 0; + + return ret; } static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { @@ -305,7 +309,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, goto _err; } - tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req, &pVnode->msgCb); + tdProcessRSmaCreate(pVnode->pSma, pVnode->pMeta, &req, &pVnode->msgCb); tDecoderClear(&coder); return 0; @@ -366,7 +370,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } } else { cRsp.code = TSDB_CODE_SUCCESS; - tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); + tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); } taosArrayPush(rsp.pArray, &cRsp); @@ -374,8 +378,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tDecoderClear(&decoder); - tsdbUpdateTbUidList(pVnode->pTsdb, pStore); - tsdbUidStoreFree(pStore); + tdUpdateTbUidList(pVnode->pSma, pStore); + tdUidStoreFree(pStore); // prepare rsp SEncoder encoder = {0}; @@ -398,20 +402,32 @@ _exit: return rcode; } -static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { - // ASSERT(0); -#if 0 - SVCreateTbReq vAlterTbReq = {0}; - vTrace("vgId:%d, process alter stb req", TD_VID(pVnode)); - tDeserializeSVCreateTbReq(pReq, &vAlterTbReq); - // TODO: to encapsule a free API - taosMemoryFree(vAlterTbReq.stbCfg.pSchema); - taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); - if (vAlterTbReq.stbCfg.pRSmaParam) { - taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); +static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVCreateStbReq req = {0}; + SDecoder dc = {0}; + + pRsp->msgType = TDMT_VND_ALTER_STB_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + tDecoderInit(&dc, pReq, len); + + // decode req + if (tDecodeSVCreateStbReq(&dc, &req) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; } - taosMemoryFree(vAlterTbReq.name); -#endif + + if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + tDecoderClear(&dc); + return -1; + } + + tDecoderClear(&dc); + return 0; } @@ -444,9 +460,32 @@ _exit: return 0; } -static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) { - // TODO - ASSERT(0); +static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVAlterTbReq vAlterTbReq = {0}; + SDecoder dc = {0}; + + pRsp->msgType = TDMT_VND_ALTER_TABLE_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + pRsp->code = TSDB_CODE_SUCCESS; + + tDecoderInit(&dc, pReq, len); + + // decode + if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; + } + + // process + if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq) < 0) { + pRsp->code = terrno; + tDecoderClear(&dc); + return -1; + } + + tDecoderClear(&dc); return 0; } @@ -514,7 +553,7 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub if (pSchema) { taosMemoryFreeClear(pSchema); } - pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 0); // TODO: use the real schema + pSchema = metaGetTbTSchema(pMeta, msgIter->suid, 1); // TODO: use the real schema if (pSchema) { suid = msgIter->suid; } @@ -649,8 +688,38 @@ _exit: // TODO: refactor if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) && (pRsp->code == TSDB_CODE_SUCCESS)) { - tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); } return 0; } + +static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { + SVCreateTSmaReq req = {0}; + SDecoder coder; + + pRsp->msgType = TDMT_VND_CREATE_SMA_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + // decode and process req + tDecoderInit(&coder, pReq, len); + + if (tDecodeSVCreateTSmaReq(&coder, &req) < 0) { + pRsp->code = terrno; + goto _err; + } + + if (metaCreateTSma(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + goto _err; + } + + tDecoderClear(&coder); + return 0; + +_err: + tDecoderClear(&coder); + return -1; +} diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 64090d0283..c96ad140a1 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2576,12 +2576,6 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - SHashObj *metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == metaCache) { - qError("taosHashInit failed, num:%d", gCtgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - code = taosHashPut(gCtgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { if (HASH_NODE_EXIST(code)) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d08345a02f..0ab172f03a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -1601,9 +1600,6 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { } } -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); -static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes); - static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; uint32_t status = BLK_DATA_NOT_LOAD; @@ -1771,100 +1767,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc return TSDB_CODE_SUCCESS; } -/* - * set tag value in SqlFunctionCtx - * e.g.,tag information into input buffer - */ -static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant* tag, int16_t type, int16_t bytes) { - taosVariantDestroy(tag); - - char* val = NULL; - // if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - // val = tsdbGetTableName(pTable); - // assert(val != NULL); - // } else { - // val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); - // } - - if (val == NULL || isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t maxLen = bytes - VARSTR_HEADER_SIZE; - int32_t len = (varDataLen(val) > maxLen) ? maxLen : varDataLen(val); - taosVariantCreateFromBinary(tag, varDataVal(val), len, type); - // taosVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); - } else { - taosVariantCreateFromBinary(tag, val, bytes, type); - } -} - -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { - assert(pTagColList != NULL && numOfTags > 0); - - for (int32_t i = 0; i < numOfTags; ++i) { - if (pTagColList[i].colId == colId) { - return &pTagColList[i]; - } - } - - return NULL; -} - -void setTagValue(SOperatorInfo* pOperatorInfo, void* pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - SExprInfo* pExpr = pOperatorInfo->pExpr; - SExprInfo* pExprInfo = &pExpr[0]; - int32_t functionId = getExprFunctionId(pExprInfo); -#if 0 - if (pQueryAttr->numOfOutput == 1 && functionId == FUNCTION_TS_COMP && pQueryAttr->stableQuery) { - assert(pExprInfo->base.numOfParams == 1); - - // int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; - int16_t tagColId = -1; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - - doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); - - } else { - // set tag value, by which the results are aggregated. - int32_t offset = 0; - memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen); - - for (int32_t idx = 0; idx < numOfOutput; ++idx) { - SExprInfo* pLocalExprInfo = &pExpr[idx]; - - // ts_comp column required the tag value for join filter - if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.pParam[0].pCol->flag)) { - continue; - } - - // todo use tag column index to optimize performance - doSetTagValueInParam(pTable, pLocalExprInfo->base.pParam[0].pCol->colId, &pCtx[idx].tag, - pLocalExprInfo->base.resSchema.type, pLocalExprInfo->base.resSchema.bytes); - - if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) || - pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_BOOL || - pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_TIMESTAMP) { - memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i, pLocalExprInfo->base.resSchema.bytes); - } else { - if (pCtx[idx].tag.pz != NULL) { - memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); - } - } - - offset += pLocalExprInfo->base.resSchema.bytes; - } - } - - // set the tsBuf start position before check each data block - if (pRuntimeEnv->pTsBuf != NULL) { - setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); - } -#endif -} - void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) { pBlock->info.rows = 0; @@ -4038,12 +3940,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { if (pProjectInfo->existDataBlock) { // TODO refactor SSDataBlock* pBlock = pProjectInfo->existDataBlock; pProjectInfo->existDataBlock = NULL; - *newgroup = true; - - // todo dynamic set tags - // if (pTableQueryInfo != NULL) { - // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs); - // } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); @@ -4084,13 +3980,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } } - // todo set tags - - // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - // if (pTableQueryInfo != NULL) { - // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs); - // } - // the pDataBlock are always the same one, no need to call this again int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); @@ -4430,10 +4319,6 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(pInfo, numOfOutput); } -void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { - SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; -} - void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); @@ -4778,7 +4663,6 @@ static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); -static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { @@ -5447,150 +5331,3 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } - -static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { - SJoinOperatorInfo* pJoinInfo = pOperator->info; - - SSDataBlock* pRes = pJoinInfo->pRes; - blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, 4096); - - int32_t nrows = 0; - - while (1) { - if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { - SOperatorInfo* ds1 = pOperator->pDownstream[0]; - publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1); - publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); - - pJoinInfo->leftPos = 0; - if (pJoinInfo->pLeft == NULL) { - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - break; - } - } - - if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { - SOperatorInfo* ds2 = pOperator->pDownstream[1]; - publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2); - publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); - - pJoinInfo->rightPos = 0; - if (pJoinInfo->pRight == NULL) { - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - break; - } - } - - SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId); - char* pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos); - - SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId); - char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); - - // only the timestamp match support for ordinary table - ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) { - for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { - SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i); - - SExprInfo* pExprInfo = &pOperator->pExpr[i]; - - int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; - int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; - int32_t rowIndex = -1; - - SColumnInfoData* pSrc = NULL; - if (pJoinInfo->pLeft->info.blockId == blockId) { - pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); - rowIndex = pJoinInfo->leftPos; - } else { - pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); - rowIndex = pJoinInfo->rightPos; - } - - if (colDataIsNull_s(pSrc, rowIndex)) { - colDataAppendNULL(pDst, nrows); - } else { - char* p = colDataGetData(pSrc, rowIndex); - colDataAppend(pDst, nrows, p, false); - } - } - - pJoinInfo->leftPos += 1; - pJoinInfo->rightPos += 1; - - nrows += 1; - } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) { - pJoinInfo->leftPos += 1; - - if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { - continue; - } - } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) { - pJoinInfo->rightPos += 1; - if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { - continue; - } - } - - // the pDataBlock are always the same one, no need to call this again - pRes->info.rows = nrows; - if (pRes->info.rows >= pOperator->resultInfo.threshold) { - break; - } - } - - return (pRes->info.rows > 0) ? pRes : NULL; -} - -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, - SExecTaskInfo* pTaskInfo) { - SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pOperator == NULL || pInfo == NULL) { - goto _error; - } - - initResultSizeInfo(pOperator, 4096); - - pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - SOperatorNode* pNode = (SOperatorNode*)pOnCondition; - setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); - setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); - - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - -void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { - pColumn->slotId = pColumnNode->slotId; - pColumn->type = pColumnNode->node.resType.type; - pColumn->bytes = pColumnNode->node.resType.bytes; - pColumn->precision = pColumnNode->node.resType.precision; - pColumn->scale = pColumnNode->node.resType.scale; -} diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c new file mode 100644 index 0000000000..d7d6d96346 --- /dev/null +++ b/source/libs/executor/src/joinoperator.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "function.h" +#include "os.h" +#include "querynodes.h" +#include "tdatablock.h" +#include "tmsg.h" +#include "executorimpl.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" + +static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); +static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); +static void destroyMergeJoinOperator(void* param, int32_t numOfOutput); +static void extractTimeCondition(SJoinOperatorInfo *Info, SLogicConditionNode* pLogicConditionNode); + +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, + int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, + SExecTaskInfo* pTaskInfo) { + SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pOperator == NULL || pInfo == NULL) { + goto _error; + } + + initResultSizeInfo(pOperator, 4096); + + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + if (nodeType(pOnCondition) == QUERY_NODE_OPERATOR) { + SOperatorNode* pNode = (SOperatorNode*)pOnCondition; + setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); + setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); + } else if (nodeType(pOnCondition) == QUERY_NODE_LOGIC_CONDITION) { + extractTimeCondition(pInfo, (SLogicConditionNode*) pOnCondition); + } + + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL); + int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { + pColumn->slotId = pColumnNode->slotId; + pColumn->type = pColumnNode->node.resType.type; + pColumn->bytes = pColumnNode->node.resType.bytes; + pColumn->precision = pColumnNode->node.resType.precision; + pColumn->scale = pColumnNode->node.resType.scale; +} + +void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { + SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; +} + +SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { + SJoinOperatorInfo* pJoinInfo = pOperator->info; + + SSDataBlock* pRes = pJoinInfo->pRes; + blockDataCleanup(pRes); + blockDataEnsureCapacity(pRes, 4096); + + int32_t nrows = 0; + + while (1) { + // todo extract method + if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { + SOperatorInfo* ds1 = pOperator->pDownstream[0]; + publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); + pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1); + publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); + + pJoinInfo->leftPos = 0; + if (pJoinInfo->pLeft == NULL) { + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); + break; + } + } + + if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { + SOperatorInfo* ds2 = pOperator->pDownstream[1]; + publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); + pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2); + publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); + + pJoinInfo->rightPos = 0; + if (pJoinInfo->pRight == NULL) { + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); + break; + } + } + + SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId); + char* pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos); + + SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId); + char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); + + // only the timestamp match support for ordinary table + ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) { + for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i); + + SExprInfo* pExprInfo = &pOperator->pExpr[i]; + + int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; + int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; + int32_t rowIndex = -1; + + SColumnInfoData* pSrc = NULL; + if (pJoinInfo->pLeft->info.blockId == blockId) { + pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); + rowIndex = pJoinInfo->leftPos; + } else { + pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); + rowIndex = pJoinInfo->rightPos; + } + + if (colDataIsNull_s(pSrc, rowIndex)) { + colDataAppendNULL(pDst, nrows); + } else { + char* p = colDataGetData(pSrc, rowIndex); + colDataAppend(pDst, nrows, p, false); + } + } + + pJoinInfo->leftPos += 1; + pJoinInfo->rightPos += 1; + + nrows += 1; + } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) { + pJoinInfo->leftPos += 1; + + if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { + continue; + } + } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) { + pJoinInfo->rightPos += 1; + if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { + continue; + } + } + + // the pDataBlock are always the same one, no need to call this again + pRes->info.rows = nrows; + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + } + + return (pRes->info.rows > 0) ? pRes : NULL; +} + +static void extractTimeCondition(SJoinOperatorInfo *pInfo, SLogicConditionNode* pLogicConditionNode) { + int32_t len = LIST_LENGTH(pLogicConditionNode->pParameterList); + + for(int32_t i = 0; i < len; ++i) { + SNode* pNode = nodesListGetNode(pLogicConditionNode->pParameterList, i); + if (nodeType(pNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pn1 = (SOperatorNode*)pNode; + setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pn1->pLeft); + setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pn1->pRight); + break; + } + } +} diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 10dc482462..79e675e2df 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -773,7 +773,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { break; } - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true); STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; @@ -1062,8 +1061,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true); if (pInfo->invertible) { @@ -1377,7 +1374,6 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { break; } - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfExprs); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, MAIN_SCAN, true); // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 57dfcaeddd..763ccbf7a0 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -584,6 +584,37 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkExchangeLogicPlanSrcGroupId = "SrcGroupId"; +static const char* jkExchangeLogicPlanSrcPrecision = "Precision"; + +static int32_t logicExchangeNodeToJson(const void* pObj, SJson* pJson) { + const SExchangeLogicNode* pNode = (const SExchangeLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcGroupId, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcPrecision, pNode->precision); + } + + return code; +} + +static int32_t jsonToLogicExchangeNode(const SJson* pJson, void* pObj) { + SExchangeLogicNode* pNode = (SExchangeLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkExchangeLogicPlanSrcGroupId, &pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkExchangeLogicPlanSrcPrecision, &pNode->precision); + } + + return code; +} + static const char* jkFillLogicPlanMode = "Mode"; static const char* jkFillLogicPlanWStartTs = "WStartTs"; static const char* jkFillLogicPlanValues = "Values"; @@ -2987,6 +3018,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicProjectNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: break; + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return logicExchangeNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_FILL: return logicFillNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SORT: @@ -3083,6 +3116,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicScanNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + return jsonToLogicExchangeNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_FILL: return jsonToLogicFillNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SORT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index ad1dca6857..250e7910d6 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -14,6 +14,7 @@ */ #include "catalog.h" +#include "cmdnodes.h" #include "parInt.h" typedef struct SAuthCxt { @@ -65,8 +66,8 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) { return code; } -static int32_t authDropUser(SAuthCxt* pCxt, SDropUserReq* pStmt) { - if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->user, TSDB_DEFAULT_USER)) { +static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { + if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->useName, TSDB_DEFAULT_USER)) { return TSDB_CODE_PAR_PERMISSION_DENIED; } return TSDB_CODE_SUCCESS; @@ -92,7 +93,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_ALTER_USER_STMT: break; case QUERY_NODE_DROP_USER_STMT: { - return authDropUser(pCxt, (SDropUserReq*)pStmt); + return authDropUser(pCxt, (SDropUserStmt*)pStmt); } case QUERY_NODE_USE_DATABASE_STMT: case QUERY_NODE_CREATE_DNODE_STMT: diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 15e4b8ef36..01833e1776 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1041,18 +1041,6 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { destroyCreateSubTbReq(&pCxt->createTblReq); } -static void destroyDataBlock(STableDataBlocks* pDataBlock) { - if (pDataBlock == NULL) { - return; - } - - taosMemoryFreeClear(pDataBlock->pData); - if (!pDataBlock->cloned) { - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - } - taosMemoryFreeClear(pDataBlock); -} - static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); @@ -1301,6 +1289,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash CHECK_CODE(buildOutput(&insertCtx)); + destroyBlockArrayList(insertCtx.pVgDataBlocks); return TSDB_CODE_SUCCESS; } @@ -1580,16 +1569,25 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields // schemaless logic start -typedef struct SmlExecHandle { - SHashObj* pBlockHash; - +typedef struct SmlExecTableHandle { SParsedDataColInfo tags; // each table SKVRowBuilder tagsBuilder; // each table SVCreateTbReq createTblReq; // each table +} SmlExecTableHandle; - SQuery* pQuery; +typedef struct SmlExecHandle { + SHashObj* pBlockHash; + SmlExecTableHandle tableExecHandle; + SQuery *pQuery; } SSmlExecHandle; +static void smlDestroyTableHandle(void* pHandle) { + SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle; + tdDestroyKVRowBuilder(&handle->tagsBuilder); + destroyBoundColumnInfo(&handle->tags); + destroyCreateSubTbReq(&handle->createTblReq); +} + static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; @@ -1692,25 +1690,26 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; + smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tags, pTagsSchema); + setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); + int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; } SKVRow row = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf); + ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, &row, &pBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; } - buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); + buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, row, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, - pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq); + pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "create data block error"); return ret; @@ -1826,9 +1825,7 @@ void smlDestroyHandle(void* pHandle) { if (!pHandle) return; SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; destroyBlockHashmap(handle->pBlockHash); - tdDestroyKVRowBuilder(&handle->tagsBuilder); - destroyBoundColumnInfo(&handle->tags); - destroyCreateSubTbReq(&handle->createTblReq); + smlDestroyTableHandle(&handle->tableExecHandle); taosMemoryFree(handle); } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 8b649350b7..deb899309e 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -237,9 +237,7 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } + taosMemoryFreeClear(pDataBlock->pTableMeta); destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6427e13ae9..9773b69ee7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2949,6 +2949,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p } static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { + SEncoder encoder = {0}; + int32_t contLen = 0; SVDropTSmaReq dropSmaReq = {0}; strcpy(dropSmaReq.indexName, pStmt->indexName); @@ -2956,16 +2958,26 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } + + int32_t ret = 0; + tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret); + if (ret < 0) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; - pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq); + pCxt->pCmdMsg->msgLen = contLen; pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } void* pBuf = pCxt->pCmdMsg->pMsg; - tSerializeSVDropTSmaReq(&pBuf, &dropSmaReq); - + if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) { + tEncoderClear(&encoder); + return TSDB_CODE_OUT_OF_MEMORY; + } + tEncoderClear(&encoder); return TSDB_CODE_SUCCESS; } @@ -3713,7 +3725,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* req.type = TD_NORMAL_TABLE; req.name = strdup(pStmt->tableName); req.ntb.schema.nCols = LIST_LENGTH(pStmt->pCols); - req.ntb.schema.sver = 0; + req.ntb.schema.sver = 1; req.ntb.schema.pSchema = taosMemoryCalloc(req.ntb.schema.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntb.schema.pSchema) { destroyCreateTbReq(&req); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 821f480b20..47424d3138 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -235,11 +235,17 @@ TEST_F(ParserSelectTest, semanticError) { TEST_F(ParserSelectTest, setOperator) { useDb("root", "test"); - // run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); + run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); - // run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); + run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); } +TEST_F(ParserSelectTest, informationSchema) { + useDb("root", "test"); + + run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'"); +} + } // namespace ParserTest diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index edac2a879f..e38c180ac6 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -392,7 +392,8 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, } static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { - if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) { + if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || + TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 1266e8ae4b..54bc24e8bb 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -303,7 +303,7 @@ static SLogicNode* unMatchByNode(SLogicNode* pNode) { } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = unMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -318,7 +318,7 @@ static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan } pExchange->srcGroupId = pCxt->groupId; // pExchange->precision = pScan->pMeta->tableInfo.precision; - pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets); + pExchange->node.pTargets = nodesCloneList(pAgg->pGroupKeys); if (NULL == pExchange->node.pTargets) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index b70cb4d19a..67c09d706e 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -47,4 +47,10 @@ TEST_F(PlanOtherTest, explain) { run("explain analyze SELECT * FROM t1"); run("explain analyze verbose true ratio 0.01 SELECT * FROM t1"); -} \ No newline at end of file +} + +TEST_F(PlanOtherTest, show) { + useDb("root", "test"); + + run("SHOW DATABASES"); +} diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp index 83c9621916..9ff035e148 100644 --- a/source/libs/planner/test/planStateTest.cpp +++ b/source/libs/planner/test/planStateTest.cpp @@ -23,13 +23,13 @@ class PlanStateTest : public PlannerTestBase {}; TEST_F(PlanStateTest, basic) { useDb("root", "test"); - run("select count(*) from t1 state_window(c1)"); + run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1)"); } TEST_F(PlanStateTest, stateExpr) { useDb("root", "test"); - run("select count(*) from t1 state_window(c1 + 10)"); + run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1 + 10)"); } TEST_F(PlanStateTest, selectFunc) { diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index 11e5e98052..6a7cb91bb9 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -25,11 +25,9 @@ TEST_F(PlanSubqeuryTest, basic) { if (0 == g_skipSql) { run("SELECT * FROM (SELECT * FROM t1)"); - - run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); } - run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); + run("SELECT LAST(c1) FROM (SELECT * FROM t1)"); } TEST_F(PlanSubqeuryTest, doubleGroupBy) { @@ -39,3 +37,11 @@ TEST_F(PlanSubqeuryTest, doubleGroupBy) { "SELECT c1 + c3 a, c1 + COUNT(*) b FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3) " "WHERE a > 100 GROUP BY b"); } + +TEST_F(PlanSubqeuryTest, withSetOperator) { + useDb("root", "test"); + + run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); + + run("SELECT c1 FROM (SELECT c1 FROM t1 UNION SELECT c1 FROM t1)"); +} diff --git a/source/libs/planner/test/planSysTbTest.cpp b/source/libs/planner/test/planSysTbTest.cpp index fff6bfcca4..e5c30030b3 100644 --- a/source/libs/planner/test/planSysTbTest.cpp +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -27,8 +27,8 @@ TEST_F(PlanSysTableTest, show) { run("show stables"); } -TEST_F(PlanSysTableTest, information) { +TEST_F(PlanSysTableTest, informationSchema) { useDb("root", "information_schema"); - run("show tables"); + run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'"); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 717958c033..6a25b23c6b 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -812,13 +812,11 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (ctx->rspCode) { - QW_TASK_ELOG("task already failed at phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, - tstrerror(ctx->rspCode)); + QW_TASK_ELOG("task already failed at phase %s, code:%s", qwPhaseStr(phase), tstrerror(ctx->rspCode)); QW_ERR_JRET(ctx->rspCode); } _return: - if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); @@ -836,7 +834,11 @@ _return: QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code)); } - QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code)); + if (code != TSDB_CODE_SUCCESS) { + QW_TASK_ELOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code)); + } else { + QW_TASK_DLOG("end to handle event at phase %s, code:%s", qwPhaseStr(phase), tstrerror(code)); + } QW_RET(code); } diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 33147a0c0a..812874dafb 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -133,7 +133,144 @@ int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) { return inputStatus; } -int32_t streamTaskProcessTriggerReq(SStreamTask* pTask, SMsgCb* pMsgCb, char* msg, int32_t msgLen) { +int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { + void* exec = pTask->exec.runners[0].executor; + + // set input + if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT); + + qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK); + } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) { + SStreamDataBlock* pBlock = (SStreamDataBlock*)data; + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); + + SArray* blocks = pBlock->blocks; + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK); + } + + // exec + while (1) { + SSDataBlock* output; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) break; + taosArrayPush(pRes, &output); + } + + // destroy + if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + streamDataSubmitRefDec((SStreamDataSubmit*)data); + } else { + taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); + } + return 0; +} + +// TODO: handle version +int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) { + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) return -1; + while (1) { + int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); + void* exec = pTask->exec.runners[0].executor; + if (execStatus == TASK_STATUS__IDLE) { + // first run, from qall, handle failure from last exec + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + + streamTaskExecImpl(pTask, data, pRes); + + taosFreeQitem(data); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + resQ->type = STREAM_INPUT__DATA_BLOCK; + resQ->blocks = pRes; + taosWriteQitem(pTask->outputQ, resQ); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) goto FAIL; + } + } + // second run, from inputQ + taosReadAllQitems(pTask->inputQ, pTask->inputQAll); + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + + streamTaskExecImpl(pTask, data, pRes); + + taosFreeQitem(data); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + resQ->type = STREAM_INPUT__DATA_BLOCK; + resQ->blocks = pRes; + taosWriteQitem(pTask->outputQ, resQ); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) goto FAIL; + } + } + // set status closing + atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); + // third run, make sure all inputQ is cleared + taosReadAllQitems(pTask->inputQ, pTask->inputQAll); + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + + streamTaskExecImpl(pTask, data, pRes); + + taosFreeQitem(data); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + resQ->type = STREAM_INPUT__DATA_BLOCK; + resQ->blocks = pRes; + taosWriteQitem(pTask->outputQ, resQ); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) goto FAIL; + } + } + // set status closing + atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); + // third run, make sure all inputQ is cleared + taosReadAllQitems(pTask->inputQ, pTask->inputQAll); + while (1) { + void* data = NULL; + taosGetQitem(pTask->inputQAll, &data); + if (data == NULL) break; + } + + atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + break; + } else if (execStatus == TASK_STATUS__CLOSING) { + continue; + } else if (execStatus == TASK_STATUS__EXECUTING) { + break; + } else { + ASSERT(0); + } + } + return 0; +FAIL: + atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + return -1; +} + +int32_t streamTaskDispatchDown(SStreamTask* pTask, SMsgCb* pMsgCb) { + // + return 0; +} + +int32_t streamTaskSink(SStreamTask* pTask) { // return 0; } @@ -156,8 +293,8 @@ int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDat // 2.3. closing: keep trying while (1) { int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); - void* exec = pTask->exec.runners[0].executor; if (execStatus == TASK_STATUS__IDLE) { + void* exec = pTask->exec.runners[0].executor; SArray* pRes = taosArrayInit(0, sizeof(void*)); const SArray* blocks = pBlock->blocks; qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK); @@ -278,7 +415,7 @@ int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) { return 0; } -int32_t streamTaskExecNew(SStreamTask* pTask) { +int32_t streamTaskRun(SStreamTask* pTask) { SArray* pRes = NULL; if (pTask->execType == TASK_EXEC__PIPE || pTask->execType == TASK_EXEC__MERGE) { // TODO remove multi runner @@ -494,11 +631,16 @@ SStreamTask* tNewSStreamTask(int64_t streamId) { pTask->inputQ = taosOpenQueue(); pTask->outputQ = taosOpenQueue(); - if (pTask->inputQ == NULL || pTask->outputQ == NULL) goto FAIL; + pTask->inputQAll = taosAllocateQall(); + pTask->outputQAll = taosAllocateQall(); + if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL) + goto FAIL; return pTask; FAIL: if (pTask->inputQ) taosCloseQueue(pTask->inputQ); if (pTask->outputQ) taosCloseQueue(pTask->outputQ); + if (pTask->inputQAll) taosFreeQall(pTask->inputQAll); + if (pTask->outputQAll) taosFreeQall(pTask->outputQAll); if (pTask) taosMemoryFree(pTask); return NULL; } @@ -507,6 +649,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { /*if (tStartEncode(pEncoder) < 0) return -1;*/ if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; @@ -552,6 +695,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { /*if (tStartDecode(pDecoder) < 0) return -1;*/ if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 63f24b104f..0a6e2428fe 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -35,6 +35,7 @@ typedef struct SSyncIndexMgr { } SSyncIndexMgr; SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode); +void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode); void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8a21eea7b7..9b655fb0fa 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -247,6 +247,7 @@ typedef struct SSyncNode { // open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); +void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); // ping -------------- @@ -271,7 +272,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); -void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig); +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig); SSyncNode* syncNodeAcquire(int64_t rid); void syncNodeRelease(SSyncNode* pNode); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 159af1610e..1b08d3f7a1 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -62,7 +62,6 @@ bool syncUtilUserPreCommit(tmsg_t msgType); bool syncUtilUserCommit(tmsg_t msgType); bool syncUtilUserRollback(tmsg_t msgType); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index aed19d042e..1a5d418e75 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -15,11 +15,11 @@ #include "syncAppendEntries.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" -#include "syncRaftCfg.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == @@ -200,7 +200,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); assert(pRollBackEntry != NULL); - //if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { + // if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { if (syncUtilUserRollback(pRollBackEntry->msgType)) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); @@ -229,7 +229,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; @@ -261,7 +261,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; @@ -324,7 +324,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - //if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -338,10 +338,15 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // config change if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); ASSERT(ret == 0); syncNodeUpdateConfig(ths, &newSyncCfg); + if (ths->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(ths); + } else { + syncNodeBecomeFollower(ths); + } } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 620f0e9cd2..0f17cf267e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -16,10 +16,10 @@ #include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" +#include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" -#include "syncRaftCfg.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, @@ -102,7 +102,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - //if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -114,12 +114,17 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } // config change - if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { - SSyncCfg newSyncCfg; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); - ASSERT(ret == 0); + if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + SSyncCfg newSyncCfg; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg); + ASSERT(ret == 0); - syncNodeUpdateConfig(pSyncNode, &newSyncCfg); + syncNodeUpdateConfig(pSyncNode, &newSyncCfg); + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + syncNodeBecomeLeader(pSyncNode); + } else { + syncNodeBecomeFollower(pSyncNode); + } } rpcFreeCont(rpcMsg.pCont); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index a3d1717c3b..8b7411c563 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -15,6 +15,7 @@ #include "syncIO.h" #include +#include "os.h" #include "syncMessage.h" #include "syncUtil.h" #include "tglobal.h" @@ -198,6 +199,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); + snprintf(rpcInit.localFqdn, sizeof(rpcInit.localFqdn), "%s", "127.0.0.1"); rpcInit.localPort = io->myAddr.eps[0].port; rpcInit.label = "SYNC-IO-SERVER"; rpcInit.numOfThreads = 1; diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index d33075054a..5809cedb90 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -31,6 +31,13 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { return pSyncIndexMgr; } +void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode) { + pSyncIndexMgr->replicas = &(pSyncNode->replicasId); + pSyncIndexMgr->replicaNum = pSyncNode->replicaNum; + pSyncIndexMgr->pSyncNode = pSyncNode; + syncIndexMgrClear(pSyncIndexMgr); +} + void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { if (pSyncIndexMgr != NULL) { taosMemoryFree(pSyncIndexMgr); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index da23d8415b..469682a7b5 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -103,6 +103,16 @@ void syncStart(int64_t rid) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); } +void syncStartStandBy(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return; + } + syncNodeStartStandBy(pSyncNode); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + void syncStop(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -116,7 +126,7 @@ void syncStop(int64_t rid) { int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { int32_t ret = 0; - char *configChange = syncCfg2Str((SSyncCfg*)pSyncCfg); + char* configChange = syncCfg2Str((SSyncCfg*)pSyncCfg); SRpcMsg rpcMsg = {0}; rpcMsg.msgType = TDMT_VND_SYNC_CONFIG_CHANGE; rpcMsg.noResp = 1; @@ -188,10 +198,9 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { (pEpSet->numOfEps)++; sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; - + sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -524,6 +533,17 @@ void syncNodeStart(SSyncNode* pSyncNode) { assert(ret == 0); } +void syncNodeStartStandBy(SSyncNode* pSyncNode) { + // state change + pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; + syncNodeStopHeartbeatTimer(pSyncNode); + + // reset elect timer, long enough + int32_t electMS = TIMER_MAX_MS; + int32_t ret = syncNodeRestartElectTimer(pSyncNode, electMS); + ASSERT(ret == 0); +} + void syncNodeClose(SSyncNode* pSyncNode) { int32_t ret; assert(pSyncNode != NULL); @@ -858,7 +878,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { return s; } -void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) { +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig) { pSyncNode->pRaftCfg->cfg = *newConfig; int32_t ret = raftCfgPersist(pSyncNode->pRaftCfg); ASSERT(ret == 0); @@ -885,6 +905,11 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) { for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } + + syncIndexMgrUpdate(pSyncNode->pNextIndex, pSyncNode); + syncIndexMgrUpdate(pSyncNode->pMatchIndex, pSyncNode); + + syncNodeLog2("==syncNodeUpdateConfig==", pSyncNode); } SSyncNode* syncNodeAcquire(int64_t rid) { @@ -1245,7 +1270,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -1267,7 +1292,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 8aeb9c4856..07a9397a58 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -58,14 +58,15 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); - sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); ASSERT(0); - } - //assert(code == 0); + } + // assert(code == 0); walFsync(pWal, true); return code; @@ -77,16 +78,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - int32_t code = walReadWithHandle(pWalHandle, index); + int32_t code = walReadWithHandle(pWalHandle, index); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); - sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); ASSERT(0); - } - //assert(walReadWithHandle(pWalHandle, index) == 0); + } + // assert(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); assert(pEntry != NULL); @@ -112,16 +114,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - //assert(walRollback(pWal, fromIndex) == 0); + // assert(walRollback(pWal, fromIndex) == 0); int32_t code = walRollback(pWal, fromIndex); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); - sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); ASSERT(0); - } + } return 0; // to avoid compiler error } @@ -145,16 +148,16 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - //assert(walCommit(pWal, index) == 0); + // assert(walCommit(pWal, index) == 0); int32_t code = walCommit(pWal, index); if (code != 0) { - int32_t err = terrno; - const char *errStr = tstrerror(err); - int32_t linuxErr = errno; - const char *linuxErrMsg = strerror(errno); + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); - } + } return 0; // to avoid compiler error } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 8afe9ff2a7..cfbdf0e961 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -37,6 +37,7 @@ add_executable(syncRaftCfgTest "") add_executable(syncRespMgrTest "") add_executable(syncSnapshotTest "") add_executable(syncApplyMsgTest "") +add_executable(syncConfigChangeTest "") target_sources(syncTest @@ -195,6 +196,10 @@ target_sources(syncApplyMsgTest PRIVATE "syncApplyMsgTest.cpp" ) +target_sources(syncConfigChangeTest + PRIVATE + "syncConfigChangeTest.cpp" +) target_include_directories(syncTest @@ -392,6 +397,11 @@ target_include_directories(syncApplyMsgTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncConfigChangeTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -550,6 +560,10 @@ target_link_libraries(syncApplyMsgTest sync gtest_main ) +target_link_libraries(syncConfigChangeTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp new file mode 100644 index 0000000000..9a2d9a6b34 --- /dev/null +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -0,0 +1,259 @@ +#include +#include +#include "os.h" +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t gPorts[] = {7010, 7110, 7210, 7310, 7410}; +const char* gDir = "./syncReplicateTest"; +int32_t gVgId = 1234; +SyncIndex gSnapshotLastApplyIndex; + +void init() { + int code = walInit(); + assert(code == 0); + + code = syncInit(); + assert(code == 0); + + sprintf(tsTempDir, "%s", "."); +} + +void cleanup() { walCleanUp(); } + +void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + SyncIndex beginIndex = SYNC_INDEX_INVALID; + if (pFsm->FpGetSnapshot != NULL) { + SSnapshot snapshot; + pFsm->FpGetSnapshot(pFsm, &snapshot); + beginIndex = snapshot.lastApplyIndex; + } + + if (cbMeta.index > beginIndex) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); + } else { + sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + } +} + +void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + +int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { + pSnapshot->data = NULL; + pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; + pSnapshot->lastApplyTerm = 100; + return 0; +} + +SSyncFSM* createFsm() { + SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; + pFsm->FpGetSnapshot = GetSnapshotCb; + return pFsm; +} + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +int64_t createSyncNode(int32_t replicaNum, int32_t myIndex, int32_t vgId, SWal* pWal, char* path, bool isStandBy) { + SSyncInfo syncInfo; + syncInfo.vgId = vgId; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = createFsm(); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s_sync_replica%d_index%d", path, replicaNum, myIndex); + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + + if (isStandBy) { + pCfg->myIndex = 0; + pCfg->replicaNum = 1; + pCfg->nodeInfo[0].nodePort = gPorts[myIndex]; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + } else { + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(pCfg->nodeInfo[i].nodeFqdn); + // snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + } + } + + int64_t rid = syncOpen(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + syncNodeRelease(pSyncNode); + + return rid; +} + +void configChange(int64_t rid, int32_t replicaNum, int32_t myIndex) { + SSyncCfg syncCfg; + + syncCfg.myIndex = myIndex; + syncCfg.replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + syncCfg.nodeInfo[i].nodePort = gPorts[i]; + taosGetFqdn(syncCfg.nodeInfo[i].nodeFqdn); + } + + syncReconfig(rid, &syncCfg); +} + +void usage(char* exe) { + printf("usage: %s replicaNum myIndex lastApplyIndex writeRecordNum isStandBy isConfigChange \n", exe); +} + +SRpcMsg* createRpcMsg(int i, int count, int myIndex) { + SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 256; + pMsg->pCont = rpcMallocCont(pMsg->contLen); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + return pMsg; +} + +int main(int argc, char** argv) { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + if (argc != 7) { + usage(argv[0]); + exit(-1); + } + + int32_t replicaNum = atoi(argv[1]); + int32_t myIndex = atoi(argv[2]); + int32_t lastApplyIndex = atoi(argv[3]); + int32_t writeRecordNum = atoi(argv[4]); + bool isStandBy = atoi(argv[5]); + bool isConfigChange = atoi(argv[6]); + gSnapshotLastApplyIndex = lastApplyIndex; + + if (!isStandBy) { + assert(replicaNum >= 1 && replicaNum <= 5); + assert(myIndex >= 0 && myIndex < replicaNum); + assert(lastApplyIndex >= -1); + assert(writeRecordNum >= 0); + } + + init(); + int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); + assert(ret == 0); + + char walPath[128]; + snprintf(walPath, sizeof(walPath), "%s_wal_replica%d_index%d", gDir, replicaNum, myIndex); + SWal* pWal = createWal(walPath, gVgId); + + int64_t rid = createSyncNode(replicaNum, myIndex, gVgId, pWal, (char*)gDir, isStandBy); + assert(rid > 0); + + if (isStandBy) { + syncStartStandBy(rid); + } else { + syncStart(rid); + } + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + if (isConfigChange) { + configChange(rid, 3, myIndex); + } + + //--------------------------- + int32_t alreadySend = 0; + while (1) { + char* s = syncNode2SimpleStr(pSyncNode); + + if (alreadySend < writeRecordNum) { + SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); + int32_t ret = syncPropose(rid, pRpcMsg, false); + if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + sTrace("%s value%d write not leader", s, alreadySend); + } else { + assert(ret == 0); + sTrace("%s value%d write ok", s, alreadySend); + } + alreadySend++; + + rpcFreeCont(pRpcMsg->pCont); + taosMemoryFree(pRpcMsg); + } else { + sTrace("%s", s); + } + + taosMsleep(1000); + taosMemoryFree(s); + taosMsleep(1000); + } + + syncNodeRelease(pSyncNode); + syncStop(rid); + walClose(pWal); + syncIOStop(); + cleanup(); + return 0; +} diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index fc840691b6..72ab250709 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -137,7 +137,9 @@ static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); -static int reallocConnRefHandle(SSrvConn* conn); +static void destroyConnRegArg(SSrvConn* conn); + +static int reallocConnRefHandle(SSrvConn* conn); static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd); @@ -429,6 +431,8 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { if (smsg->type == Release) { pHead->msgType = 0; pConn->status = ConnNormal; + + destroyConnRegArg(pConn); transUnrefSrvHandle(pConn); } else { pHead->msgType = pMsg->msgType; @@ -800,6 +804,12 @@ static void destroyConn(SSrvConn* conn, bool clear) { // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } } +static void destroyConnRegArg(SSrvConn* conn) { + if (conn->regArg.init == 1) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } +} static int reallocConnRefHandle(SSrvConn* conn) { uvReleaseExHandle(conn->refId); uvRemoveExHandle(conn->refId); @@ -827,16 +837,9 @@ static void uvDestroyConn(uv_handle_t* handle) { // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); - if (conn->regArg.init == 1) { - transFreeMsg(conn->regArg.msg.pCont); - conn->regArg.init = 0; - } QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); - if (conn->regArg.init == 1) { - transFreeMsg(conn->regArg.msg.pCont); - conn->regArg.init = 0; - } + destroyConnRegArg(conn); taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a2b0648a4b..b5e64242e4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -322,6 +322,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST, "SMA not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST, "Table does not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION, "Invalid table action") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS, "Table column not exists") + // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -354,6 +358,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "Tsma already exists") // query diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 8531f2c8ea..9970ac24d7 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -94,6 +94,7 @@ int32_t tqDebugFlag = 135; int32_t fsDebugFlag = 135; int32_t metaDebugFlag = 135; int32_t fnDebugFlag = 135; +int32_t smaDebugFlag = 135; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; @@ -755,6 +756,7 @@ void taosSetAllDebugFlag(int32_t flag) { tqDebugFlag = flag; fsDebugFlag = flag; fnDebugFlag = flag; + smaDebugFlag = flag; uInfo("all debug flag are set to %d", flag); } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 0d16a2129a..c1053ce7de 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -33,13 +33,13 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) -# freelistTest -add_executable(freelistTest "") -target_sources(freelistTest - PRIVATE - "freelistTest.cpp" -) -target_link_libraries(freelistTest os util gtest gtest_main) +# # freelistTest +# add_executable(freelistTest "") +# target_sources(freelistTest +# PRIVATE +# "freelistTest.cpp" +# ) +# target_link_libraries(freelistTest os util gtest gtest_main) # # encodeTest # add_executable(encodeTest "encodeTest.cpp") diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp deleted file mode 100644 index a445a16ad3..0000000000 --- a/source/util/test/freelistTest.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include - -#include "tfreelist.h" - -TEST(TD_UTIL_FREELIST_TEST, simple_test) { - SFreeList fl; - - tFreeListInit(&fl); - - for (size_t i = 0; i < 1000; i++) { - void *ptr = NULL; - TFL_MALLOC(ptr, void*, 1024, &fl); - GTEST_ASSERT_NE(ptr, nullptr); - } - - tFreeListClear(&fl); -}