diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 97bfcfb8c0..aba955ff3b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -226,10 +226,10 @@ endif(${BUILD_WITH_NURAFT}) if(${BUILD_PTHREAD}) set(CMAKE_BUILD_TYPE release) add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) + add_subdirectory(pthread EXCLUDE_FROM_ALL) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) - add_library(pthread STATIC IMPORTED GLOBAL) - SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) + add_library(pthread INTERFACE) + target_link_libraries(pthread INTERFACE libpthreadVC3) endif() # iconv diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index c80b8db58a..e13705d403 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -29,27 +29,42 @@ extern "C" { typedef struct SSchema SSchema; typedef struct STColumn STColumn; typedef struct STSchema STSchema; +typedef struct SColVal SColVal; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; -typedef struct SKVIdx SKVIdx; - -// STSchema - -// STSRow2 -int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); -int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); +typedef struct STagVal STagVal; +typedef struct STag STag; // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); +// SColVal +#define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) +#define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) +#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) + +// STSRow2 +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); +int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); +int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); +void tTSRowFree(STSRow2 *pRow); +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); + // STSRowBuilder -int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols); +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema); void tTSRowBuilderClear(STSRowBuilder *pBuilder); void tTSRowBuilderReset(STSRowBuilder *pBuilder); -int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData); +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); +// STag +int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag); +void tTagFree(STag *pTag); +void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData); +int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag); +int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag); + // STRUCT ================= struct STColumn { col_id_t colId; @@ -68,31 +83,47 @@ struct STSchema { STColumn columns[]; }; +#define TSROW_HAS_NONE ((uint8_t)0x1) +#define TSROW_HAS_NULL ((uint8_t)0x2U) +#define TSROW_HAS_VAL ((uint8_t)0x4U) +#define TSROW_KV_ROW ((uint8_t)0x10U) struct STSRow2 { TSKEY ts; - uint32_t flags; - union { - int32_t sver; - int32_t ncols; - }; - uint32_t nData; - const uint8_t *pData; + uint8_t flags; + int32_t sver; + uint32_t nData; + uint8_t *pData; }; struct STSRowBuilder { - STColumn *pTColumn; STSchema *pTSchema; + int32_t szBitMap1; + int32_t szBitMap2; int32_t szKVBuf; uint8_t *pKVBuf; int32_t szTPBuf; uint8_t *pTPBuf; - int32_t nCols; - int32_t kvVLen; - int32_t tpVLen; + int32_t iCol; + int32_t vlenKV; + int32_t vlenTP; STSRow2 row; }; -#if 1 //==================================== +typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT; +struct SColVal { + EColValT type; + uint32_t nData; + uint8_t *pData; +}; + +struct STagVal { + int16_t cid; + int8_t type; + uint32_t nData; + uint8_t *pData; +}; + +#if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP #define TD_SUPPORT_READ2 diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3ff566afe2..7a60542313 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -275,10 +275,10 @@ int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); void tFreeSSubmitRsp(SSubmitRsp* pRsp); -#define COL_SMA_ON ((int8_t)0x1) -#define COL_IDX_ON ((int8_t)0x2) -#define COL_VAL_SET ((int8_t)0x4) - +#define COL_SMA_ON ((int8_t)0x1) +#define COL_IDX_ON ((int8_t)0x2) +#define COL_SET_NULL ((int8_t)0x10) +#define COL_SET_VAL ((int8_t)0x20) typedef struct SSchema { int8_t type; int8_t flags; @@ -287,6 +287,9 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +#define COL_IS_SET(FLG) ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0) +#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) + #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) #define SSCHMEA_TYPE(s) ((s)->type) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 162b6fb2ed..8f300c96c5 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -42,7 +42,7 @@ typedef struct SReadHandle { #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2 typedef enum { - OPTR_EXEC_MODEL_BATCH = 0x1, + OPTR_EXEC_MODEL_BATCH = 0x1, OPTR_EXEC_MODEL_STREAM = 0x2, } EOPTR_EXEC_MODEL; @@ -81,7 +81,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO * @param isAdd * @return */ -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json @@ -169,7 +169,7 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type); void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); -int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes); +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes); #ifdef __cplusplus } diff --git a/include/util/tencode.h b/include/util/tencode.h index e49429f865..938e3018a8 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -456,52 +456,189 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { return p; } -static FORCE_INLINE int32_t tPutBinary(uint8_t* p, const uint8_t* pData, uint32_t nData) { - int n = 0; - uint32_t v = nData; +// =========================================== +#define tPutV(p, v) \ + do { \ + int32_t n = 0; \ + for (;;) { \ + if (v <= 0x7f) { \ + if (p) p[n] = v; \ + n++; \ + break; \ + } \ + if (p) p[n] = (v & 0x7f) | 0x80; \ + n++; \ + v >>= 7; \ + } \ + return n; \ + } while (0) - for (;;) { - if (v <= 0x7f) { - if (p) p[n] = v; - n++; - break; - } +#define tGetV(p, v) \ + do { \ + int32_t n = 0; \ + if (v) *v = 0; \ + for (;;) { \ + if (p[n] <= 0x7f) { \ + if (v) (*v) |= (p[n] << (7 * n)); \ + n++; \ + break; \ + } \ + if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ + n++; \ + } \ + return n; \ + } while (0) - if (p) p[n] = (v & 0x7f) | 0x80; - n++; - v >>= 7; - } +// PUT +static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { + if (p) ((uint8_t*)p)[0] = v; + return sizeof(uint8_t); +} - if (p) { - memcpy(p + n, pData, nData); - } +static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { + if (p) ((int8_t*)p)[0] = v; + return sizeof(int8_t); +} + +static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) { + if (p) ((uint16_t*)p)[0] = v; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) { + if (p) ((int16_t*)p)[0] = v; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) { + if (p) ((uint32_t*)p)[0] = v; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) { + if (p) ((int32_t*)p)[0] = v; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) { + if (p) ((uint64_t*)p)[0] = v; + return sizeof(uint64_t); +} + +static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { + if (p) ((int64_t*)p)[0] = v; + return sizeof(int64_t); +} + +static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } + +static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } + +static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } + +// GET +static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { + if (v) *v = ((uint8_t*)p)[0]; + return sizeof(uint8_t); +} + +static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { + if (v) *v = ((int8_t*)p)[0]; + return sizeof(int8_t); +} + +static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) { + if (v) *v = ((uint16_t*)p)[0]; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) { + if (v) *v = ((int16_t*)p)[0]; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) { + if (v) *v = ((uint32_t*)p)[0]; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) { + if (v) *v = ((int32_t*)p)[0]; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) { + if (v) *v = ((uint64_t*)p)[0]; + return sizeof(uint64_t); +} + +static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { + if (v) *v = ((int64_t*)p)[0]; + return sizeof(int64_t); +} + +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { + int32_t n; + uint16_t tv; + + n = tGetU16v(p, &tv); + if (v) *v = ZIGZAGD(int16_t, tv); + + return n; +} + +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { + int32_t n; + uint32_t tv; + + n = tGetU32v(p, &tv); + if (v) *v = ZIGZAGD(int32_t, tv); + + return n; +} + +static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { + int32_t n; + uint64_t tv; + + n = tGetU64v(p, &tv); + if (v) *v = ZIGZAGD(int64_t, tv); + + return n; +} + +// ===================== +static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { + int n = 0; + + n += tPutU32v(p ? p + n : p, nData); + if (p) memcpy(p + n, pData, nData); n += nData; return n; } -static FORCE_INLINE int32_t tGetBinary(const uint8_t* p, const uint8_t** ppData, uint32_t* nData) { +static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) { int32_t n = 0; - uint32_t tv = 0; - uint32_t t; + uint32_t nt; - for (;;) { - if (p[n] <= 0x7f) { - t = p[n]; - tv |= (t << (7 * n)); - n++; - break; - } - - t = p[n] & 0x7f; - tv |= (t << (7 * n)); - n++; - } - - if (nData) *nData = n; + n += tGetU32v(p, &nt); + if (nData) *nData = nt; if (ppData) *ppData = p + n; + n += nt; - n += tv; return n; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 884756ced9..f6f8db8590 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -714,25 +714,31 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { char *endPtr = NULL; - double ts = (double)strtoll(value, &endPtr, 10); + int64_t tsInt64 = strtoll(value, &endPtr, 10); if(value + len != endPtr){ return -1; } + double ts = tsInt64; switch (type) { case TSDB_TIME_PRECISION_HOURS: ts *= (3600 * 1e9); + tsInt64 *= (3600 * 1e9); break; case TSDB_TIME_PRECISION_MINUTES: ts *= (60 * 1e9); + tsInt64 *= (60 * 1e9); break; case TSDB_TIME_PRECISION_SECONDS: ts *= (1e9); + tsInt64 *= (1e9); break; case TSDB_TIME_PRECISION_MILLI: ts *= (1e6); + tsInt64 *= (1e6); break; case TSDB_TIME_PRECISION_MICRO: ts *= (1e3); + tsInt64 *= (1e3); break; case TSDB_TIME_PRECISION_NANO: break; @@ -743,7 +749,7 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { return -1; } - return (int64_t)ts; + return tsInt64; } static int64_t smlGetTimeNow(int8_t precision) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index f2c5fdec0f..764571e71e 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -260,8 +260,8 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { - if (STMT_TYPE_QUERY != pStmt->sql.type || freeRequest) { +int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { + if (STMT_TYPE_QUERY != pStmt->sql.type || deepClean) { taos_free_result(pStmt->exec.pRequest); pStmt->exec.pRequest = NULL; } @@ -280,7 +280,11 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { continue; } - qFreeStmtDataBlock(pBlocks); + if (STMT_TYPE_MULTI_INSERT == pStmt->sql.type) { + qFreeStmtDataBlock(pBlocks); + } else { + qDestroyStmtDataBlock(pBlocks); + } taosHashRemove(pStmt->exec.pBlockHash, key, keyLen); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -320,11 +324,11 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { taosHashCleanup(pStmt->sql.pTableCache); pStmt->sql.pTableCache = NULL; - memset(&pStmt->sql, 0, sizeof(pStmt->sql)); - STMT_ERR_RET(stmtCleanExecInfo(pStmt, false, true)); STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + memset(&pStmt->sql, 0, sizeof(pStmt->sql)); + return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ecfc991331..2a2fdea537 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -830,10 +830,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } // init hb timer - tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + if (tmq->hbTimer == NULL) { + tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + } // init auto commit timer - if (tmq->autoCommit) { + if (tmq->autoCommit && tmq->commitTimer == NULL) { tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer); } @@ -1456,9 +1458,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { - tmq_list_t* lst = tmq_list_new(); - tmq_resp_err_t rsp = tmq_subscribe(tmq, lst); + tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL); + if (rsp == TMQ_RESP_ERR__SUCCESS) { + // TODO: free resources + return TMQ_RESP_ERR__SUCCESS; + } else { + return TMQ_RESP_ERR__FAIL; + } + + tmq_list_t* lst = tmq_list_new(); + rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); + if (rsp == TMQ_RESP_ERR__SUCCESS) { // TODO: free resources return TMQ_RESP_ERR__SUCCESS; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index eeab844712..d9a81ad3e6 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1188,3 +1188,36 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { destroyRequest(request); smlDestroyInfo(info); } + +TEST(testCase, sml_TD15662_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_15662 precision 'ns'"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_15662"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_NE(info, nullptr); + + const char *sql[] = { + "iyyyje,id=iyyyje_41943_1303,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + }; + int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0])); + ASSERT_EQ(ret, 0); + + // case 1 + TAOS_RES *res = taos_query(taos, "select * from t_a5615048edae55218a22a149edebdc82"); + ASSERT_NE(res, nullptr); + + TAOS_ROW row = taos_fetch_row(res); + int64_t ts = *(int64_t*)row[0]; + ASSERT_EQ(ts, 1626006833639000000); + + taos_free_result(res); +} \ No newline at end of file diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad020fe7d9..8aa8ed2f14 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,37 +19,209 @@ #include "tdatablock.h" #include "tlog.h" -#define TD_KV_ROW 0x1U - -struct SKVIdx { +typedef struct SKVIdx { int32_t cid; int32_t offset; +} SKVIdx; + +#pragma pack(push, 1) +typedef struct { + int16_t nCols; + SKVIdx idx[]; +} STSKVRow; +#pragma pack(pop) + +typedef struct STagIdx { + int16_t cid; + uint16_t offset; +} STagIdx; + +#pragma pack(push, 1) +struct STag { + uint16_t len; + uint16_t nTag; + STagIdx idx[]; }; +#pragma pack(pop) -int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { - if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; - if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; - if (pRow->flags & TD_KV_ROW) { - if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; - } else { - if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; +#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) +#define BIT1_SIZE(n) (((n)-1) / 8 + 1) +#define BIT2_SIZE(n) (((n)-1) / 4 + 1) +#define SET_BIT1(p, i, v) ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8))) +#define SET_BIT2(p, i, v) ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4))) +#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) +#define GET_BIT2(p, i) (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3)) + +static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); + +// STSRow2 +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { + int32_t n = 0; + + n += tPutI64(p ? p + n : p, pRow->ts); + n += tPutI8(p ? p + n : p, pRow->flags); + n += tPutI32v(p ? p + n : p, pRow->sver); + + ASSERT(pRow->flags & 0xf); + + switch (pRow->flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + break; + default: + n += tPutBinary(p ? p + n : p, pRow->pData, pRow->nData); + break; } - if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData) < 0) return -1; + + return n; +} + +int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) { + int32_t n = 0; + uint8_t flags; + + n += tGetI64(p + n, pRow ? &pRow->ts : NULL); + n += tGetI8(p + n, pRow ? &pRow->flags : &flags); + n += tGetI32v(p + n, pRow ? &pRow->sver : NULL); + + if (pRow) flags = pRow->flags; + switch (flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + break; + default: + n += tGetBinary(p + n, pRow ? &pRow->pData : NULL, pRow ? &pRow->nData : NULL); + break; + } + + return n; +} + +int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow) { + (*ppRow) = taosMemoryMalloc(sizeof(*pRow) + pRow->nData); + if (*ppRow == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (*ppRow)->ts = pRow->ts; + (*ppRow)->flags = pRow->flags; + (*ppRow)->sver = pRow->sver; + (*ppRow)->nData = pRow->nData; + if (pRow->nData) { + (*ppRow)->pData = (uint8_t *)(&(*ppRow)[1]); + memcpy((*ppRow)->pData, pRow->pData, pRow->nData); + } else { + (*ppRow)->pData = NULL; + } + return 0; } -int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { - if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1; - if (tDecodeU32v(pDecoder, &pRow->flags) < 0) return -1; - if (pRow->flags & TD_KV_ROW) { - if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; - } else { - if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; +void tTSRowFree(STSRow2 *pRow) { + if (pRow) taosMemoryFree(pRow); +} + +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + uint32_t n; + uint8_t *p; + uint8_t v; + int32_t bidx = iCol - 1; + STColumn *pTColumn = &pTSchema->columns[iCol]; + STSKVRow *pTSKVRow; + SKVIdx *pKVIdx; + + ASSERT(iCol != 0); + ASSERT(pTColumn->colId != 0); + + ASSERT(pRow->flags & 0xf != 0); + switch (pRow->flags & 0xf) { + case TSROW_HAS_NONE: + *pColVal = ColValNONE; + return 0; + case TSROW_HAS_NULL: + *pColVal = ColValNULL; + return 0; } - if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData) < 0) return -1; + + if (TSROW_IS_KV_ROW(pRow)) { + ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL); + + pTSKVRow = (STSKVRow *)pRow->pData; + pKVIdx = + bsearch(&((SKVIdx){.cid = pTColumn->colId}), pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); + if (pKVIdx == NULL) { + *pColVal = ColValNONE; + } else if (pKVIdx->offset < 0) { + *pColVal = ColValNULL; + } else { + p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset; + pColVal->type = COL_VAL_DATA; + tGetBinary(p, &pColVal->pData, &pColVal->nData); + } + } else { + // get bitmap + p = pRow->pData; + switch (pRow->flags & 0xf) { + case TSROW_HAS_NULL | TSROW_HAS_NONE: + v = GET_BIT1(p, bidx); + if (v == 0) { + *pColVal = ColValNONE; + } else { + *pColVal = ColValNULL; + } + return 0; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + v = GET_BIT1(p, bidx); + if (v == 1) { + p = p + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } else { + *pColVal = ColValNONE; + return 0; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL: + v = GET_BIT1(p, bidx); + if (v == 1) { + p = p + BIT1_SIZE(pTSchema->numOfCols - 1); + break; + } else { + *pColVal = ColValNULL; + return 0; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + v = GET_BIT2(p, bidx); + if (v == 0) { + *pColVal = ColValNONE; + return 0; + } else if (v == 1) { + *pColVal = ColValNULL; + return 0; + } else if (v == 2) { + p = p + BIT2_SIZE(pTSchema->numOfCols - 1); + break; + } else { + ASSERT(0); + } + default: + break; + } + + // get real value + p = p + pTColumn->offset; + pColVal->type = COL_VAL_DATA; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + tGetBinary(p + pTSchema->flen + *(int32_t *)p, &pColVal->pData, &pColVal->nData); + } else { + pColVal->pData = p; + pColVal->nData = pTColumn->bytes; + } + } + return 0; } +// STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); if (*ppTSchema == NULL) { @@ -85,170 +257,360 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema * return 0; } -void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } - -int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { - int32_t kvBufLen; - int32_t tpBufLen; - uint8_t *p; +void tTSchemaDestroy(STSchema *pTSchema) { + if (pTSchema) taosMemoryFree(pTSchema); +} +// STSRowBuilder +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema) { if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; - kvBufLen = sizeof(SKVIdx) * nCols + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; - tpBufLen = pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; - - if (pBuilder->szKVBuf < kvBufLen) { - p = taosMemoryRealloc(pBuilder->pKVBuf, kvBufLen); - if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pBuilder->pKVBuf = p; - pBuilder->szKVBuf = kvBufLen; + pBuilder->szBitMap1 = BIT1_SIZE(nCols - 1); + pBuilder->szBitMap2 = BIT2_SIZE(nCols - 1); + pBuilder->szKVBuf = + sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf); + if (pBuilder->pKVBuf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tTSchemaDestroy(pBuilder->pTSchema); + return -1; } - - if (pBuilder->szTPBuf < tpBufLen) { - p = taosMemoryRealloc(pBuilder->pTPBuf, tpBufLen); - if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pBuilder->pTPBuf = p; - pBuilder->szTPBuf = tpBufLen; + pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf); + if (pBuilder->pTPBuf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pBuilder->pKVBuf); + tTSchemaDestroy(pBuilder->pTSchema); + return -1; } - tTSRowBuilderReset(pBuilder); - return 0; } void tTSRowBuilderClear(STSRowBuilder *pBuilder) { - taosMemoryFree(pBuilder->pKVBuf); - taosMemoryFree(pBuilder->pTPBuf); + if (pBuilder->pTPBuf) { + taosMemoryFree(pBuilder->pTPBuf); + pBuilder->pTPBuf = NULL; + } + if (pBuilder->pKVBuf) { + taosMemoryFree(pBuilder->pKVBuf); + pBuilder->pKVBuf = NULL; + } + tTSchemaDestroy(pBuilder->pTSchema); + pBuilder->pTSchema = NULL; } void tTSRowBuilderReset(STSRowBuilder *pBuilder) { for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - - pBuilder->pTColumn->flags &= (~COL_VAL_SET); + STColumn *pTColumn = &pBuilder->pTSchema->columns[iCol]; + COL_CLR_SET(pTColumn->flags); } - pBuilder->nCols = 0; - pBuilder->kvVLen = 0; - pBuilder->tpVLen = 0; + pBuilder->iCol = 0; + ((STSKVRow *)pBuilder->pKVBuf)->nCols = 0; + pBuilder->vlenKV = 0; + pBuilder->vlenTP = 0; pBuilder->row.flags = 0; } -int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { - int32_t iCol; - uint8_t *p; +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData) { + STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol]; + uint8_t *p; + int32_t iCol; + STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; - // search column - if (pBuilder->pTColumn->colId < cid) { - iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; - for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pBuilder->pTColumn->colId == cid) break; + // use interp search + if (pTColumn->colId < cid) { // right search + for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pTColumn->colId >= cid) break; } - } else if (pBuilder->pTColumn->colId > cid) { - iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; - for (; iCol >= 0; iCol--) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pBuilder->pTColumn->colId == cid) break; + } else if (pTColumn->colId > cid) { // left search + for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) { + pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pTColumn->colId <= cid) break; } } - // check - if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) { + if (pTColumn->colId != cid || COL_IS_SET(pTColumn->flags)) { return -1; } + pBuilder->iCol = iCol; + // set value if (cid == 0) { - ASSERT(pData && nData == sizeof(TSKEY)); + ASSERT(pData && nData == sizeof(TSKEY) && iCol == 0); pBuilder->row.ts = *(TSKEY *)pData; + pTColumn->flags |= COL_SET_VAL; } else { if (pData) { - // ASSERT(!IS_NULL(pData)); + // set VAL - // set tuple data - p = pBuilder->pTPBuf + pBuilder->pTColumn->offset; - if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { - *(int32_t *)p = pBuilder->tpVLen; + pBuilder->row.flags |= TSROW_HAS_VAL; + pTColumn->flags |= COL_SET_VAL; - // encode the variant-length data - p = pBuilder->pTPBuf + pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->tpVLen += tPutBinary(p, pData, nData); - } else { - memcpy(p, pData, nData); + /* KV */ + if (1) { // avoid KV at some threshold (todo) + pTSKVRow->idx[pTSKVRow->nCols].cid = cid; + pTSKVRow->idx[pTSKVRow->nCols].offset = pBuilder->vlenKV; + + p = pBuilder->pKVBuf + sizeof(STSKVRow) + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + + pBuilder->vlenKV; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ASSERT(nData <= pTColumn->bytes); + pBuilder->vlenKV += tPutBinary(p, pData, nData); + } else { + ASSERT(nData == pTColumn->bytes); + memcpy(p, pData, nData); + pBuilder->vlenKV += nData; + } } - // set kv data - p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; - ((SKVIdx *)p)->cid = cid; - ((SKVIdx *)p)->offset = pBuilder->kvVLen; + /* TUPLE */ + p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pTColumn->offset; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ASSERT(nData <= pTColumn->bytes); + *(int32_t *)p = pBuilder->vlenTP; - p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols + pBuilder->kvVLen; - if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { - pBuilder->kvVLen += tPutBinary(p, pData, nData); + p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP; + pBuilder->vlenTP += tPutBinary(p, pData, nData); } else { + ASSERT(nData == pTColumn->bytes); memcpy(p, pData, nData); - pBuilder->kvVLen += nData; } } else { - // set NULL val + // set NULL + + pBuilder->row.flags |= TSROW_HAS_NULL; + pTColumn->flags |= COL_SET_NULL; + + pTSKVRow->idx[pTSKVRow->nCols].cid = cid; + pTSKVRow->idx[pTSKVRow->nCols].offset = -1; } + + pTSKVRow->nCols++; } - pBuilder->pTColumn->flags |= COL_VAL_SET; - pBuilder->nCols++; return 0; } +static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) { + SKVIdx *pKVIdx1 = (SKVIdx *)p1; + SKVIdx *pKVIdx2 = (SKVIdx *)p2; + if (pKVIdx1->cid > pKVIdx2->cid) { + return 1; + } else if (pKVIdx1->cid < pKVIdx2->cid) { + return -1; + } + return 0; +} +static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) { + int32_t bidx; + STColumn *pTColumn; + + for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { + pTColumn = &pTSchema->columns[iCol]; + bidx = iCol - 1; + + switch (flags) { + case TSROW_HAS_NULL | TSROW_HAS_NONE: + if (pTColumn->flags & COL_SET_NULL) { + SET_BIT1(p, bidx, (uint8_t)1); + } else { + SET_BIT1(p, bidx, (uint8_t)0); + } + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + if (pTColumn->flags & COL_SET_NULL) { + SET_BIT2(p, bidx, (uint8_t)1); + } else if (pTColumn->flags & COL_SET_VAL) { + SET_BIT2(p, bidx, (uint8_t)2); + } else { + SET_BIT2(p, bidx, (uint8_t)0); + } + break; + default: + if (pTColumn->flags & COL_SET_VAL) { + SET_BIT1(p, bidx, (uint8_t)1); + } else { + SET_BIT1(p, bidx, (uint8_t)0); + } + + break; + } + } +} int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { - if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) { + int32_t nDataTP, nDataKV; + uint32_t flags; + STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; + int32_t nCols = pBuilder->pTSchema->numOfCols; + + // error not set ts + if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) { return -1; } - if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) { - // encode as TD_KV_ROW - pBuilder->row.flags |= TD_KV_ROW; - pBuilder->row.ncols = pBuilder->nCols; - pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen; + ASSERT(pTSKVRow->nCols < nCols); + if (pTSKVRow->nCols < nCols - 1) { + pBuilder->row.flags |= TSROW_HAS_NONE; + } + + ASSERT(pBuilder->row.flags & 0xf != 0); + *(ppRow) = &pBuilder->row; + switch (pBuilder->row.flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + pBuilder->row.nData = 0; + pBuilder->row.pData = NULL; + return 0; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataTP = pBuilder->szBitMap1; + break; + case TSROW_HAS_VAL: + nDataTP = pBuilder->pTSchema->flen + pBuilder->vlenTP; + break; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + nDataTP = pBuilder->szBitMap1 + pBuilder->pTSchema->flen + pBuilder->vlenTP; + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataTP = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP; + break; + default: + ASSERT(0); + } + + nDataKV = sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pBuilder->vlenKV; + pBuilder->row.sver = pBuilder->pTSchema->version; + if (nDataKV < nDataTP) { + // generate KV row + + ASSERT(pBuilder->row.flags & 0xf != TSROW_HAS_VAL); + + pBuilder->row.flags |= TSROW_KV_ROW; + pBuilder->row.nData = nDataKV; pBuilder->row.pData = pBuilder->pKVBuf; - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { - memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, - pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen); + qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); + if (pTSKVRow->nCols < nCols - 1) { + memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[nCols - 1], pBuilder->vlenKV); } } else { - // encode as TD_TUPLE_ROW - pBuilder->row.flags &= (~TD_KV_ROW); - pBuilder->row.sver = pBuilder->pTSchema->version; - pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->row.pData = pBuilder->pTPBuf; + // generate TUPLE row - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { - // set non-set cols as None - for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pBuilder->pTColumn->flags & COL_VAL_SET) continue; + pBuilder->row.nData = nDataTP; - { - // set None (todo) - } + uint8_t *p; + uint8_t flags = pBuilder->row.flags & 0xf; - pBuilder->pTColumn->flags |= COL_VAL_SET; + if (flags == TSROW_HAS_VAL) { + pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2; + } else { + if (flags == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) { + pBuilder->row.pData = pBuilder->pTPBuf; + } else { + pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; } + + setBitMap(pBuilder->row.pData, pBuilder->pTSchema, flags); } } - *ppRow = &pBuilder->row; return 0; } -#if 1 // ==================== +static FORCE_INLINE int tTagIdxCmprFn(const void *p1, const void *p2) { + STagIdx *pTagIdx1 = (STagIdx *)p1; + STagIdx *pTagIdx2 = (STagIdx *)p2; + if (pTagIdx1->cid < pTagIdx1->cid) { + return -1; + } else if (pTagIdx1->cid > pTagIdx1->cid) { + return 1; + } + return 0; +} +int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag) { + STagVal *pTagVal; + uint8_t *p; + int32_t n; + uint16_t tsize = sizeof(STag) + sizeof(STagIdx) * nTag; + + for (int16_t iTag = 0; iTag < nTag; iTag++) { + pTagVal = &pTagVals[iTag]; + + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + tsize += tPutBinary(NULL, pTagVal->pData, pTagVal->nData); + } else { + ASSERT(pTagVal->nData == TYPE_BYTES[pTagVal->type]); + tsize += pTagVal->nData; + } + } + + (*ppTag) = (STag *)taosMemoryMalloc(tsize); + if (*ppTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + p = (uint8_t *)&((*ppTag)->idx[nTag]); + n = 0; + + (*ppTag)->len = tsize; + (*ppTag)->nTag = nTag; + for (int16_t iTag = 0; iTag < nTag; iTag++) { + pTagVal = &pTagVals[iTag]; + + (*ppTag)->idx[iTag].cid = pTagVal->cid; + (*ppTag)->idx[iTag].offset = n; + + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + n += tPutBinary(p + n, pTagVal->pData, pTagVal->nData); + } else { + memcpy(p + n, pTagVal->pData, pTagVal->nData); + n += pTagVal->nData; + } + } + + qsort((*ppTag)->idx, (*ppTag)->nTag, sizeof(STagIdx), tTagIdxCmprFn); + return 0; +} + +void tTagFree(STag *pTag) { + if (pTag) taosMemoryFree(pTag); +} + +void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData) { + STagIdx *pTagIdx = bsearch(&((STagIdx){.cid = cid}), pTag->idx, pTag->nTag, sizeof(STagIdx), tTagIdxCmprFn); + if (pTagIdx == NULL) { + *ppData = NULL; + *nData = 0; + } else { + uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag] + pTagIdx->offset; + if (IS_VAR_DATA_TYPE(type)) { + tGetBinary(p, ppData, nData); + } else { + *ppData = p; + *nData = TYPE_BYTES[type]; + } + } +} + +int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag) { + // return tEncodeBinary(pEncoder, (uint8_t *)pTag, pTag->len); + ASSERT(0); + return 0; +} + +int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag) { + // uint32_t n; + // return tDecodeBinary(pDecoder, (const uint8_t **)ppTag, &n); + ASSERT(0); + return 0; +} + +#if 1 // =================================================================================================================== static void dataColSetNEleNull(SDataCol *pCol, int nEle); int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; @@ -260,8 +622,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += (int)nBitmapBytes; // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered. - // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove - // the additional space + // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus + // remove the additional space #endif if (pCol->spaceSize < spaceNeeded) { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index c2892c070f..62a7179626 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -374,39 +374,135 @@ char getPrecisionUnit(int32_t precision) { } int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { - assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || + assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || + fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); - assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || + assert(toPrecision == TSDB_TIME_PRECISION_MILLI || + toPrecision == TSDB_TIME_PRECISION_MICRO || toPrecision == TSDB_TIME_PRECISION_NANO); - static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}}; - return (int64_t)((double)time * factors[fromPrecision][toPrecision]); + double tempResult = (double)time; + switch(fromPrecision) { + case TSDB_TIME_PRECISION_MILLI: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time; + case TSDB_TIME_PRECISION_MICRO: + tempResult *= 1000; + time *= 1000; + goto end_; + case TSDB_TIME_PRECISION_NANO: + tempResult *= 1000000; + time *= 1000000; + goto end_; + } + } // end from milli + case TSDB_TIME_PRECISION_MICRO: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time / 1000; + case TSDB_TIME_PRECISION_MICRO: + return time; + case TSDB_TIME_PRECISION_NANO: + tempResult *= 1000; + time *= 1000; + goto end_; + } + } //end from micro + case TSDB_TIME_PRECISION_NANO: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time / 1000000; + case TSDB_TIME_PRECISION_MICRO: + return time / 1000; + case TSDB_TIME_PRECISION_NANO: + return time; + } + } //end from nano + default: { + assert(0); + return time; // only to pass windows compilation + } + } //end switch fromPrecision +end_: + if (tempResult >= (double)INT64_MAX) return INT64_MAX; + if (tempResult <= (double)INT64_MIN) return INT64_MIN; // INT64_MIN means NULL + return time; } +// !!!!notice:there are precision problems, double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = 1626006833631000064 +//int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { +// assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || +// fromPrecision == TSDB_TIME_PRECISION_NANO); +// assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || +// toPrecision == TSDB_TIME_PRECISION_NANO); +// static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}}; +// ((double)time * factors[fromPrecision][toPrecision]); +//} + + +// !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = 1626006833631000064 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); - static double factors[3] = {1000000., 1000., 1.}; + int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; + double tmp = time; switch (toUnit) { - case 's': - return time * factors[fromPrecision] / NANOSECOND_PER_SEC; + case 's':{ + tmp /= (NANOSECOND_PER_SEC/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_SEC/factors[fromPrecision]); + break; + } case 'm': - return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE; + tmp /= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); + break; case 'h': - return time * factors[fromPrecision] / NANOSECOND_PER_HOUR; + tmp /= (NANOSECOND_PER_HOUR/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_HOUR/factors[fromPrecision]); + break; case 'd': - return time * factors[fromPrecision] / NANOSECOND_PER_DAY; + tmp /= (NANOSECOND_PER_DAY/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_DAY/factors[fromPrecision]); + break; case 'w': - return time * factors[fromPrecision] / NANOSECOND_PER_WEEK; + tmp /= (NANOSECOND_PER_WEEK/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_WEEK/factors[fromPrecision]); + break; case 'a': - return time * factors[fromPrecision] / NANOSECOND_PER_MSEC; + tmp /= (NANOSECOND_PER_MSEC/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_MSEC/factors[fromPrecision]); + break; case 'u': - return time * factors[fromPrecision] / NANOSECOND_PER_USEC; + // the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double + switch (fromPrecision) { + case TSDB_TIME_PRECISION_MILLI:{ + tmp *= 1000; + time *= 1000; + break; + } + case TSDB_TIME_PRECISION_MICRO:{ + tmp /= 1; + time /= 1; + break; + } + case TSDB_TIME_PRECISION_NANO:{ + tmp /= 1000; + time /= 1000; + break; + } + } + break; case 'b': - return time * factors[fromPrecision]; + tmp *= factors[fromPrecision]; + time *= factors[fromPrecision]; + break; default: { return -1; } } + if (tmp >= (double)INT64_MAX) return INT64_MAX; + if (tmp <= (double)INT64_MIN) return INT64_MIN; + return time; } int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) { diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index a0406e099c..0535b08be7 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -17,6 +17,15 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/libs/common/inc" ) +# dataformatTest.cpp +add_executable(dataformatTest "") +target_sources( + dataformatTest + PRIVATE + "dataformatTest.cpp" +) +target_link_libraries(dataformatTest gtest gtest_main util) + # tmsg test # add_executable(tmsgTest "") # target_sources(tmsgTest diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp new file mode 100644 index 0000000000..3497014c22 --- /dev/null +++ b/source/common/test/dataformatTest.cpp @@ -0,0 +1 @@ +#include "gtest/gtest.h" \ No newline at end of file diff --git a/source/common/test/trowTest.cpp b/source/common/test/trowTest.cpp deleted file mode 100644 index d7f0783d4a..0000000000 --- a/source/common/test/trowTest.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -#include "trow.h" - -TEST(td_row_test, build_row_to_target) { -#if 0 - char dst[1024]; - SRow* pRow = (SRow*)dst; - int ncols = 10; - col_id_t cid; - void* pData; - SRowBuilder rb = trbInit(TD_OR_ROW_BUILDER, NULL, 0, pRow, 1024); - - trbSetRowInfo(&rb, false, 0); - trbSetRowTS(&rb, 1637550210000); - for (int c = 0; c < ncols; c++) { - cid = c; - if (trbWriteCol(&rb, pData, cid) < 0) { - // TODO - } - } -#endif -} \ No newline at end of file diff --git a/source/common/type/type.c b/source/common/type/type.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/type.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBigint.c b/source/common/type/typeBigint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBigint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBinary.c b/source/common/type/typeBinary.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBinary.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBlob.c b/source/common/type/typeBlob.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBlob.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBool.c b/source/common/type/typeBool.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBool.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeDecimal.c b/source/common/type/typeDecimal.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeDecimal.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeDouble.c b/source/common/type/typeDouble.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeDouble.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeFloat.c b/source/common/type/typeFloat.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeFloat.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeInt.c b/source/common/type/typeInt.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeInt.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeJson.c b/source/common/type/typeJson.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeJson.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeLongblob.c b/source/common/type/typeLongblob.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeLongblob.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeNchar.c b/source/common/type/typeNchar.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeNchar.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeNull.c b/source/common/type/typeNull.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeNull.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeSmallint.c b/source/common/type/typeSmallint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeSmallint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeTimestamp.c b/source/common/type/typeTimestamp.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeTimestamp.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeTinyint.c b/source/common/type/typeTinyint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeTinyint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUBigint.c b/source/common/type/typeUBigint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUBigint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUSmallint.c b/source/common/type/typeUSmallint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUSmallint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUTinyint.c b/source/common/type/typeUTinyint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUTinyint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUint.c b/source/common/type/typeUint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeVarchar.c b/source/common/type/typeVarchar.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeVarchar.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 0377aaeb79..757f609731 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -137,13 +137,13 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg queue->tail = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, remain - 8); - memcpy(queue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8)); + memcpy(queue->pBuffer, (char*)pHead + remain - 8, rawHeadLen - (remain - 8)); if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); queue->tail = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen); - if (rawBodyLen > 0) memcpy(queue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + if (rawBodyLen > 0) memcpy(queue->pBuffer, (char*)pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); queue->tail = bodyLen - (remain - 8 - headLen); } else { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 503c0f404a..274876567e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -627,21 +627,26 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); - SArray *tmp = pOldConsumer->rebNewTopics; - pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; - pNewConsumer->rebNewTopics = tmp; - tmp = pOldConsumer->rebRemovedTopics; - pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; - pNewConsumer->rebRemovedTopics = tmp; + if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { + pOldConsumer->status = MQ_CONSUMER_STATUS__READY; + } else { + SArray *tmp = pOldConsumer->rebNewTopics; + pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; + pNewConsumer->rebNewTopics = tmp; - tmp = pOldConsumer->assignedTopics; - pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; - pNewConsumer->assignedTopics = tmp; + tmp = pOldConsumer->rebRemovedTopics; + pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; + pNewConsumer->rebRemovedTopics = tmp; - pOldConsumer->subscribeTime = pNewConsumer->upTime; + tmp = pOldConsumer->assignedTopics; + pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; + pNewConsumer->assignedTopics = tmp; - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pOldConsumer->subscribeTime = pNewConsumer->upTime; + + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + } } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index d09198f66f..fd2f20c242 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -213,8 +213,9 @@ int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valL terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; return -1; } - - memcpy(pVal, pRaw->pData + dataPos, valLen); + if (pVal != NULL) { + memcpy(pVal, pRaw->pData + dataPos, valLen); + } return 0; } @@ -235,4 +236,4 @@ int32_t sdbGetRawTotalSize(SSdbRaw *pRaw) { } return sizeof(SSdbRaw) + pRaw->dataLen; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f7eac3a9d9..ecb022426b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -210,7 +210,7 @@ struct SMetaReader { }; struct SMTbCursor { - TDBC *pDbc; + TBC *pDbc; void *pKey; void *pVal; int kLen; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index d3abc95da9..693f4a0a2b 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -22,8 +22,8 @@ extern "C" { #endif -typedef struct SMetaIdx SMetaIdx; -typedef struct SMetaDB SMetaDB; +typedef struct SMetaIdx SMetaIdx; +typedef struct SMetaDB SMetaDB; // metaDebug ================== // clang-format off @@ -63,16 +63,16 @@ struct SMeta { char* path; SVnode* pVnode; - TENV* pEnv; + TDB* pEnv; TXN txn; - TDB* pTbDb; - TDB* pSkmDb; - TDB* pUidIdx; - TDB* pNameIdx; - TDB* pCtbIdx; - TDB* pTagIdx; - TDB* pTtlIdx; - TDB* pSmaIdx; + TTB* pTbDb; + TTB* pSkmDb; + TTB* pUidIdx; + TTB* pNameIdx; + TTB* pCtbIdx; + TTB* pTagIdx; + TTB* pTtlIdx; + TTB* pSmaIdx; SMetaIdx* pIdx; }; @@ -118,7 +118,7 @@ typedef struct { int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); +int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 2efe600b3d..0601df61e7 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -46,7 +46,7 @@ struct SSmaEnv { TXN txn; void *pPool; // SPoolMem SDiskID did; - TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? char *path; // relative path SSmaStat *pStat; }; @@ -93,16 +93,16 @@ typedef struct SDBFile SDBFile; struct SDBFile { int32_t fid; - TDB *pDB; + TTB *pDB; char *path; }; int32_t tdSmaBeginCommit(SSmaEnv *pEnv); int32_t tdSmaEndCommit(SSmaEnv *pEnv); -int32_t smaOpenDBEnv(TENV **ppEnv, const char *path); -int32_t smaCloseDBEnv(TENV *pEnv); -int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF); +int32_t smaOpenDBEnv(TDB **ppEnv, const char *path); +int32_t smaCloseDBEnv(TDB *pEnv); +int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF); int32_t smaCloseDBF(SDBFile *pDBF); int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9727d9df9f..6a5ad63822 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -96,8 +96,8 @@ STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy) SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); SArray* metaGetSmaTbUids(SMeta* pMeta); -int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); -int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); +int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); +int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); // tsdb int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); @@ -117,6 +117,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqCommit(STQ*); +int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 127eb43f2b..135f44e939 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -50,63 +50,63 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { taosMkDir(pMeta->path); // open env - ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); + ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); if (ret < 0) { metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTbDb - ret = tdbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); + ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); if (ret < 0) { metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSkmDb - ret = tdbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); + ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); if (ret < 0) { metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pUidIdx - ret = tdbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); + ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); if (ret < 0) { metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pNameIdx - ret = tdbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); + ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); if (ret < 0) { metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pCtbIdx - ret = tdbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); + ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTagIdx - ret = tdbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); + ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTtlIdx - ret = tdbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); + ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); if (ret < 0) { metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSmaIdx - ret = tdbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); + ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); if (ret < 0) { metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -125,15 +125,15 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { _err: if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); return -1; @@ -142,15 +142,15 @@ _err: int metaClose(SMeta *pMeta) { if (pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index f02b6402c4..2bcb68c82a 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -35,7 +35,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u STbDbKey tbDbKey = {.version = version, .uid = uid}; // query table.db - if (tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _err; } @@ -58,7 +58,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { int64_t version; // query uid.idx - if (tdbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -72,7 +72,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { tb_uid_t uid; // query name.idx - if (tdbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -100,9 +100,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { metaReaderInit(&pTbCur->mr, pMeta, 0); - tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); + tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); - tdbDbcMoveToFirst(pTbCur->pDbc); + tdbTbcMoveToFirst(pTbCur->pDbc); return pTbCur; } @@ -113,7 +113,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { tdbFree(pTbCur->pVal); metaReaderClear(&pTbCur->mr); if (pTbCur->pDbc) { - tdbDbcClose(pTbCur->pDbc); + tdbTbcClose(pTbCur->pDbc); } taosMemoryFree(pTbCur); } @@ -125,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { STbCfg tbCfg; for (;;) { - ret = tdbDbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -159,7 +159,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo pKey = &skmDbKey; kLen = sizeof(skmDbKey); metaRLock(pMeta); - ret = tdbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); + ret = tdbTbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); metaULock(pMeta); if (ret < 0) { return NULL; @@ -184,7 +184,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo struct SMCtbCursor { SMeta *pMeta; - TDBC *pCur; + TBC *pCur; tb_uid_t suid; void *pKey; void *pVal; @@ -207,7 +207,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { pCtbCur->suid = uid; metaRLock(pMeta); - ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); + ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); if (ret < 0) { metaULock(pMeta); taosMemoryFree(pCtbCur); @@ -217,9 +217,9 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { // move to the suid ctbIdxKey.suid = uid; ctbIdxKey.uid = INT64_MIN; - tdbDbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); + tdbTbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); if (c > 0) { - tdbDbcMoveToNext(pCtbCur->pCur); + tdbTbcMoveToNext(pCtbCur->pCur); } return pCtbCur; @@ -229,7 +229,7 @@ void metaCloseCtbCursor(SMCtbCursor *pCtbCur) { if (pCtbCur) { if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta); if (pCtbCur->pCur) { - tdbDbcClose(pCtbCur->pCur); + tdbTbcClose(pCtbCur->pCur); tdbFree(pCtbCur->pKey); tdbFree(pCtbCur->pVal); @@ -243,7 +243,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { int ret; SCtbIdxKey *pCtbIdxKey; - ret = tdbDbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + ret = tdbTbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); if (ret < 0) { return 0; } @@ -299,7 +299,7 @@ int metaGetTbNum(SMeta *pMeta) { typedef struct { SMeta *pMeta; - TDBC *pCur; + TBC *pCur; tb_uid_t uid; void *pKey; void *pVal; @@ -323,7 +323,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { pSmaCur->uid = uid; metaRLock(pMeta); - ret = tdbDbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); + ret = tdbTbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); if (ret < 0) { metaULock(pMeta); taosMemoryFree(pSmaCur); @@ -333,9 +333,9 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // move to the suid smaIdxKey.uid = uid; smaIdxKey.smaUid = INT64_MIN; - tdbDbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); + tdbTbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); if (c > 0) { - tdbDbcMoveToNext(pSmaCur->pCur); + tdbTbcMoveToNext(pSmaCur->pCur); } return pSmaCur; @@ -345,7 +345,7 @@ void metaCloseSmaCursor(SMSmaCursor *pSmaCur) { if (pSmaCur) { if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta); if (pSmaCur->pCur) { - tdbDbcClose(pSmaCur->pCur); + tdbTbcClose(pSmaCur->pCur); tdbFree(pSmaCur->pKey); tdbFree(pSmaCur->pVal); @@ -359,7 +359,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) { int ret; SSmaIdxKey *pSmaIdxKey; - ret = tdbDbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); + ret = tdbTbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); if (ret < 0) { return 0; } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 1fd81bc2cb..75595d83a6 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -130,17 +130,17 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; - return tdbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); } static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 273f7885e6..3d1854927c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -71,9 +71,9 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - TDBC *pNameIdxc = NULL; - TDBC *pUidIdxc = NULL; - TDBC *pCtbIdxc = NULL; + TBC *pNameIdxc = NULL; + TBC *pUidIdxc = NULL; + TBC *pCtbIdxc = NULL; SCtbIdxKey *pCtbIdxKey; const void *pKey = NULL; int nKey; @@ -82,43 +82,43 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { int c, ret; // prepare uid idx cursor - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c != 0) { terrno = TSDB_CODE_VND_TB_NOT_EXIST; - tdbDbcClose(pUidIdxc); + tdbTbcClose(pUidIdxc); goto _err; } // prepare name idx cursor - tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); if (ret < 0 || c != 0) { ASSERT(0); } - tdbDbcDelete(pUidIdxc); - tdbDbcDelete(pNameIdxc); - tdbDbcClose(pUidIdxc); - tdbDbcClose(pNameIdxc); + tdbTbcDelete(pUidIdxc); + tdbTbcDelete(pNameIdxc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pNameIdxc); // loop to drop each child table - tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { - tdbDbcClose(pCtbIdxc); + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || (c < 0 && tdbTbcMoveToNext(pCtbIdxc) < 0)) { + tdbTbcClose(pCtbIdxc); goto _exit; } for (;;) { - tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); + tdbTbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); pCtbIdxKey = (SCtbIdxKey *)pKey; if (pCtbIdxKey->suid > pReq->suid) break; // drop the child table (TODO) - if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; + if (tdbTbcMoveToNext(pCtbIdxc) < 0) break; } _exit: @@ -134,8 +134,8 @@ _err: int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TDBC *pUidIdxc = NULL; - TDBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -143,14 +143,14 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t ret; int32_t c; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c) { ASSERT(0); return -1; } - ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -158,11 +158,11 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { oversion = *(int64_t *)pData; - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(ret == 0 && c == 0); - ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ASSERT(ret == 0); tDecoderInit(&dc, pData, nData); @@ -191,12 +191,12 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { metaSaveToTbDb(pMeta, &nStbEntry); // update uid index - tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); + tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); metaULock(pMeta); tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; } @@ -256,9 +256,9 @@ _err: } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { - TDBC *pTbDbc = NULL; - TDBC *pUidIdxc = NULL; - TDBC *pNameIdxc = NULL; + TBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pNameIdxc = NULL; const void *pData; int nData; tb_uid_t uid; @@ -271,15 +271,15 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int c = 0, ret; // search & delete the name idx - tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); - if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { - tdbDbcClose(pNameIdxc); + tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || !tdbTbcIsValid(pNameIdxc) || c) { + tdbTbcClose(pNameIdxc); terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; } - ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pNameIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -287,36 +287,36 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { uid = *(tb_uid_t *)pData; - tdbDbcDelete(pNameIdxc); - tdbDbcClose(pNameIdxc); + tdbTbcDelete(pNameIdxc); + tdbTbcClose(pNameIdxc); // search & delete uid idx - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; } tver = *(int64_t *)pData; - tdbDbcDelete(pUidIdxc); - tdbDbcClose(pUidIdxc); + tdbTbcDelete(pUidIdxc); + tdbTbcClose(pUidIdxc); // search and get meta entry - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbTbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -345,21 +345,21 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { taosMemoryFree(pDataCopy); tDecoderClear(&coder); - tdbDbcClose(pTbDbc); + tdbTbcClose(pTbDbc); if (type == TSDB_CHILD_TABLE) { // remove the pCtbIdx - TDBC *pCtbIdxc = NULL; - tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); + ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - tdbDbcDelete(pCtbIdxc); - tdbDbcClose(pCtbIdxc); + tdbTbcDelete(pCtbIdxc); + tdbTbcClose(pCtbIdxc); // remove tags from pTagIdx (todo) } else if (type == TSDB_NORMAL_TABLE) { @@ -389,7 +389,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl int c; // search name index - ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -400,22 +400,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl pVal = NULL; // search uid index - TDBC *pUidIdxc = NULL; + TBC *pUidIdxc = NULL; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); - tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = *(int64_t *)pData; // search table.db - TDBC *pTbDbc = NULL; + TBC *pTbDbc = NULL; - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); - tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry SDecoder dc = {0}; @@ -505,21 +505,21 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // save to table db metaSaveToTbDb(pMeta, &entry); - tdbDbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); + tdbTbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); metaSaveToSkmDb(pMeta, &entry); metaULock(pMeta); tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; _err: tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return -1; } @@ -536,7 +536,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA int nData = 0; // search name index - ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -547,24 +547,24 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA pVal = NULL; // search uid index - TDBC *pUidIdxc = NULL; + TBC *pUidIdxc = NULL; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); - tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = *(int64_t *)pData; // search table.db - TDBC *pTbDbc = NULL; + TBC *pTbDbc = NULL; SDecoder dc = {0}; /* get ctbEntry */ - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); - tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ctbEntry.pBuf = taosMemoryMalloc(nData); memcpy(ctbEntry.pBuf, pData, nData); @@ -573,9 +573,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc); /* get stbEntry*/ - tdbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); - tdbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), - (void **)&stbEntry.pBuf, &nVal); + tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); + tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), + (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); tDecoderInit(&dc, stbEntry.pBuf, nVal); metaDecodeEntry(&dc, &stbEntry); @@ -632,19 +632,19 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaSaveToTbDb(pMeta, &ctbEntry); // save to uid.idx - tdbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); + tdbTbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; _err: if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return -1; } @@ -708,7 +708,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -721,11 +721,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -748,12 +748,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, @@ -801,10 +801,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SDecoder dc = {0}; // get super table - tdbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = *(int64_t *)pData; - tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); tDecoderInit(&dc, pData, nData); metaDecodeEntry(&dc, &stbEntry); @@ -817,7 +817,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } - tdbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); tDecoderClear(&dc); @@ -859,7 +859,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderInit(&coder, pVal, vLen); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c index 6e57656802..cac986d053 100644 --- a/source/dnode/vnode/src/sma/smaTDBImpl.c +++ b/source/dnode/vnode/src/sma/smaTDBImpl.c @@ -17,12 +17,12 @@ #include "sma.h" -int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { +int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) { int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { smaError("failed to create tsdb db env, ret = %d", ret); @@ -32,7 +32,7 @@ int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { return 0; } -int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } +int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { const SSmaKey *pKey1 = (const SSmaKey *)arg1; @@ -54,21 +54,21 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int return 0; } -static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { +static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { tdb_cmpr_fn_t compFunc; // Create a database compFunc = tdSmaKeyCmpr; - if (tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { + if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { return -1; } return 0; } -static int32_t smaCloseDBDb(TDB *pDB) { return tdbClose(pDB); } +static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } -int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { +int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile if (!pEnv || !pDBF) { terrno = TSDB_CODE_INVALID_PTR; @@ -99,7 +99,7 @@ int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, in int32_t ret; printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); - ret = tdbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { smaError("failed to upsert tsma data into db, ret = %d", ret); return -1; @@ -112,7 +112,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_ void *pVal = NULL; int ret; - ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { smaError("failed to get tsma data from db, ret = %d", ret); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 1d54d75ad5..f771e73c8a 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -55,7 +55,6 @@ typedef enum { SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma } ESmaStorageLevel; - // static func static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); @@ -69,18 +68,15 @@ static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t f static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); + TXN *txn); // expired window - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version); +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); // read data - // implementation /** @@ -157,7 +153,6 @@ static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { return false; } - /** * @brief Approximate value for week/month/year. * @@ -239,9 +234,8 @@ static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, return interval; } - static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { + int8_t intervalUnit) { pSmaH->pSma = pSma; pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); pSmaH->pDataBlocks = pDataBlocks; @@ -493,11 +487,12 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { smaCloseDBF(&tSmaH.dFile); } tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64, + smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 + " queryKey:%" PRIi64, SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tdDestroyTSmaWriteH(&tSmaH); tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; @@ -523,9 +518,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { tdResetExpiredWindow(pSma, pStat, indexUid, skey); } else { smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - SMA_VID(pSma), skey, tlen, indexUid); + SMA_VID(pSma), skey, tlen, indexUid); } - } } tdSmaEndCommit(pEnv); // TODO: not commit for every insert @@ -557,7 +551,7 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; - // TODO: insert tsma data blocks into B+Tree(TDB) + // TODO: insert tsma data blocks into B+Tree(TTB) if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); @@ -600,12 +594,12 @@ static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUi if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { // error handling tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), - skey, indexUid); + smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, + indexUid); return TSDB_CODE_FAILED; } smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), - skey, indexUid); + skey, indexUid); // TODO: use a standalone interface to received state upate notification from stream computing module. /** * @brief state @@ -666,8 +660,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); taosSsleep(1); if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, - refVal); + smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); break; }; } @@ -730,17 +723,17 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query tdUnRefSmaStat(pSma, pStat); terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, - tstrerror(terrno), smaStat); + tstrerror(terrno), smaStat); return TSDB_CODE_FAILED; } if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { // TODO: mark this window as expired. - smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), - querySKey, indexUid); + smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); } else { smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); + indexUid); } STSma *pTSma = pItem->pTSma; @@ -755,7 +748,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query tdInitTSmaFile(&tReadH, indexUid, querySKey); smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, - SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); + SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); return TSDB_CODE_FAILED; @@ -766,18 +759,18 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query int64_t queryGroupId = 0; tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), - tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); + smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, + *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); void *result = NULL; int32_t valueSize = 0; if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); + SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); smaCloseDBF(&tReadH.dFile); return TSDB_CODE_FAILED; } - #endif +#endif #ifdef _TEST_SMA_PRINT_DEBUG_LOG_ for (uint32_t v = 0; v < valueSize; v += 8) { @@ -878,7 +871,7 @@ static SSmaStatItem *tdNewSmaStatItem(int8_t state) { } static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { + int64_t version) { SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); if (!pItem) { // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later @@ -923,17 +916,15 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde taosMemoryFreeClear(pItem->pTSma); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, - winSKey); + winSKey); return TSDB_CODE_FAILED; } smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, - winSKey); + winSKey); return TSDB_CODE_SUCCESS; } - - /** * @brief Update expired window according to msg from stream computing module. * @@ -1035,7 +1026,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) } } else { smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - SMA_VID(pSma), pTSma->indexUid, winSKey); + SMA_VID(pSma), pTSma->indexUid, winSKey); } } } @@ -1044,4 +1035,3 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) return TSDB_CODE_SUCCESS; } - diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6cc986d54b..80aa350e4d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -101,6 +101,21 @@ static void tdSRowDemo() { taosMemoryFree(pTSChema); } +int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) { + void* pIter = NULL; + STqExec* pExec = NULL; + while (1) { + pIter = taosHashIterate(pTq->execs, pIter); + if (pIter == NULL) break; + pExec = (STqExec*)pIter; + for (int32_t i = 0; i < 5; i++) { + int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true); + ASSERT(code == 0); + } + } + return 0; +} + int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType != TDMT_VND_SUBMIT) return 0; void* pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index dc782cc022..18cf18dbad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -53,7 +53,7 @@ struct SSmaEnv { TXN txn; SPoolMem *pPool; SDiskID did; - TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? char *path; // relative path SSmaStat *pStat; }; @@ -876,7 +876,7 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; - // TODO: insert tsma data blocks into B+Tree(TDB) + // TODO: insert tsma data blocks into B+Tree(TTB) if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index bed61a186f..a553f32bee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -17,12 +17,12 @@ #include "tsdb.h" -int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { +int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) { int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { tsdbError("Failed to create tsdb db env, ret = %d", ret); @@ -32,7 +32,7 @@ int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { return 0; } -int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } +int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { const SSmaKey *pKey1 = (const SSmaKey *)arg1; @@ -54,20 +54,20 @@ static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, i return 0; } -static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { +static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { int ret; tdb_cmpr_fn_t compFunc; // Create a database compFunc = tsdbSmaKeyCmpr; - ret = tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); + ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); return 0; } -static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbClose(pDB); } +static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } -int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { +int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile if (!pEnv || !pDBF) { terrno = TSDB_CODE_INVALID_PTR; @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; @@ -110,7 +110,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32 void *pVal = NULL; int ret; - ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { tsdbError("Failed to get sma data from db, ret = %d", ret); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464331319e..5c8cd362fd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -333,6 +333,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, SVCreateTbRsp cRsp = {0}; char tbName[TSDB_TABLE_FNAME_LEN]; STbUidStore *pStore = NULL; + SArray *tbUids = NULL; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -348,7 +349,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); - if (rsp.pArray == NULL) { + tbUids = taosArrayInit(req.nReqs, sizeof(int64_t)); + if (rsp.pArray == NULL || tbUids == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -376,6 +378,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } else { cRsp.code = TSDB_CODE_SUCCESS; tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); + taosArrayPush(tbUids, &pCreateReq->uid); } taosArrayPush(rsp.pArray, &cRsp); @@ -383,6 +386,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tDecoderClear(&decoder); + tqUpdateTbUidList(pVnode->pTq, tbUids); tdUpdateTbUidList(pVnode->pSma, pStore); tdUidStoreFree(pStore); @@ -402,6 +406,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, _exit: taosArrayDestroy(rsp.pArray); + taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); return rcode; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1bf21ad7d1..857c708852 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -131,6 +131,7 @@ typedef struct SCtgCacheStat { uint64_t dbNum; uint64_t tblNum; uint64_t stblNum; + uint64_t userNum; uint64_t vgHitNum; uint64_t vgMissNum; uint64_t tblHitNum; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c96ad140a1..d3045f68a1 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -122,6 +122,11 @@ void ctgFreeDbCache(SCtgDBCache *dbCache) { ctgFreeTableMetaCache(&dbCache->tbCache); } +void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { + taosHashCleanup(userCache->createdDbs); + taosHashCleanup(userCache->readDbs); + taosHashCleanup(userCache->writeDbs); +} void ctgFreeHandle(SCatalog* pCtg) { ctgFreeMetaRent(&pCtg->dbRent); @@ -145,7 +150,24 @@ void ctgFreeHandle(SCatalog* pCtg) { CTG_CACHE_STAT_SUB(dbNum, dbNum); } - + + if (pCtg->userCache) { + int32_t userNum = taosHashGetSize(pCtg->userCache); + + void *pIter = taosHashIterate(pCtg->userCache, NULL); + while (pIter) { + SCtgUserAuth *userCache = pIter; + + ctgFreeSCtgUserAuth(userCache); + + pIter = taosHashIterate(pCtg->userCache, pIter); + } + + taosHashCleanup(pCtg->userCache); + + CTG_CACHE_STAT_SUB(userNum, userNum); + } + taosMemoryFree(pCtg); } @@ -2031,10 +2053,13 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); + + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); if (!inCache) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); + continue; } @@ -2306,6 +2331,8 @@ int32_t ctgActUpdateUser(SCtgMetaAction *action) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + taosMemoryFreeClear(msg); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 320450eb6e..66073b70eb 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -126,7 +126,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return pTaskInfo; } -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; // traverse to the stream scanner node to add this table id @@ -141,12 +141,12 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA SMetaReader mr = {0}; metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); - for(int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); int32_t code = metaGetTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%"PRIu64" code:%s", *id, tstrerror(terrno)); + qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); continue; } @@ -160,7 +160,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA metaReaderClear(&mr); - qDebug(" %d qualified child tables added into stream scanner", (int32_t) taosArrayGetSize(qa)); + qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eaa1197a84..d042e463f0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -668,6 +668,7 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti } pDataBlock->info.rows = size; pDataBlock->info.type = STREAM_REPROCESS; + blockDataUpdateTsWindow(pDataBlock); taosArrayClear(pInfo->tsArray); return pDataBlock; } @@ -768,6 +769,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } rows = pBlockInfo->rows; doFilter(pInfo->pCondition, pInfo->pRes, NULL); + blockDataUpdateTsWindow(pInfo->pRes); break; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index de96526a25..f28c5f1eaa 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2497,7 +2497,7 @@ bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo pInfo->min = MAX_TS_KEY; pInfo->max = 0; - if (pCtx->numOfParams == 3) { + if (pCtx->numOfParams == 2) { pInfo->timeUnit = pCtx->param[1].param.i; } else { pInfo->timeUnit = 1; @@ -2521,7 +2521,6 @@ int32_t elapsedFunction(SqlFunctionCtx *pCtx) { } if (pInput->colDataAggIsSet) { - if (pInfo->min == MAX_TS_KEY) { pInfo->min = GET_INT64_VAL(&pAgg->min); pInfo->max = GET_INT64_VAL(&pAgg->max); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 74e2861037..802cdf7aad 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -316,52 +316,63 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { case TSDB_DATA_TYPE_TIMESTAMP: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int64_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint8_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_TINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int8_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_SMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int16_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_USMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint16_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_INT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int32_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint32_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%" PRIu64, *(uint64_t*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UBIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint64_t*)src, *dst, 1); + tlen = strlen(*dst); case TSDB_DATA_TYPE_FLOAT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(float*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_DOUBLE: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(double*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_NCHAR: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - *dst = *dst - tlen; + *dst = (char*) * dst - tlen; break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 9533d3429e..163bb53163 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -473,17 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR int32_t sz = 0; char* ch = (char*)fstSliceData(s, &sz); - // char* tmp = taosMemoryCalloc(1, sz + 1); - // memcpy(tmp, ch, sz); + char* tmp = taosMemoryCalloc(1, sz + 1); + memcpy(tmp, ch, sz); - if (0 != strncmp(ch, p, skip)) { + if (0 != strncmp(tmp, p, skip)) { swsResultDestroy(rt); - // taosMemoryFree(tmp); + taosMemoryFree(tmp); break; } - TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); - + TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); if (MATCH == cond) { tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); } else if (CONTINUE == cond) { @@ -491,7 +490,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR swsResultDestroy(rt); break; } - // taosMemoryFree(tmp); + taosMemoryFree(tmp); swsResultDestroy(rt); } streamWithStateDestroy(st); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 135ae61e83..ffb102ef2e 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -66,6 +66,17 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp indexMultiTermDestroy(terms); } + +static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, + int8_t operType = DEL_VALUE) { + SIndexTerm* term = + indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + tIndexJsonPut(index, terms, (int64_t)tableId); + + indexMultiTermDestroy(terms); +} static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType, SArray** result) { std::string colName(colNam); @@ -578,3 +589,48 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) { EXPECT_EQ(1000, taosArrayGetSize(res)); } } +TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) { + { + double val = 10.0; + for (int i = 0; i < 1000; i++) { + WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); + } + } + { + double val = 2.0; + for (int i = 0; i < 1000; i++) { + WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); + } + } + { + SArray* res = NULL; + std::string colName("test1"); + double val = 1.9; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(2000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 2.1; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 2.1; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 10.0; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_EQUAL, &res); + EXPECT_EQ(2000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 10.0; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_THAN, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } +} diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 01833e1776..29951ccdee 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1045,6 +1045,7 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); taosHashCleanup(pCxt->pSubTableHashObj); + taosHashCleanup(pCxt->pTableNameHashObj); destroyBlockHashmap(pCxt->pTableBlockHashObj); destroyBlockArrayList(pCxt->pVgDataBlocks); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index deb899309e..f82c792c96 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -235,12 +235,12 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); - if (!pDataBlock->cloned) { +// if (!pDataBlock->cloned) { // free the refcount for metermeta taosMemoryFreeClear(pDataBlock->pTableMeta); destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - } +// } taosMemoryFreeClear(pDataBlock); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 4fcdb35e0d..bdf674ab93 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1438,24 +1438,24 @@ int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { } int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { + SSchedulerHbRsp rsp = {0}; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; + if (code) { qError("hb rsp error:%s", tstrerror(code)); - SCH_ERR_RET(code); + SCH_ERR_JRET(code); } - SSchedulerHbRsp rsp = {0}; if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) { qError("invalid hb rsp msg, size:%d", pMsg->len); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SSchTrans trans = {0}; trans.transInst = pParam->transport; trans.transHandle = pMsg->handle; - SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans)); + SCH_ERR_JRET(schUpdateHbConnection(&rsp.epId, &trans)); int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, @@ -1483,6 +1483,7 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { _return: tFreeSSchedulerHbRsp(&rsp); + taosMemoryFree(param); SCH_RET(code); } diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 01490030f2..405fb1c5a0 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -7,7 +7,7 @@ target_sources(tdb "src/db/tdbUtil.c" "src/db/tdbBtree.c" "src/db/tdbDb.c" - "src/db/tdbEnv.c" + "src/db/tdbTable.c" "src/db/tdbTxn.c" "src/db/tdbPage.c" "src/db/tdbOs.c" diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 8ba66f1d26..521b61e3c8 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -25,40 +25,40 @@ extern "C" { typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); // exposed types -typedef struct STEnv TENV; -typedef struct STDB TDB; -typedef struct STDBC TDBC; -typedef struct STxn TXN; - -// TENV -int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv); -int tdbEnvClose(TENV *pEnv); -int tdbBegin(TENV *pEnv, TXN *pTxn); -int tdbCommit(TENV *pEnv, TXN *pTxn); +typedef struct STDB TDB; +typedef struct STTB TTB; +typedef struct STBC TBC; +typedef struct STxn TXN; // TDB -int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); +int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb); int tdbClose(TDB *pDb); -int tdbDrop(TDB *pDb); -int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); -int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int tdbBegin(TDB *pDb, TXN *pTxn); +int tdbCommit(TDB *pDb, TXN *pTxn); -// TDBC -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); -int tdbDbcClose(TDBC *pDbc); -int tdbDbcIsValid(TDBC *pDbc); -int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); -int tdbDbcMoveToFirst(TDBC *pDbc); -int tdbDbcMoveToLast(TDBC *pDbc); -int tdbDbcMoveToNext(TDBC *pDbc); -int tdbDbcMoveToPrev(TDBC *pDbc); -int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); -int tdbDbcDelete(TDBC *pDbc); -int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); +// TTB +int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int tdbTbClose(TTB *pTb); +int tdbTbDrop(TTB *pTb); +int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); +int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); + +// TBC +int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); +int tdbTbcClose(TBC *pTbc); +int tdbTbcIsValid(TBC *pTbc); +int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); +int tdbTbcMoveToFirst(TBC *pTbc); +int tdbTbcMoveToLast(TBC *pTbc); +int tdbTbcMoveToNext(TBC *pTbc); +int tdbTbcMoveToPrev(TBC *pTbc); +int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); +int tdbTbcDelete(TBC *pTbc); +int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index fdad797333..912c1ecaaf 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,134 +15,164 @@ #include "tdbInt.h" -struct STDB { - TENV *pEnv; - SBTree *pBt; -}; - -struct STDBC { - SBTC btc; -}; - -int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) { - TDB *pDb; - SPager *pPager; - int ret; - char fFullName[TDB_FILENAME_LEN]; - SPage *pPage; - SPgno pgno; +int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { + TDB *pDb; + int dsize; + int zsize; + int tsize; + u8 *pPtr; + int ret; *ppDb = NULL; - pDb = (TDB *)tdbOsCalloc(1, sizeof(*pDb)); - if (pDb == NULL) { + dsize = strlen(rootDir); + zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; + + pPtr = (uint8_t *)tdbOsCalloc(1, zsize); + if (pPtr == NULL) { return -1; } - // pDb->pEnv - pDb->pEnv = pEnv; + pDb = (TDB *)pPtr; + pPtr += sizeof(*pDb); + // pDb->rootDir + pDb->rootDir = pPtr; + memcpy(pDb->rootDir, rootDir, dsize); + pDb->rootDir[dsize] = '\0'; + pPtr = pPtr + dsize + 1; + // pDb->jfname + pDb->jfname = pPtr; + memcpy(pDb->jfname, rootDir, dsize); + pDb->jfname[dsize] = '/'; + memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; - pPager = tdbEnvGetPager(pEnv, fname); - if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); - ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); - if (ret < 0) { - return -1; - } + pDb->jfd = -1; - tdbEnvAddPager(pEnv, pPager); - } - - ASSERT(pPager != NULL); - - // pDb->pBt - ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt)); + ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); if (ret < 0) { return -1; } + pDb->nPgrHash = 8; + tsize = sizeof(SPager *) * pDb->nPgrHash; + pDb->pgrHash = tdbOsMalloc(tsize); + if (pDb->pgrHash == NULL) { + return -1; + } + memset(pDb->pgrHash, 0, tsize); + + mkdir(rootDir, 0755); + *ppDb = pDb; return 0; } int tdbClose(TDB *pDb) { + SPager *pPager; + if (pDb) { - tdbBtreeClose(pDb->pBt); + for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) { + pDb->pgrList = pPager->pNext; + tdbPagerClose(pPager); + } + + tdbPCacheClose(pDb->pCache); + tdbOsFree(pDb->pgrHash); tdbOsFree(pDb); } + return 0; } -int tdbDrop(TDB *pDb) { - // TODO - return 0; -} +int tdbBegin(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; -int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { - return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); -} - -int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } - -int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { - return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); -} - -int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { - return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); -} - -int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { - return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); -} - -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn) { - int ret; - TDBC *pDbc = NULL; - - *ppDbc = NULL; - pDbc = (TDBC *)tdbOsMalloc(sizeof(*pDbc)); - if (pDbc == NULL) { - return -1; - } - - tdbBtcOpen(&pDbc->btc, pDb->pBt, pTxn); - - *ppDbc = pDbc; - return 0; -} - -int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pDbc->btc, pKey, kLen, c); } - -int tdbDbcMoveToFirst(TDBC *pDbc) { return tdbBtcMoveToFirst(&pDbc->btc); } - -int tdbDbcMoveToLast(TDBC *pDbc) { return tdbBtcMoveToLast(&pDbc->btc); } - -int tdbDbcMoveToNext(TDBC *pDbc) { return tdbBtcMoveToNext(&pDbc->btc); } - -int tdbDbcMoveToPrev(TDBC *pDbc) { return tdbBtcMoveToPrev(&pDbc->btc); } - -int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { - return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); -} - -int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } - -int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); -} - -int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { - return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert); -} - -int tdbDbcClose(TDBC *pDbc) { - if (pDbc) { - tdbBtcClose(&pDbc->btc); - tdbOsFree(pDbc); + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerBegin(pPager, pTxn); + if (ret < 0) { + ASSERT(0); + return -1; + } } return 0; } -int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } \ No newline at end of file +int tdbCommit(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; + + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerCommit(pPager, pTxn); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + + return 0; +} + +SPager *tdbEnvGetPager(TDB *pDb, const char *fname) { + u32 hash; + SPager **ppPager; + + hash = tdbCstringHash(fname); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) { + } + + return *ppPager; +} + +void tdbEnvAddPager(TDB *pDb, SPager *pPager) { + u32 hash; + SPager **ppPager; + + // rehash if neccessary + if (pDb->nPager + 1 > pDb->nPgrHash) { + // TODO + } + + // add to list + pPager->pNext = pDb->pgrList; + pDb->pgrList = pPager; + + // add to hash + hash = tdbCstringHash(pPager->dbFileName); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + pPager->pHashNext = *ppPager; + *ppPager = pPager; + + // increase the counter + pDb->nPager++; +} + +void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { + u32 hash; + SPager **ppPager; + + // remove from the list + for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { + } + ASSERT(*ppPager == pPager); + *ppPager = pPager->pNext; + + // remove from hash + hash = tdbCstringHash(pPager->dbFileName); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { + } + ASSERT(*ppPager == pPager); + *ppPager = pPager->pNext; + + // decrease the counter + pDb->nPager--; + + // rehash if necessary + if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) { + // TODO + } +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c deleted file mode 100644 index c0c1343a4f..0000000000 --- a/source/libs/tdb/src/db/tdbEnv.c +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) { - TENV *pEnv; - int dsize; - int zsize; - int tsize; - u8 *pPtr; - int ret; - - *ppEnv = NULL; - - dsize = strlen(rootDir); - zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; - - pPtr = (uint8_t *)tdbOsCalloc(1, zsize); - if (pPtr == NULL) { - return -1; - } - - pEnv = (TENV *)pPtr; - pPtr += sizeof(*pEnv); - // pEnv->rootDir - pEnv->rootDir = pPtr; - memcpy(pEnv->rootDir, rootDir, dsize); - pEnv->rootDir[dsize] = '\0'; - pPtr = pPtr + dsize + 1; - // pEnv->jfname - pEnv->jfname = pPtr; - memcpy(pEnv->jfname, rootDir, dsize); - pEnv->jfname[dsize] = '/'; - memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); - pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; - - pEnv->jfd = -1; - - ret = tdbPCacheOpen(szPage, pages, &(pEnv->pCache)); - if (ret < 0) { - return -1; - } - - pEnv->nPgrHash = 8; - tsize = sizeof(SPager *) * pEnv->nPgrHash; - pEnv->pgrHash = tdbOsMalloc(tsize); - if (pEnv->pgrHash == NULL) { - return -1; - } - memset(pEnv->pgrHash, 0, tsize); - - mkdir(rootDir, 0755); - - *ppEnv = pEnv; - return 0; -} - -int tdbEnvClose(TENV *pEnv) { - SPager *pPager; - - if (pEnv) { - for (pPager = pEnv->pgrList; pPager; pPager = pEnv->pgrList) { - pEnv->pgrList = pPager->pNext; - tdbPagerClose(pPager); - } - - tdbPCacheClose(pEnv->pCache); - tdbOsFree(pEnv->pgrHash); - tdbOsFree(pEnv); - } - - return 0; -} - -int tdbBegin(TENV *pEnv, TXN *pTxn) { - SPager *pPager; - int ret; - - for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { - ret = tdbPagerBegin(pPager, pTxn); - if (ret < 0) { - ASSERT(0); - return -1; - } - } - - return 0; -} - -int tdbCommit(TENV *pEnv, TXN *pTxn) { - SPager *pPager; - int ret; - - for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { - ret = tdbPagerCommit(pPager, pTxn); - if (ret < 0) { - ASSERT(0); - return -1; - } - } - - return 0; -} - -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) { - u32 hash; - SPager **ppPager; - - hash = tdbCstringHash(fname); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) { - } - - return *ppPager; -} - -void tdbEnvAddPager(TENV *pEnv, SPager *pPager) { - u32 hash; - SPager **ppPager; - - // rehash if neccessary - if (pEnv->nPager + 1 > pEnv->nPgrHash) { - // TODO - } - - // add to list - pPager->pNext = pEnv->pgrList; - pEnv->pgrList = pPager; - - // add to hash - hash = tdbCstringHash(pPager->dbFileName); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - pPager->pHashNext = *ppPager; - *ppPager = pPager; - - // increase the counter - pEnv->nPager++; -} - -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) { - u32 hash; - SPager **ppPager; - - // remove from the list - for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { - } - ASSERT(*ppPager == pPager); - *ppPager = pPager->pNext; - - // remove from hash - hash = tdbCstringHash(pPager->dbFileName); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { - } - ASSERT(*ppPager == pPager); - *ppPager = pPager->pNext; - - // decrease the counter - pEnv->nPager--; - - // rehash if necessary - if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) { - // TODO - } -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c new file mode 100644 index 0000000000..ce7da0b471 --- /dev/null +++ b/source/libs/tdb/src/db/tdbTable.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct STTB { + TDB *pEnv; + SBTree *pBt; +}; + +struct STBC { + SBTC btc; +}; + +int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { + TTB *pTb; + SPager *pPager; + int ret; + char fFullName[TDB_FILENAME_LEN]; + SPage *pPage; + SPgno pgno; + + *ppTb = NULL; + + pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb)); + if (pTb == NULL) { + return -1; + } + + // pTb->pEnv + pTb->pEnv = pEnv; + + pPager = tdbEnvGetPager(pEnv, fname); + if (pPager == NULL) { + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); + if (ret < 0) { + return -1; + } + + tdbEnvAddPager(pEnv, pPager); + } + + ASSERT(pPager != NULL); + + // pTb->pBt + ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pTb->pBt)); + if (ret < 0) { + return -1; + } + + *ppTb = pTb; + return 0; +} + +int tdbTbClose(TTB *pTb) { + if (pTb) { + tdbBtreeClose(pTb->pBt); + tdbOsFree(pTb); + } + return 0; +} + +int tdbTbDrop(TTB *pTb) { + // TODO + return 0; +} + +int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { + return tdbBtreeInsert(pTb->pBt, pKey, keyLen, pVal, valLen, pTxn); +} + +int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pTb->pBt, pKey, kLen, pTxn); } + +int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + return tdbBtreeUpsert(pTb->pBt, pKey, kLen, pVal, vLen, pTxn); +} + +int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen) { + return tdbBtreeGet(pTb->pBt, pKey, kLen, ppVal, vLen); +} + +int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { + return tdbBtreePGet(pTb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); +} + +int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) { + int ret; + TBC *pTbc = NULL; + + *ppTbc = NULL; + pTbc = (TBC *)tdbOsMalloc(sizeof(*pTbc)); + if (pTbc == NULL) { + return -1; + } + + tdbBtcOpen(&pTbc->btc, pTb->pBt, pTxn); + + *ppTbc = pTbc; + return 0; +} + +int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pTbc->btc, pKey, kLen, c); } + +int tdbTbcMoveToFirst(TBC *pTbc) { return tdbBtcMoveToFirst(&pTbc->btc); } + +int tdbTbcMoveToLast(TBC *pTbc) { return tdbBtcMoveToLast(&pTbc->btc); } + +int tdbTbcMoveToNext(TBC *pTbc) { return tdbBtcMoveToNext(&pTbc->btc); } + +int tdbTbcMoveToPrev(TBC *pTbc) { return tdbBtcMoveToPrev(&pTbc->btc); } + +int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { + return tdbBtcGet(&pTbc->btc, ppKey, pkLen, ppVal, pvLen); +} + +int tdbTbcDelete(TBC *pTbc) { return tdbBtcDelete(&pTbc->btc); } + +int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + return tdbBtreeNext(&pTbc->btc, ppKey, kLen, ppVal, vLen); +} + +int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { + return tdbBtcUpsert(&pTbc->btc, pKey, nKey, pData, nData, insert); +} + +int tdbTbcClose(TBC *pTbc) { + if (pTbc) { + tdbBtcClose(&pTbc->btc); + tdbOsFree(pTbc); + } + + return 0; +} + +int tdbTbcIsValid(TBC *pTbc) { return tdbBtcIsValid(&pTbc->btc); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index c00706ce0c..bb59a7fc61 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -103,9 +103,9 @@ typedef struct SPage SPage; #define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED) // tdbEnv.c ==================================== -void tdbEnvAddPager(TENV *pEnv, SPager *pPager); -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); +void tdbEnvAddPager(TDB *pEnv, SPager *pPager); +void tdbEnvRemovePager(TDB *pEnv, SPager *pPager); +SPager *tdbEnvGetPager(TDB *pEnv, const char *fname); // tdbBtree.c ==================================== typedef struct SBTree SBTree; @@ -334,7 +334,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { return pCell; } -struct STEnv { +struct STDB { char *rootDir; char *jfname; int jfd; @@ -357,8 +357,8 @@ struct SPager { SPgno dbOrigSize; SPage *pDirty; u8 inTran; - SPager *pNext; // used by TENV - SPager *pHashNext; // used by TENV + SPager *pNext; // used by TDB + SPager *pHashNext; // used by TDB }; #ifdef __cplusplus diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 3c8bcee3f7..a161f1c589 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -117,8 +117,8 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in TEST(tdb_test, simple_insert1) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; TXN txn; @@ -126,12 +126,12 @@ TEST(tdb_test, simple_insert1) { taosRemoveDir("tdb"); // Open Env - ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + ret = tdbOpen("tdb", 4096, 64, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database compFunc = tKeyCmpr; - ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -152,7 +152,7 @@ TEST(tdb_test, simple_insert1) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -181,7 +181,7 @@ TEST(tdb_test, simple_insert1) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); - ret = tdbGet(pDb, key, strlen(key), &pVal, &vLen); + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); ASSERT(ret == 0); GTEST_ASSERT_EQ(ret, 0); @@ -193,19 +193,19 @@ TEST(tdb_test, simple_insert1) { } { // Iterate to query the DB data - TDBC *pDBC; + TBC *pDBC; void *pKey = NULL; void *pVal = NULL; int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC, NULL); + ret = tdbTbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); - tdbDbcMoveToFirst(pDBC); + tdbTbcMoveToFirst(pDBC); for (;;) { - ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -217,28 +217,28 @@ TEST(tdb_test, simple_insert1) { GTEST_ASSERT_EQ(count, nData); - tdbDbcClose(pDBC); + tdbTbcClose(pDBC); tdbFree(pKey); tdbFree(pVal); } } - ret = tdbDrop(pDb); + ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbClose(pDb); + tdbTbClose(pDb); // Close Env - ret = tdbEnvClose(pEnv); + ret = tdbClose(pEnv); GTEST_ASSERT_EQ(ret, 0); } TEST(tdb_test, simple_insert2) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; TXN txn; @@ -246,12 +246,12 @@ TEST(tdb_test, simple_insert2) { taosRemoveDir("tdb"); // Open Env - ret = tdbEnvOpen("tdb", 1024, 10, &pEnv); + ret = tdbOpen("tdb", 1024, 10, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database compFunc = tDefaultKeyCmpr; - ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -271,24 +271,24 @@ TEST(tdb_test, simple_insert2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } { // Iterate to query the DB data - TDBC *pDBC; + TBC *pDBC; void *pKey = NULL; void *pVal = NULL; int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC, NULL); + ret = tdbTbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); - tdbDbcMoveToFirst(pDBC); + tdbTbcMoveToFirst(pDBC); for (;;) { - ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -300,7 +300,7 @@ TEST(tdb_test, simple_insert2) { GTEST_ASSERT_EQ(count, nData); - tdbDbcClose(pDBC); + tdbTbcClose(pDBC); tdbFree(pKey); tdbFree(pVal); @@ -311,29 +311,29 @@ TEST(tdb_test, simple_insert2) { tdbCommit(pEnv, &txn); tdbTxnClose(&txn); - ret = tdbDrop(pDb); + ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbClose(pDb); + tdbTbClose(pDb); // Close Env - ret = tdbEnvClose(pEnv); + ret = tdbClose(pEnv); GTEST_ASSERT_EQ(ret, 0); } TEST(tdb_test, simple_delete1) { int ret; - TDB *pDb; + TTB *pDb; char key[128]; char data[128]; TXN txn; - TENV *pEnv; + TDB *pEnv; SPoolMem *pPool; void *pKey = NULL; void *pData = NULL; int nKey; - TDBC *pDbc; + TBC *pDbc; int nData; int nKV = 69; @@ -342,11 +342,11 @@ TEST(tdb_test, simple_delete1) { pPool = openPool(); // open env - ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + ret = tdbOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); @@ -356,7 +356,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -365,7 +365,7 @@ TEST(tdb_test, simple_delete1) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); } @@ -374,7 +374,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); - ret = tdbDelete(pDb, key, strlen(key), &txn); + ret = tdbTbDelete(pDb, key, strlen(key), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -382,7 +382,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); if (iData <= 30) { GTEST_ASSERT_EQ(ret, 0); } else { @@ -391,15 +391,15 @@ TEST(tdb_test, simple_delete1) { } // loop to iterate the data - tdbDbcOpen(pDb, &pDbc, NULL); + tdbTbcOpen(pDb, &pDbc, NULL); - ret = tdbDbcMoveToFirst(pDbc); + ret = tdbTbcMoveToFirst(pDbc); GTEST_ASSERT_EQ(ret, 0); pKey = NULL; pData = NULL; for (;;) { - ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); + ret = tdbTbcNext(pDbc, &pKey, &nKey, &pData, &nData); if (ret < 0) break; std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; @@ -407,20 +407,20 @@ TEST(tdb_test, simple_delete1) { std::cout << std::endl; } - tdbDbcClose(pDbc); + tdbTbcClose(pDbc); tdbCommit(pEnv, &txn); closePool(pPool); - tdbClose(pDb); - tdbEnvClose(pEnv); + tdbTbClose(pDb); + tdbClose(pEnv); } TEST(tdb_test, simple_upsert1) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; int nData = 100000; char key[64]; char data[64]; @@ -431,11 +431,11 @@ TEST(tdb_test, simple_upsert1) { taosRemoveDir("tdb"); // open env - ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + ret = tdbOpen("tdb", 4096, 64, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, NULL, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); pPool = openPool(); @@ -446,7 +446,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -454,7 +454,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } @@ -463,7 +463,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -473,11 +473,11 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } - tdbClose(pDb); - tdbEnvClose(pEnv); + tdbTbClose(pDb); + tdbClose(pEnv); } \ No newline at end of file diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d2d38bd9bc..92c5e9faf7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -145,9 +145,9 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); } while (0) #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) -#define CONN_PERSIST_TIME(para) (para * 1000 * 10) -#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_PERSIST_TIME(para) (para * 1000 * 10) +#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ @@ -227,7 +227,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); #define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) -#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) +#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) static void* cliWorkThread(void* arg); @@ -924,8 +924,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { arg->param1 = pMsg; arg->param2 = pThrd; transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - - cliDestroy((uv_handle_t*)pConn->stream); + cliDestroyConn(pConn, true); return -1; } } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index d1bd78f809..84adeb4bf6 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -264,6 +264,7 @@ static void uvHandleReq(SSrvConn* pConn) { CONN_SHOULD_RELEASE(pConn, pHead); STransMsg transMsg; + memset(&transMsg, 0, sizeof(transMsg)); transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = pHead->content; transMsg.msgType = pHead->msgType; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 546075a1d9..ba1cd00fcb 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -251,16 +251,12 @@ CaseCtrl gCaseCtrl = { .bindColNum = 0, .bindTagNum = 0, .bindRowNum = 0, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, .bindTagTypeNum = 0, .bindTagTypeList = NULL, - .optrIdxListNum = tListLen(optrIdxList), - .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = false, .runTimes = 0, - .caseIdx = 23, + .caseIdx = 1, .caseNum = 1, .caseRunIdx = -1, .caseRunNum = 1, @@ -1101,7 +1097,9 @@ void destroyData(BindData *data) { taosMemoryFree(data->binaryLen); taosMemoryFree(data->isNull); taosMemoryFree(data->pBind); + taosMemoryFree(data->pTags); taosMemoryFree(data->colTypes); + taosMemoryFree(data->sql); } void bpFetchRows(TAOS_RES *result, bool printr, int32_t *rows) { diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index d2cccd0532..b536a70515 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -219,6 +219,41 @@ class TDTestCase: tdSql.query("drop topic %s"%topicName1) + tdLog.info("creat the same topic name , and start to consume") + self.initConsumerTable() + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self, cfgPath, buildPath):