From 5e981e43ada23b5c6ad024570086baa56ebebc74 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 9 Dec 2021 03:39:15 -0500 Subject: [PATCH 01/22] TD-11819 Parsing insert statement and assembling binary objects. --- .gitignore | 1 + include/common/ttime.h | 4 +- include/common/tvariant.h | 2 +- include/libs/parser/parser.h | 31 +- include/os/osString.h | 4 +- include/util/tutil.h | 2 +- source/common/src/ttime.c | 12 +- source/common/src/tvariant.c | 2 +- source/libs/parser/inc/dataBlockMgt.h | 204 ++++ source/libs/parser/inc/insertParser.h | 4 + source/libs/parser/inc/parserInt.h | 8 - source/libs/parser/inc/parserUtil.h | 4 +- source/libs/parser/inc/ttoken.h | 4 +- source/libs/parser/src/astGenerator.c | 2 +- source/libs/parser/src/dataBlockMgt.c | 685 +++++++++++++ source/libs/parser/src/insertParser.c | 1272 +++++++++++++++++++++++++ source/libs/parser/src/parser.c | 7 +- source/libs/parser/src/parserUtil.c | 29 +- source/libs/parser/src/ttokenizer.c | 15 +- source/os/src/osString.c | 4 +- source/util/src/tutil.c | 4 +- 21 files changed, 2250 insertions(+), 50 deletions(-) create mode 100644 source/libs/parser/inc/dataBlockMgt.h create mode 100644 source/libs/parser/src/dataBlockMgt.c create mode 100644 source/libs/parser/src/insertParser.c diff --git a/.gitignore b/.gitignore index ba8543e518..c651127992 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ debug/ release/ target/ debs/ +deps/ rpms/ mac/ *.pyc diff --git a/include/common/ttime.h b/include/common/ttime.h index 926d23c239..ed6496aca6 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -44,10 +44,10 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); -int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); +int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision); -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); +int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); void deltaToUtcInitOnce(); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); diff --git a/include/common/tvariant.h b/include/common/tvariant.h index 552c7eaa3f..e068c6eb6c 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -40,7 +40,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* bool taosVariantIsValid(SVariant *pVar); -void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type); +void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type); void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 0e6c352d71..ea8caa691d 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -131,6 +131,18 @@ struct SInsertStmtInfo; */ bool qIsInsertSql(const char* pStr, size_t length); +typedef struct SParseContext { + const char* pSql; // sql string + size_t sqlLen; // length of the sql string + int64_t id; // operator id, generated by uuid generator + const char* pDbname; + const SEpSet* pEpSet; + int8_t schemaAttached; // denote if submit block is built with table schema or not + + char* pMsg; // extended error message if exists to help avoid the problem in sql statement. + int32_t msgLen; // max length of the msg +} SParseContext; + /** * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST. * @param pSql sql string @@ -141,16 +153,29 @@ bool qIsInsertSql(const char* pStr, size_t length); */ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen); +typedef enum { + PAYLOAD_TYPE_KV = 0, + PAYLOAD_TYPE_RAW = 1, +} EPayloadType; + +typedef struct SInsertStmtInfo { + // SHashObj* pTableBlockHashList; // data block for each table + SArray* pDataBlocks; // SArray. Merged submit block for each vgroup + int8_t schemaAttache; // denote if submit block is built with table schema or not + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* sql; // current sql statement position +} SInsertStmtInfo; + /** * Parse the insert sql statement. * @param pStr sql string * @param length length of the sql string - * @param pInsertParam data in binary format to submit to vnode directly. * @param id operator id, generated by uuid generator. * @param msg extended error message if exists to help avoid the problem in sql statement. - * @return + * @return data in binary format to submit to vnode directly. */ -int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen); + int32_t qParseInsertSql(SParseContext* pContext, struct SInsertStmtInfo** pInfo); /** * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that diff --git a/include/os/osString.h b/include/os/osString.h index 358324075f..582411d444 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -38,11 +38,11 @@ extern "C" { (dst)[(size)-1] = 0; \ } while (0) -int64_t taosStr2int64(char *str); +int64_t taosStr2int64(const char *str); // USE_LIBICONV int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); -bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); +bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); char * taosCharsetReplace(char *charsetstr); diff --git a/include/util/tutil.h b/include/util/tutil.h index 573dee9339..070ca6b601 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -29,7 +29,7 @@ int32_t strdequote(char *src); int32_t strndequote(char *dst, const char* z, int32_t len); int32_t strRmquote(char *z, int32_t len); size_t strtrim(char *src); -char * strnchr(char *haystack, char needle, int32_t len, bool skipquote); +char * strnchr(const char *haystack, char needle, int32_t len, bool skipquote); char ** strsplit(char *src, const char *delim, int32_t *num); char * strtolower(char *dst, const char *src); char * strntolower(char *dst, const char *src, int32_t n); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 6c795bb0cc..5c6b9ff6f3 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -82,18 +82,18 @@ void deltaToUtcInitOnce() { } static int64_t parseFraction(char* str, char** end, int32_t timePrec); -static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim); +static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec); static int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec); static char* forwardToTimeStringEnd(char* str); -static bool checkTzPresent(char *str, int32_t len); +static bool checkTzPresent(const char *str, int32_t len); static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = { parseLocaltime, parseLocaltimeDst }; -int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { +int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { return parseTimeWithTz(timestr, time, timePrec, 'T'); @@ -104,7 +104,7 @@ int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePre } } -bool checkTzPresent(char *str, int32_t len) { +bool checkTzPresent(const char *str, int32_t len) { char *seg = forwardToTimeStringEnd(str); int32_t seg_len = len - (int32_t)(seg - str); @@ -237,7 +237,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { * 2013-04-12T15:52:01+0800 * 2013-04-12T15:52:01.123+0800 */ -int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim) { +int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); @@ -432,7 +432,7 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time * d - Days (24 hours) * w - Weeks (7 days) */ -int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { +int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { errno = 0; char* endPtr = NULL; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index ed1f4d2bfd..8a5a300fcf 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -75,7 +75,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* return 0; } -void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) { +void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) { int32_t ret = 0; memset(pVar, 0, sizeof(SVariant)); diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h new file mode 100644 index 0000000000..d163d0b741 --- /dev/null +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_DATABLOCKMGT_H +#define TDENGINE_DATABLOCKMGT_H + +#include "catalog.h" +#include "os.h" +#include "ttypes.h" +#include "tname.h" + +#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) + +typedef enum EOrderStatus { + ORDER_STATUS_UNKNOWN = 0, + ORDER_STATUS_ORDERED = 1, + ORDER_STATUS_DISORDERED = 2, +} EOrderStatus; + +typedef enum EValStat { + VAL_STAT_HAS = 0x0, // 0 means has val + VAL_STAT_NONE = 0x01, // 1 means no val +} EValStat; + +typedef enum ERowCompareStat { + ROW_COMPARE_NO_NEED = 0, + ROW_COMPARE_NEED = 1, +} ERowCompareStat; + +typedef struct SBoundColumn { + int32_t offset; // all column offset value + int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future + uint8_t valStat; // denote if current column bound or not(0 means has val, 1 means no val) +} SBoundColumn; + +typedef struct { + uint16_t schemaColIdx; + uint16_t boundIdx; + uint16_t finalIdx; +} SBoundIdxInfo; + +typedef struct SParsedDataColInfo { + int16_t numOfCols; + int16_t numOfBound; + uint16_t flen; // TODO: get from STSchema + uint16_t allNullLen; // TODO: get from STSchema + uint16_t extendedVarLen; + int32_t *boundedColumns; // bound column idx according to schema + SBoundColumn *cols; + SBoundIdxInfo *colIdxInfo; + int8_t orderStatus; // bound columns +} SParsedDataColInfo; + +typedef struct SParamInfo { + int32_t idx; + uint8_t type; + uint8_t timePrec; + int16_t bytes; + uint32_t offset; +} SParamInfo; + +typedef struct { + int32_t dataLen; // len of SDataRow + int32_t kvLen; // len of SKVRow +} SMemRowInfo; + +typedef struct { + uint8_t memRowType; // default is 0, that is SDataRow + uint8_t compareStat; // 0 no need, 1 need compare + TDRowTLenT kvRowInitLen; + SMemRowInfo *rowInfo; +} SMemRowBuilder; + +typedef struct SBlockKeyTuple { + TSKEY skey; + void* payloadAddr; +} SBlockKeyTuple; + +typedef struct SBlockKeyInfo { + int32_t maxBytesAlloc; + SBlockKeyTuple* pKeyTuple; +} SBlockKeyInfo; + +typedef struct STableDataBlocks { + SName tableName; + int8_t tsSource; // where does the UNIX timestamp come from, server or client + bool ordered; // if current rows are ordered or not + int64_t vgId; // virtual group id + int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending + int32_t numOfTables; // number of tables in current submit block + int32_t rowSize; // row size for current table + uint32_t nAllocSize; + uint32_t headerSize; // header for table info (uid, tid, submit metadata) + uint32_t size; + STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache + char *pData; + bool cloned; + STagData tagData; + + SParsedDataColInfo boundColumnInfo; + + // for parameter ('?') binding + uint32_t numOfAllocedParams; + uint32_t numOfParams; + SParamInfo * params; + SMemRowBuilder rowBuilder; +} STableDataBlocks; + +static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) { + memRowSetType(row, memRowType); + if (isDataRowT(memRowType)) { + dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion); + dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen)); + } else { + ASSERT(nBoundCols > 0); + memRowSetKvVersion(row, pBlock->pTableMeta->sversion); + kvRowSetNCols(memRowKvBody(row), nBoundCols); + kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); + } +} + +static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { + ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize); + return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen; +} + +// Applicable to consume by one row +static FORCE_INLINE void appendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, + int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen, + uint8_t compareStat) { + tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset); + if (compareStat == ROW_COMPARE_NEED) { + tdGetColAppendDeltaLen(value, colType, dataLen, kvLen); + } +} + +static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd, + int32_t idx, int32_t *toffset, int16_t *colId) { + int32_t schemaIdx = 0; + if (IS_DATA_COL_ORDERED(spd)) { + schemaIdx = spd->boundedColumns[idx]; + if (isDataRowT(memRowType)) { + *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart + } else { + *toffset = idx * sizeof(SColIdx); // the offset of SColIdx + } + } else { + ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); + schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; + if (isDataRowT(memRowType)) { + *toffset = (spd->cols + schemaIdx)->toffset; + } else { + *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); + } + } + *colId = pSchema[schemaIdx].colId; +} + +static FORCE_INLINE void checkAndConvertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) { + if (isDataRow(row)) { + if (kvLen < (dataLen * KVRatioConvert)) { + memRowSetConvert(row); + } + } else if (kvLen > dataLen) { + memRowSetConvert(row); + } +} + +static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { + pBlocks->tid = pTableMeta->suid; + pBlocks->uid = pTableMeta->uid; + pBlocks->sversion = pTableMeta->sversion; + + if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } else { + pBlocks->numOfRows += numOfRows; + return TSDB_CODE_SUCCESS; + } +} + +int32_t schemaIdxCompar(const void *lhs, const void *rhs); +int32_t boundIdxCompar(const void *lhs, const void *rhs); +void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); +void destroyBoundColumnInfo(SParsedDataColInfo* pColList); +int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); +int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); +int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, + SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap); + +#endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/insertParser.h b/source/libs/parser/inc/insertParser.h index 49e678cd54..a2a910fcca 100644 --- a/source/libs/parser/inc/insertParser.h +++ b/source/libs/parser/inc/insertParser.h @@ -16,4 +16,8 @@ #ifndef TDENGINE_INSERTPARSER_H #define TDENGINE_INSERTPARSER_H +#include "parser.h" + +int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo); + #endif // TDENGINE_INSERTPARSER_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index ca02165382..47f655d446 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -26,14 +26,6 @@ extern "C" { struct SSqlNode; -typedef struct SInsertStmtInfo { - SHashObj *pTableBlockHashList; // data block for each table - SArray *pDataBlocks; // SArray. Merged submit block for each vgroup - int8_t schemaAttached; // denote if submit block is built with table schema or not - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - uint32_t insertType; // insert data from [file|sql statement| bound statement] - char *sql; // current sql statement position -} SInsertStmtInfo; typedef struct SInternalField { TAOS_FIELD field; diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 12ffe696c1..6c95c4327b 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -46,7 +46,7 @@ SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index); int32_t parserValidateIdToken(SToken* pToken); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); -int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr); +int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo); @@ -61,6 +61,8 @@ void cleanupColumnCond(SArray** pCond); uint32_t convertRelationalOperator(SToken *pToken); int32_t getExprFunctionId(SExprInfo *pExprInfo); +STableMeta* tableMetaDup(const STableMeta* pTableMeta); + #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/ttoken.h b/source/libs/parser/inc/ttoken.h index bacabe299e..c041edf2b2 100644 --- a/source/libs/parser/inc/ttoken.h +++ b/source/libs/parser/inc/ttoken.h @@ -44,7 +44,7 @@ typedef struct SToken { * @param tokenType * @return */ -uint32_t tGetToken(char *z, uint32_t *tokenType); +uint32_t tGetToken(const char *z, uint32_t *tokenType); /** * enhanced tokenizer for sql string. @@ -54,7 +54,7 @@ uint32_t tGetToken(char *z, uint32_t *tokenType); * @param isPrevOptr * @return */ -SToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr); +SToken tStrGetToken(const char *str, int32_t *i, bool isPrevOptr); /** * check if it is a keyword or not diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index f328486556..0d184fd74a 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -191,7 +191,7 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { pExpr->type = SQL_NODE_EXPR; if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { - char* endPos = pRight->exprToken.z + pRight->exprToken.n; + const char* endPos = pRight->exprToken.z + pRight->exprToken.n; pExpr->exprToken.z = pLeft->exprToken.z; pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z); pExpr->exprToken.type = pLeft->exprToken.type; diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c new file mode 100644 index 0000000000..a08f4fb15b --- /dev/null +++ b/source/libs/parser/src/dataBlockMgt.c @@ -0,0 +1,685 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "dataBlockMgt.h" + +// #include "astGenerator.h" +// #include "parserInt.h" +#include "catalog.h" +#include "parserUtil.h" +#include "queryInfoUtil.h" +// #include "ttoken.h" +// #include "function.h" +// #include "ttime.h" +// #include "tglobal.h" +#include "taosmsg.h" + +#define IS_RAW_PAYLOAD(t) \ + (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + +static int32_t rowDataCompar(const void *lhs, const void *rhs) { + TSKEY left = *(TSKEY *)lhs; + TSKEY right = *(TSKEY *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols) { + pColList->numOfCols = numOfCols; + pColList->numOfBound = numOfCols; + pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode + pColList->boundedColumns = calloc(pColList->numOfCols, sizeof(int32_t)); + pColList->cols = calloc(pColList->numOfCols, sizeof(SBoundColumn)); + pColList->colIdxInfo = NULL; + pColList->flen = 0; + pColList->allNullLen = 0; + + int32_t nVar = 0; + for (int32_t i = 0; i < pColList->numOfCols; ++i) { + uint8_t type = pSchema[i].type; + if (i > 0) { + pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; + pColList->cols[i].toffset = pColList->flen; + } + pColList->flen += TYPE_BYTES[type]; + switch (type) { + case TSDB_DATA_TYPE_BINARY: + pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); + ++nVar; + break; + case TSDB_DATA_TYPE_NCHAR: + pColList->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + ++nVar; + break; + default: + break; + } + pColList->boundedColumns[i] = pSchema[i].colId; + } + pColList->allNullLen += pColList->flen; + pColList->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); +} + +int32_t schemaIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)lhs; + uint16_t right = *(uint16_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +int32_t boundIdxCompar(const void *lhs, const void *rhs) { + uint16_t left = *(uint16_t *)POINTER_SHIFT(lhs, sizeof(uint16_t)); + uint16_t right = *(uint16_t *)POINTER_SHIFT(rhs, sizeof(uint16_t)); + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} + +void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { + tfree(pColList->boundedColumns); + tfree(pColList->cols); + tfree(pColList->colIdxInfo); +} + +static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name, + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { + STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); + if (dataBuf == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + dataBuf->nAllocSize = (uint32_t)defaultSize; + dataBuf->headerSize = startOffset; + + // the header size will always be the startOffset value, reserved for the subumit block header + if (dataBuf->nAllocSize <= dataBuf->headerSize) { + dataBuf->nAllocSize = dataBuf->headerSize * 2; + } + + //dataBuf->pData = calloc(1, dataBuf->nAllocSize); + dataBuf->pData = malloc(dataBuf->nAllocSize); + if (dataBuf->pData == NULL) { + tfree(dataBuf); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); + + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tableMetaDup(pTableMeta); + + SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; + SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); + setBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); + + dataBuf->ordered = true; + dataBuf->prevTS = INT64_MIN; + dataBuf->rowSize = rowSize; + dataBuf->size = startOffset; + dataBuf->tsSource = -1; + dataBuf->vgId = dataBuf->pTableMeta->vgId; + + tNameAssign(&dataBuf->tableName, name); + + assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + + *dataBlocks = dataBuf; + return TSDB_CODE_SUCCESS; +} + +int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, + SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, + SArray* pBlockList) { + *dataBlocks = NULL; + STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); + if (t1 != NULL) { + *dataBlocks = *t1; + } + + if (*dataBlocks == NULL) { + int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); + if (pBlockList) { + taosArrayPush(pBlockList, dataBlocks); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t getRowExpandSize(STableMeta* pTableMeta) { + int32_t result = TD_MEM_ROW_DATA_HEAD_SIZE; + int32_t columns = getNumOfColumns(pTableMeta); + SSchema* pSchema = getTableColumnSchema(pTableMeta); + for (int32_t i = 0; i < columns; i++) { + if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { + result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; + } + } + return result; +} + +/** + * TODO: Move to tdataformat.h and refactor when STSchema available. + * - fetch flen and toffset from STSChema and remove param spd + */ +static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, + SParsedDataColInfo *spd) { + ASSERT(isKvRow(src)); + SKVRow kvRow = memRowKvBody(src); + SDataRow dataRow = memRowDataBody(dest); + + memRowSetType(dest, SMEM_ROW_DATA); + dataRowSetVersion(dataRow, memRowKvVersion(src)); + dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen)); + + int32_t kvIdx = 0; + for (int i = 0; i < nCols; ++i) { + SSchema *schema = pSchema + i; + void * val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx); + tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type, + (spd->cols + i)->toffset); + } +} + +// TODO: Move to tdataformat.h and refactor when STSchema available. +static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, + SParsedDataColInfo *spd) { + ASSERT(isDataRow(src)); + + SDataRow dataRow = memRowDataBody(src); + SKVRow kvRow = memRowKvBody(dest); + + memRowSetType(dest, SMEM_ROW_KV); + memRowSetKvVersion(kvRow, dataRowVersion(dataRow)); + kvRowSetNCols(kvRow, nBoundCols); + kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols)); + + int32_t toffset = 0, kvOffset = 0; + for (int i = 0; i < nCols; ++i) { + if ((spd->cols + i)->valStat == VAL_STAT_HAS) { + SSchema *schema = pSchema + i; + toffset = (spd->cols + i)->toffset; + void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE); + tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset); + kvOffset += sizeof(SColIdx); + } + } +} + +// TODO: Move to tdataformat.h and refactor when STSchema available. +static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) { + STableMeta * pTableMeta = pBlock->pTableMeta; + STableComInfo tinfo = getTableInfo(pTableMeta); + SSchema * pSchema = getTableColumnSchema(pTableMeta); + SParsedDataColInfo *spd = &pBlock->boundColumnInfo; + + ASSERT(dest != src); + + if (isDataRow(src)) { + // TODO: Can we use pBlock -> numOfParam directly? + ASSERT(spd->numOfBound > 0); + convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd); + } else { + convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd); + } +} + +void destroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { + if (pDataBlock == NULL) { + return; + } + + tfree(pDataBlock->pData); + + if (removeMeta) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pDataBlock->tableName, name); + + // taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + } + + if (!pDataBlock->cloned) { + tfree(pDataBlock->params); + + // free the refcount for metermeta + if (pDataBlock->pTableMeta != NULL) { + tfree(pDataBlock->pTableMeta); + } + + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + } + + tfree(pDataBlock); +} + +void* destroyBlockArrayList(SArray* pDataBlockList) { + if (pDataBlockList == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(pDataBlockList); + for (int32_t i = 0; i < size; i++) { + void* d = taosArrayGetP(pDataBlockList, i); + destroyDataBlock(d, false); + } + + taosArrayDestroy(pDataBlockList); + return NULL; +} + +// data block is disordered, sort it in ascending order +void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; + + // size is less than the total size, since duplicated rows may be removed yet. + assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); + + if (!dataBuf->ordered) { + char *pBlockData = pBlocks->data; + qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); + + int32_t i = 0; + int32_t j = 1; + + while (j < pBlocks->numOfRows) { + TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); + TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); + + if (ti == tj) { + ++j; + continue; + } + + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); + } + + ++j; + } + + dataBuf->ordered = true; + + pBlocks->numOfRows = i + 1; + dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; + } + + dataBuf->prevTS = INT64_MIN; +} + +// data block is disordered, sort it in ascending order +int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKeyInfo) { + SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; + int16_t nRows = pBlocks->numOfRows; + + // size is less than the total size, since duplicated rows may be removed yet. + + // allocate memory + size_t nAlloc = nRows * sizeof(SBlockKeyTuple); + if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { + char *tmp = realloc(pBlkKeyInfo->pKeyTuple, nAlloc); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; + pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; + } + memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); + + int32_t extendedRowSize = getExtendedRowSize(dataBuf); + SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + char * pBlockData = pBlocks->data; + int n = 0; + while (n < nRows) { + pBlkKeyTuple->skey = memRowKey(pBlockData); + pBlkKeyTuple->payloadAddr = pBlockData; + + // next loop + pBlockData += extendedRowSize; + ++pBlkKeyTuple; + ++n; + } + + if (!dataBuf->ordered) { + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); + + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + int32_t i = 0; + int32_t j = 1; + while (j < nRows) { + TSKEY ti = (pBlkKeyTuple + i)->skey; + TSKEY tj = (pBlkKeyTuple + j)->skey; + + if (ti == tj) { + ++j; + continue; + } + + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); + } + ++j; + } + + dataBuf->ordered = true; + pBlocks->numOfRows = i + 1; + } + + dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; + dataBuf->prevTS = INT64_MIN; + + return 0; +} + +// Erase the empty space reserved for binary data +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, int8_t schemaAttached, bool isRawPayload) { + // TODO: optimize this function, handle the case while binary is not presented + STableMeta* pTableMeta = pTableDataBlock->pTableMeta; + STableComInfo tinfo = getTableInfo(pTableMeta); + SSchema* pSchema = getTableColumnSchema(pTableMeta); + + SSubmitBlk* pBlock = pDataBlock; + memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); + pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk); + + int32_t flen = 0; // original total length of row + + // schema needs to be included into the submit data block + if (schemaAttached) { + int32_t numOfCols = getNumOfColumns(pTableDataBlock->pTableMeta); + for(int32_t j = 0; j < numOfCols; ++j) { + STColumn* pCol = (STColumn*) pDataBlock; + pCol->colId = htons(pSchema[j].colId); + pCol->type = pSchema[j].type; + pCol->bytes = htons(pSchema[j].bytes); + pCol->offset = 0; + + pDataBlock = (char*)pDataBlock + sizeof(STColumn); + flen += TYPE_BYTES[pSchema[j].type]; + } + + int32_t schemaSize = sizeof(STColumn) * numOfCols; + pBlock->schemaLen = schemaSize; + } else { + if (isRawPayload) { + for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { + flen += TYPE_BYTES[pSchema[j].type]; + } + } + pBlock->schemaLen = 0; + } + + char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); + pBlock->dataLen = 0; + int32_t numOfRows = htons(pBlock->numOfRows); + + if (isRawPayload) { + for (int32_t i = 0; i < numOfRows; ++i) { + SMemRow memRow = (SMemRow)pDataBlock; + memRowSetType(memRow, SMEM_ROW_DATA); + SDataRow trow = memRowDataBody(memRow); + dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen)); + dataRowSetVersion(trow, pTableMeta->sversion); + + int toffset = 0; + for (int32_t j = 0; j < tinfo.numOfColumns; j++) { + tdAppendColVal(trow, p, pSchema[j].type, toffset); + toffset += TYPE_BYTES[pSchema[j].type]; + p += pSchema[j].bytes; + } + + pDataBlock = (char*)pDataBlock + memRowTLen(memRow); + pBlock->dataLen += memRowTLen(memRow); + } + } else { + for (int32_t i = 0; i < numOfRows; ++i) { + char* payload = (blkKeyTuple + i)->payloadAddr; + if (isNeedConvertRow(payload)) { + convertSMemRow(pDataBlock, payload, pTableDataBlock); + TDRowTLenT rowTLen = memRowTLen(pDataBlock); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; + } else { + TDRowTLenT rowTLen = memRowTLen(payload); + memcpy(pDataBlock, payload, rowTLen); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; + } + } + } + + int32_t len = pBlock->dataLen + pBlock->schemaLen; + pBlock->dataLen = htonl(pBlock->dataLen); + pBlock->schemaLen = htonl(pBlock->schemaLen); + + return len; +} + +static void extractTableNameList(SHashObj* pHashObj, bool freeBlockMap) { + // pInsertParam->numOfTables = (int32_t) taosHashGetSize(pInsertParam->pTableBlockHashList); + // if (pInsertParam->pTableNameList == NULL) { + // pInsertParam->pTableNameList = malloc(pInsertParam->numOfTables * POINTER_BYTES); + // } + + // STableDataBlocks **p1 = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); + // int32_t i = 0; + // while(p1) { + // STableDataBlocks* pBlocks = *p1; + // pInsertParam->pTableNameList[i++] = tNameDup(&pBlocks->tableName); + // p1 = taosHashIterate(pInsertParam->pTableBlockHashList, p1); + // } + + // if (freeBlockMap) { + // pInsertParam->pTableBlockHashList = tscDestroyBlockHashTable(pInsertParam->pTableBlockHashList, false); + // } +} + +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap) { + const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + int code = 0; + bool isRawPayload = IS_RAW_PAYLOAD(payloadType); + void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); + + STableDataBlocks** p = taosHashIterate(pHashObj, NULL); + + STableDataBlocks* pOneTableBlock = *p; + + SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock + + while(pOneTableBlock) { + SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; + if (pBlocks->numOfRows > 0) { + // the maximum expanded size in byte when a row-wise data is converted to SDataRow format + int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; + STableDataBlocks* dataBuf = NULL; + + int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, + INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + if (ret != TSDB_CODE_SUCCESS) { + taosHashCleanup(pVnodeDataBlockHashList); + destroyBlockArrayList(pVnodeDataBlockList); + tfree(blkKeyInfo.pKeyTuple); + return ret; + } + + int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); + + if (dataBuf->nAllocSize < destSize) { + dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); + + char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); + if (tmp != NULL) { + dataBuf->pData = tmp; + //memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); + } else { // failed to allocate memory, free already allocated memory and return error code + taosHashCleanup(pVnodeDataBlockHashList); + destroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); + + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + if (isRawPayload) { + sortRemoveDataBlockDupRowsRaw(pOneTableBlock); + } else { + if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { + taosHashCleanup(pVnodeDataBlockHashList); + destroyBlockArrayList(pVnodeDataBlockList); + tfree(dataBuf->pData); + tfree(blkKeyInfo.pKeyTuple); + return code; + } + ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); + } + + int32_t len = pBlocks->numOfRows * + (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); + + pBlocks->tid = htonl(pBlocks->tid); + pBlocks->uid = htobe64(pBlocks->uid); + pBlocks->sversion = htonl(pBlocks->sversion); + pBlocks->numOfRows = htons(pBlocks->numOfRows); + pBlocks->schemaLen = 0; + + // erase the empty space reserved for binary data + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); + assert(finalLen <= len); + + dataBuf->size += (finalLen + sizeof(SSubmitBlk)); + assert(dataBuf->size <= dataBuf->nAllocSize); + + // the length does not include the SSubmitBlk structure + pBlocks->dataLen = htonl(finalLen); + dataBuf->numOfTables += 1; + + pBlocks->numOfRows = 0; + } + + p = taosHashIterate(pHashObj, p); + if (p == NULL) { + break; + } + + pOneTableBlock = *p; + } + + extractTableNameList(pHashObj, freeBlockMap); + + // free the table data blocks; + // pInsertParam->pDataBlocks = pVnodeDataBlockList; + taosHashCleanup(pVnodeDataBlockHashList); + tfree(blkKeyInfo.pKeyTuple); + + return TSDB_CODE_SUCCESS; +} + +int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) { + size_t remain = pDataBlock->nAllocSize - pDataBlock->size; + const int factor = 5; + uint32_t nAllocSizeOld = pDataBlock->nAllocSize; + + // expand the allocated size + if (remain < rowSize * factor) { + while (remain < rowSize * factor) { + pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); + remain = pDataBlock->nAllocSize - pDataBlock->size; + } + + char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); + if (tmp != NULL) { + pDataBlock->pData = tmp; + memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); + } else { + // do nothing, if allocate more memory failed + pDataBlock->nAllocSize = nAllocSizeOld; + *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; + return TSDB_CODE_SUCCESS; +} + +int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen) { + ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols)); + if (nRows > 0) { + // already init(bind multiple rows by single column) + if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) { + return TSDB_CODE_SUCCESS; + } + } + + // default compareStat is ROW_COMPARE_NO_NEED + if (nBoundCols == 0) { // file input + pBuilder->memRowType = SMEM_ROW_DATA; + return TSDB_CODE_SUCCESS; + } else { + float boundRatio = ((float)nBoundCols / (float)nCols); + + if (boundRatio < KVRatioKV) { + pBuilder->memRowType = SMEM_ROW_KV; + return TSDB_CODE_SUCCESS; + } else if (boundRatio > KVRatioData) { + pBuilder->memRowType = SMEM_ROW_DATA; + return TSDB_CODE_SUCCESS; + } + pBuilder->compareStat = ROW_COMPARE_NEED; + + if (boundRatio < KVRatioPredict) { + pBuilder->memRowType = SMEM_ROW_KV; + } else { + pBuilder->memRowType = SMEM_ROW_DATA; + } + } + + pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx); + + if (nRows > 0) { + pBuilder->rowInfo = calloc(nRows, sizeof(SMemRowInfo)); + if (pBuilder->rowInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int i = 0; i < nRows; ++i) { + (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen; + (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen; + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c new file mode 100644 index 0000000000..61f92eebe0 --- /dev/null +++ b/source/libs/parser/src/insertParser.c @@ -0,0 +1,1272 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "insertParser.h" + +// #include "astGenerator.h" +#include "dataBlockMgt.h" +#include "parserInt.h" +#include "parserUtil.h" +#include "queryInfoUtil.h" +#include "tglobal.h" +#include "ttime.h" +#include "ttoken.h" +// #include "function.h" +#include "ttypes.h" + +#define NEXT_TOKEN(pSql, sToken) \ + do { \ + int32_t index = 0; \ + sToken = tStrGetToken(pSql, &index, false); \ + pSql += index; \ + } while (0) + +#define CHECK_CODE(expr) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + terrno = code; \ + return terrno; \ + } \ + } while (0) + +#define CHECK_CODE_1(expr, destroy) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + (void)destroy; \ + terrno = code; \ + return terrno; \ + } \ + } while (0) + +#define CHECK_CODE_2(expr, destroy1, destroy2) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + (void)destroy1; \ + (void)destroy2; \ + terrno = code; \ + return terrno; \ + } \ + } while (0) + +enum { + TSDB_USE_SERVER_TS = 0, + TSDB_USE_CLI_TS = 1, +}; + +typedef struct SInsertParseContext { + SParseContext* pComCxt; + const char* pSql; + SMsgBuf msg; + struct SCatalog* pCatalog; + SMetaData meta; // need release + const STableMeta* pTableMeta; + SHashObj* pTableBlockHashObj; // data block for each table. need release + int32_t totalNum; + SInsertStmtInfo* pOutput; +} SInsertParseContext; + +static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; +static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; + +static bool isNullStr(SToken *pToken) { + return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && + (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); +} + +static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) { + errno = 0; + *value = strtold(pToken->z, endPtr); + + // not a valid integer number, return error + if ((*endPtr - pToken->z) != pToken->n) { + return TK_ILLEGAL; + } + + return pToken->type; +} + +static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) { + errno = 0; + int32_t ret = 0; + + char* endPtr = NULL; + if (type == TK_FLOAT) { + double v = strtod(z, &endPtr); + if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) { + ret = -1; + } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) { + ret = -1; + } else { + *value = (int64_t) round(v); + } + + errno = 0; + return ret; + } + + int32_t radix = 10; + if (type == TK_HEX) { + radix = 16; + } else if (type == TK_BIN) { + radix = 2; + } + + // the string may be overflow according to errno + if (!issigned) { + const char *p = z; + while(*p != 0 && *p == ' ') p++; + if (*p != 0 && *p == '-') { return -1;} + + *value = strtoull(z, &endPtr, radix); + } else { + *value = strtoll(z, &endPtr, radix); + } + + // not a valid integer number, return error + if (endPtr - z != n || errno == ERANGE) { + ret = -1; + } + + errno = 0; + return ret; +} + +static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) { + SInsertStmtInfo *info = calloc(1, sizeof(SQueryStmtInfo)); + if (NULL == info) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + // info->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + // if (NULL == info->pTableBlockHashList) { + // tfree(info); + // return TSDB_CODE_TSC_OUT_OF_MEMORY; + // } + *pInsertInfo = info; + return TSDB_CODE_SUCCESS; +} + +static int32_t skipInsertInto(SInsertParseContext* pCxt) { + SToken sToken; + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_INSERT != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword INSERT is expected", sToken.z); + } + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_INTO != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword INTO is expected", sToken.z); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) { + if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(&pCxt->msg, "invalid table name"); + } + + SName name = {0}; + strndequote(name.tname, pStname->z, pStname->n); + taosArrayPush(tableNameList, &name); + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SMetaReq* pMetaReq) { + pMetaReq->pTableName = taosArrayInit(4, sizeof(SName)); + return buildTableName(pCxt, pStname, pMetaReq->pTableName); +} + +static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { + SMetaReq req; + CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); + CHECK_CODE(catalogGetMetaData(pCxt->pCatalog, &req, &pCxt->meta)); + pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); + return TSDB_CODE_SUCCESS; +} + +// todo speedup by using hash list +static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { + while (start < end) { + if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { + return start; + } + ++start; + } + return -1; +} + +static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { + // once the data block is disordered, we do NOT keep previous timestamp any more + if (!pDataBlocks->ordered) { + return TSDB_CODE_SUCCESS; + } + + TSKEY k = *(TSKEY *)start; + + if (k == INT64_MIN) { + if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { + return -1; + } else if (pDataBlocks->tsSource == -1) { + pDataBlocks->tsSource = TSDB_USE_SERVER_TS; + } + } else { + if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { + return -1; // client time/server time can not be mixed + } else if (pDataBlocks->tsSource == -1) { + pDataBlocks->tsSource = TSDB_USE_CLI_TS; + } + } + + if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { + pDataBlocks->ordered = false; + // tscWarn("NOT ordered input timestamp"); + } + + pDataBlocks->prevTS = k; + return TSDB_CODE_SUCCESS; +} + +static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec, int64_t *time) { + int32_t index = 0; + SToken sToken; + int64_t interval; + int64_t useconds = 0; + const char* pTokenEnd = pCxt->pSql; + + if (pToken->type == TK_NOW) { + useconds = taosGetTimestamp(timePrec); + } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) { + // do nothing + } else if (pToken->type == TK_INTEGER) { + useconds = taosStr2int64(pToken->z); + } else { + // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm); + if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp format", pToken->z); + } + + return TSDB_CODE_SUCCESS; + } + + for (int k = pToken->n; pToken->z[k] != '\0'; k++) { + if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; + if (pToken->z[k] == ',') { + pCxt->pSql = pTokenEnd; + *time = useconds; + return 0; + } + + break; + } + + /* + * time expression: + * e.g., now+12a, now-5h + */ + SToken valueToken; + index = 0; + sToken = tStrGetToken(pTokenEnd, &index, false); + pTokenEnd += index; + + if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) { + index = 0; + valueToken = tStrGetToken(pTokenEnd, &index, false); + pTokenEnd += index; + + if (valueToken.n < 2) { + return buildSyntaxErrMsg(&pCxt->msg, "value expected in timestamp", sToken.z); + } + + char unit = 0; + if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (sToken.type == TK_PLUS) { + useconds += interval; + } else { + useconds = useconds - interval; + } + + pCxt->pSql = pTokenEnd; + } + + *time = useconds; + return TSDB_CODE_SUCCESS; +} + +static int32_t parseOneColumn(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, bool primaryKey, int16_t timePrec, char* payload) { + int64_t iv; + int32_t ret; + char *endptr = NULL; + + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); + } + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { + if (isNullStr(pToken)) { + *((uint8_t *)payload) = TSDB_DATA_BOOL_NULL; + } else { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + *(uint8_t *)payload = TSDB_TRUE; + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + *(uint8_t *)payload = TSDB_FALSE; + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_INTEGER) { + iv = strtoll(pToken->z, NULL, 10); + *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); + } else if (pToken->type == TK_FLOAT) { + double dv = strtod(pToken->z, NULL); + *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + } + break; + } + case TSDB_DATA_TYPE_TINYINT: + if (isNullStr(pToken)) { + *((uint8_t *)payload) = TSDB_DATA_TINYINT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z); + } + *((uint8_t *)payload) = (uint8_t)iv; + } + break; + case TSDB_DATA_TYPE_UTINYINT: + if (isNullStr(pToken)) { + *((uint8_t *)payload) = TSDB_DATA_UTINYINT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); + } + *((uint8_t *)payload) = (uint8_t)iv; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + if (isNullStr(pToken)) { + *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); + } + *((int16_t *)payload) = (int16_t)iv; + } + break; + case TSDB_DATA_TYPE_USMALLINT: + if (isNullStr(pToken)) { + *((uint16_t *)payload) = TSDB_DATA_USMALLINT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); + } + *((uint16_t *)payload) = (uint16_t)iv; + } + break; + case TSDB_DATA_TYPE_INT: + if (isNullStr(pToken)) { + *((int32_t *)payload) = TSDB_DATA_INT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); + } + + *((int32_t *)payload) = (int32_t)iv; + } + + break; + + case TSDB_DATA_TYPE_UINT: + if (isNullStr(pToken)) { + *((uint32_t *)payload) = TSDB_DATA_UINT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); + } + + *((uint32_t *)payload) = (uint32_t)iv; + } + + break; + + case TSDB_DATA_TYPE_BIGINT: + if (isNullStr(pToken)) { + *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z); + } + + *((int64_t *)payload) = iv; + } + break; + + case TSDB_DATA_TYPE_UBIGINT: + if (isNullStr(pToken)) { + *((uint64_t *)payload) = TSDB_DATA_UBIGINT_NULL; + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z); + } + + *((uint64_t *)payload) = iv; + } + break; + + case TSDB_DATA_TYPE_FLOAT: + if (isNullStr(pToken)) { + *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; + } else { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + + SET_FLOAT_VAL(payload, dv); + } + break; + + case TSDB_DATA_TYPE_DOUBLE: + if (isNullStr(pToken)) { + *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; + } else { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + + *((double *)payload) = dv; + } + break; + + case TSDB_DATA_TYPE_BINARY: + // binary data cannot be null-terminated char string, otherwise the last char of the string is lost + if (pToken->type == TK_NULL) { + setVardataNull(payload, TSDB_DATA_TYPE_BINARY); + } else { // too long values will return invalid sql, not be truncated automatically + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { //todo refactor + return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); + } + + STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); + } + + break; + + case TSDB_DATA_TYPE_NCHAR: + if (pToken->type == TK_NULL) { + setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); + } else { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); + } + + varDataSetLen(payload, output); + } + break; + + case TSDB_DATA_TYPE_TIMESTAMP: { + if (pToken->type == TK_NULL) { + if (primaryKey) { + *((int64_t *)payload) = 0; + } else { + *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; + } + } else { + int64_t temp; + if (parseTime(pCxt, pToken, timePrec, &temp) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); + } + + *((int64_t *)payload) = temp; + } + + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t parseOneColumnKV(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, bool primaryKey, int16_t timePrec, + SMemRow row, int32_t toffset, int16_t colId, int32_t* dataLen, int32_t* kvLen, uint8_t compareStat) { + int64_t iv; + int32_t ret; + char * endptr = NULL; + + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); + } + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { // bool + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + appendMemRowColValEx(row, &TRUE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + appendMemRowColValEx(row, &FALSE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_INTEGER) { + iv = strtoll(pToken->z, NULL, 10); + appendMemRowColValEx(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, + dataLen, kvLen, compareStat); + } else if (pToken->type == TK_FLOAT) { + double dv = strtod(pToken->z, NULL); + appendMemRowColValEx(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, + dataLen, kvLen, compareStat); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + } + break; + } + + case TSDB_DATA_TYPE_TINYINT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z); + } + + uint8_t tmpVal = (uint8_t)iv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_UTINYINT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); + } + + uint8_t tmpVal = (uint8_t)iv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_SMALLINT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); + } + + int16_t tmpVal = (int16_t)iv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_USMALLINT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); + } + + uint16_t tmpVal = (uint16_t)iv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_INT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); + } + + int32_t tmpVal = (int32_t)iv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_UINT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); + } + + uint32_t tmpVal = (uint32_t)iv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + + case TSDB_DATA_TYPE_BIGINT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z); + } + + appendMemRowColValEx(row, &iv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_UBIGINT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); + if (ret != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z); + } + + uint64_t tmpVal = (uint64_t)iv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_FLOAT: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + + float tmpVal = (float)dv; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_DOUBLE: + if (isNullStr(pToken)) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + + appendMemRowColValEx(row, &dv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_BINARY: + // binary data cannot be null-terminated char string, otherwise the last char of the string is lost + if (pToken->type == TK_NULL) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { // too long values will return invalid sql, not be truncated automatically + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); + } + // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); + char *rowEnd = memRowEnd(row); + STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n); + appendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_NCHAR: + if (pToken->type == TK_NULL) { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } else { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + char * rowEnd = memRowEnd(row); + if (!taosMbsToUcs4(pToken->z, pToken->n, (char *)varDataVal(rowEnd), pSchema->bytes - VARSTR_HEADER_SIZE, + &output)) { + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); + } + varDataSetLen(rowEnd, output); + appendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + break; + + case TSDB_DATA_TYPE_TIMESTAMP: { + if (pToken->type == TK_NULL) { + if (primaryKey) { + // When building SKVRow primaryKey, we should not skip even with NULL value. + int64_t tmpVal = 0; + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } else { + appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, + compareStat); + } + } else { + int64_t tmpVal; + if (parseTime(pCxt, pToken, timePrec, &tmpVal) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); + } + appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + } + + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +// pSql -> tag1_name, ...) +static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { + int32_t nCols = pColList->numOfCols; + + pColList->numOfBound = 0; + memset(pColList->boundedColumns, 0, sizeof(int32_t) * nCols); + for (int32_t i = 0; i < nCols; ++i) { + pColList->cols[i].valStat = VAL_STAT_NONE; + } + + SToken sToken; + bool isOrdered = true; + int32_t lastColIdx = -1; // last column found + while (1) { + NEXT_TOKEN(pCxt->pSql, sToken); + + if (TK_RP == sToken.type) { + break; + } + + int32_t t = lastColIdx + 1; + int32_t index = findCol(&sToken, t, nCols, pSchema); + if (index < 0 && t > 0) { + index = findCol(&sToken, 0, t, pSchema); + isOrdered = false; + } + if (index < 0) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid column/tag name", sToken.z); + } + if (pColList->cols[index].valStat == VAL_STAT_HAS) { + return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z); + } + lastColIdx = index; + pColList->cols[index].valStat = VAL_STAT_HAS; + pColList->boundedColumns[pColList->numOfBound] = index; + ++pColList->numOfBound; + } + + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (!isOrdered) { + pColList->colIdxInfo = calloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + if (NULL == pColList->colIdxInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = pColList->colIdxInfo; + for (uint16_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = (uint16_t)pColList->boundedColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (uint16_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + memset(&pColList->boundedColumns[pColList->numOfBound], 0, sizeof(int32_t) * (pColList->numOfCols - pColList->numOfBound)); + + return TSDB_CODE_SUCCESS; +} + +// pSql -> tag1_value, ...) +static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pSpd, SSchema* pTagsSchema, uint8_t precision) { + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + destroyBoundColumnInfo(pSpd); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SToken sToken; + for (int i = 0; i < pSpd->numOfBound; ++i) { + SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]]; + + NEXT_TOKEN(pCxt->pSql, sToken); + + if (TK_ILLEGAL == sToken.type) { + tdDestroyKVRowBuilder(&kvRowBuilder); + destroyBoundColumnInfo(pSpd); + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } + + if (sToken.n == 0 || sToken.type == TK_RP) { + break; + } + + // Remove quotation marks + if (TK_STRING == sToken.type) { + sToken.z++; + sToken.n -= 2; + } + + char tagVal[TSDB_MAX_TAGS_LEN]; + CHECK_CODE_2(parseOneColumn(pCxt, &sToken, pSchema, false, precision, tagVal), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd)); + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); + } + + destroyBoundColumnInfo(pSpd); + + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + tdDestroyKVRowBuilder(&kvRowBuilder); + if (NULL == row) { + return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); + } + tdSortKVRowByColIdx(row); + + // pInsertParam->tagData.dataLen = kvRowLen(row); + // if (pInsertParam->tagData.dataLen <= 0){ + // return buildInvalidOperationMsg(msgBuf, "tag value expected"); + // } + + // char* pTag = realloc(pInsertParam->tagData.data, pInsertParam->tagData.dataLen); + // if (pTag == NULL) { + // return TSDB_CODE_TSC_OUT_OF_MEMORY; + // } + + // kvRowCpy(pTag, row); + tfree(row); + // pInsertParam->tagData.data = pTag; +} + +// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) +static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) { + SToken sToken; + + // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) + NEXT_TOKEN(pCxt->pSql, sToken); + CHECK_CODE(getTableMeta(pCxt, &sToken)); + if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { + return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + } + + SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); + SParsedDataColInfo spd = {0}; + setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); + + // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_LP == sToken.type) { + CHECK_CODE_1(parseBoundColumns(pCxt, &spd, pTagsSchema), destroyBoundColumnInfo(&spd)); + NEXT_TOKEN(pCxt->pSql, sToken); + } + + if (TK_TAGS != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z); + } + // pSql -> (tag1_value, ...) + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_LP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); + } + CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); + + return TSDB_CODE_SUCCESS; +} + +static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) { + int32_t index = 0; + SToken sToken = {0}; + + char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header + + SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; + STableMeta * pTableMeta = pDataBlocks->pTableMeta; + SSchema * schema = getTableColumnSchema(pTableMeta); + SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder; + int32_t dataLen = spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE; + int32_t kvLen = pBuilder->kvRowInitLen; + bool isParseBindParam = false; + + initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); + + // 1. set the parsed value from sql string + for (int i = 0; i < spd->numOfBound; ++i) { + // the start position in data block buffer of current value in sql + int32_t colIndex = spd->boundedColumns[i]; + + char *start = row + spd->cols[colIndex].offset; + + SSchema *pSchema = &schema[colIndex]; // get colId here + + NEXT_TOKEN(pCxt->pSql, sToken); + + // if (sToken.type == TK_QUESTION) { + // if (!isParseBindParam) { + // isParseBindParam = true; + // } + // if (pInsertParam->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { + // return buildSyntaxErrMsg(pInsertParam->msg, "? only allowed in binding insertion", *str); + // } + + // uint32_t offset = (uint32_t)(start - pDataBlocks->pData); + // if (tscAddParamToDataBlock(pDataBlocks, pSchema->type, (uint8_t)timePrec, pSchema->bytes, offset) != NULL) { + // continue; + // } + + // strcpy(pInsertParam->msg, "client out of memory"); + // return TSDB_CODE_TSC_OUT_OF_MEMORY; + // } + + int16_t type = sToken.type; + if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && + type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || + (sToken.n == 0) || (type == TK_RP)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", sToken.z); + } + + // Remove quotation marks + if (TK_STRING == sToken.type) { + // delete escape character: \\, \', \" + char delim = sToken.z[0]; + + int32_t cnt = 0; + int32_t j = 0; + if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) { + return buildSyntaxErrMsg(&pCxt->msg, "too long string", sToken.z); + } + + for (uint32_t k = 1; k < sToken.n - 1; ++k) { + if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { + tmpTokenBuf[j] = sToken.z[k + 1]; + + cnt++; + j++; + k++; + continue; + } + + tmpTokenBuf[j] = sToken.z[k]; + j++; + } + + tmpTokenBuf[j] = 0; + sToken.z = tmpTokenBuf; + sToken.n -= 2 + cnt; + } + + bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_ID); + int32_t toffset = -1; + int16_t colId = -1; + getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); + + int32_t ret = parseOneColumnKV(pCxt, &sToken, pSchema, isPrimaryKey, timePrec, row, toffset, + colId, &dataLen, &kvLen, pBuilder->compareStat); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + if (isPrimaryKey) { + TSKEY tsKey = memRowKey(row); + if (checkTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) { + buildSyntaxErrMsg(&pCxt->msg, "client time/server time can not be mixed up", sToken.z); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + } + + if (!isParseBindParam) { + // 2. check and set convert flag + if (pBuilder->compareStat == ROW_COMPARE_NEED) { + checkAndConvertMemRow(row, dataLen, kvLen); + } + + // 3. set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) { + SDataRow dataRow = memRowDataBody(row); + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { + tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset); + } + } + } + } + + *len = getExtendedRowSize(pDataBlocks); + + return TSDB_CODE_SUCCESS; +} + +// pSql -> (field1_value, ...) [(field1_value2, ...) ...] +static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows, char* tmpTokenBuf) { + STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + CHECK_CODE(initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound, pDataBlock->boundColumnInfo.allNullLen)); + + (*numOfRows) = 0; + SToken sToken; + while (1) { + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_LP != sToken.type) { + break; + } + + if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { + int32_t tSize; + CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize)); + ASSERT(tSize >= maxRows); + maxRows = tSize; + } + + int32_t len = 0; + CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &len, tmpTokenBuf)); + pDataBlock->size += len; + + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_RP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z); + } + + (*numOfRows)++; + } + + if (0 == (*numOfRows)) { + return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) { + int32_t maxNumOfRows; + CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows)); + + int32_t numOfRows = 0; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows, tmpTokenBuf)); + + for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { + SParamInfo *param = dataBuf->params + i; + if (param->idx == -1) { + // param->idx = pInsertParam->numOfParams++; + param->offset -= sizeof(SSubmitBlk); + } + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows)) { + return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767"); + } + + dataBuf->numOfTables = 1; + pCxt->totalNum += numOfRows; + return TSDB_CODE_SUCCESS; +} + +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +static int32_t parseInsertBody(SInsertParseContext* pCxt) { + while (1) { + SToken sToken; + // pSql -> tb_name ... + NEXT_TOKEN(pCxt->pSql, sToken); + + // no data in the sql string anymore. + if (sToken.n == 0) { + if (0 == pCxt->totalNum) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + break; + } + + SToken tbnameToken = sToken; + + NEXT_TOKEN(pCxt->pSql, sToken); + + // USING cluase + if (TK_USING == sToken.type) { + CHECK_CODE(parseUsingClause(pCxt, &tbnameToken)); + NEXT_TOKEN(pCxt->pSql, sToken); + } else { + CHECK_CODE(getTableMeta(pCxt, &sToken)); + } + + STableDataBlocks *dataBuf = NULL; + CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, NULL/* tbname */, pCxt->pTableMeta, &dataBuf, NULL)); + + if (TK_LP == sToken.type) { + // pSql -> field1_name, ...) + NEXT_TOKEN(pCxt->pSql, sToken); + // todo col_list + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_RP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); + } + NEXT_TOKEN(pCxt->pSql, sToken); + } + + if (TK_VALUES == sToken.type) { + // pSql -> (field1_value, ...) [(field1_value2, ...) ...] + CHECK_CODE(parseValuesClause(pCxt, dataBuf)); + continue; + } + + // FILE csv_file_path + if (TK_FILE == sToken.type) { + // pSql -> csv_file_path + NEXT_TOKEN(pCxt->pSql, sToken); + + if (0 == sToken.n || (TK_STRING != sToken.type && TK_ID != sToken.type)) { + return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); + } + + // todo + continue; + } + + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); + } + // merge according to vgId + if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, true)); + } + return TSDB_CODE_SUCCESS; +} + +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { + CHECK_CODE(createInsertStmtInfo(pInfo)); + + SInsertParseContext context = { + .pComCxt = pContext, + .pSql = pContext->pSql, + .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, + .pCatalog = getCatalogHandle(pContext->pEpSet), + .pTableMeta = NULL, + .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), + .totalNum = 0, + .pOutput = *pInfo + }; + + if (NULL == context.pTableBlockHashObj) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + CHECK_CODE(skipInsertInto(&context)); + CHECK_CODE(parseInsertBody(&context)); + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 29561f7f54..a8642e2535 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -18,6 +18,7 @@ #include "parserUtil.h" #include "ttoken.h" #include "function.h" +#include "insertParser.h" bool qIsInsertSql(const char* pStr, size_t length) { int32_t index = 0; @@ -46,8 +47,8 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); } -int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen) { - return 0; +int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { + return parseInsertSql(pContext, pInfo); } int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) { @@ -173,7 +174,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet assert(t != NULL); if (t->n >= TSDB_FUNC_NAME_LEN) { - return buildSyntaxErrMsg(msg, msgBufLen, "too long function name", t->z); + return buildSyntaxErrMsg(&msgBuf, "too long function name", t->z); } // Let's assume that it is an UDF/UDAF, if it is not a built-in function. diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index e98443e2a1..d81e2ffe0f 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "parserUtil.h" + #include "taosmsg.h" #include "parser.h" #include "taoserror.h" @@ -18,7 +34,6 @@ typedef struct STableFilterCond { static STableMetaInfo* addTableMetaInfo(SQueryStmtInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); -STableMeta* tableMetaDup(STableMeta* pTableMeta); int32_t parserValidateIdToken(SToken* pToken) { if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) { @@ -87,7 +102,7 @@ int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { return TSDB_CODE_TSC_INVALID_OPERATION; } -int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr) { +int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { const char* msgFormat1 = "syntax error near \'%s\'"; const char* msgFormat2 = "syntax error near \'%s\' (%s)"; const char* msgFormat3 = "%s"; @@ -95,7 +110,7 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn const char* prefix = "syntax error"; if (sourceStr == NULL) { assert(additionalInfo != NULL); - snprintf(dst, dstBufLen, msgFormat1, additionalInfo); + snprintf(pBuf->buf, pBuf->len, msgFormat1, additionalInfo); return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } @@ -103,10 +118,10 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn strncpy(buf, sourceStr, tListLen(buf) - 1); if (additionalInfo != NULL) { - snprintf(dst, dstBufLen, msgFormat2, buf, additionalInfo); + snprintf(pBuf->buf, pBuf->len, msgFormat2, buf, additionalInfo); } else { const char* msgFormat = (0 == strncmp(sourceStr, prefix, strlen(prefix))) ? msgFormat3 : msgFormat1; - snprintf(dst, dstBufLen, msgFormat, buf); + snprintf(pBuf->buf, pBuf->len, msgFormat, buf); } return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; @@ -1490,7 +1505,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { return pTableMeta; } -uint32_t getTableMetaSize(STableMeta* pTableMeta) { +uint32_t getTableMetaSize(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); int32_t totalCols = 0; @@ -1505,7 +1520,7 @@ uint32_t getTableMetaMaxSize() { return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); } -STableMeta* tableMetaDup(STableMeta* pTableMeta) { +STableMeta* tableMetaDup(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); size_t size = getTableMetaSize(pTableMeta); diff --git a/source/libs/parser/src/ttokenizer.c b/source/libs/parser/src/ttokenizer.c index 683fe142ef..d69fa2b95b 100644 --- a/source/libs/parser/src/ttokenizer.c +++ b/source/libs/parser/src/ttokenizer.c @@ -284,7 +284,7 @@ static int32_t tKeywordCode(const char* z, int n) { * Return the length of the token that begins at z[0]. * Store the token type in *type before returning. */ -uint32_t tGetToken(char* z, uint32_t* tokenId) { +uint32_t tGetToken(const char* z, uint32_t* tokenId) { uint32_t i; switch (*z) { case ' ': @@ -595,7 +595,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { return ntoken; } -SToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { +SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { SToken t0 = {0}; // here we reach the end of sql string, null-terminated string @@ -689,13 +689,12 @@ void taosCleanupKeywordsTable() { } SToken taosTokenDup(SToken* pToken, char* buf, int32_t len) { - assert(pToken != NULL && buf != NULL); + assert(pToken != NULL && buf != NULL && len > pToken->n); + + strncpy(buf, pToken->z, pToken->n); + buf[pToken->n] = 0; + SToken token = *pToken; token.z = buf; - - assert(len > token.n); - strncpy(token.z, pToken->z, pToken->n); - token.z[token.n] = 0; - return token; } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 8054dc42be..6063f81682 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -37,7 +37,7 @@ char *taosCharsetReplace(char *charsetstr) { return strdup(charsetstr); } -int64_t taosStr2int64(char *str) { +int64_t taosStr2int64(const char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); } @@ -107,7 +107,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { return len; } -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { +bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) { memset(ucs4, 0, ucs4_max_len); mbstate_t state = {0}; int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index b9d2da6939..46deaf0259 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -166,7 +166,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { return split; } -char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) { +char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) { for (int32_t i = 0; i < len; ++i) { // skip the needle in quote, jump to the end of quoted string @@ -179,7 +179,7 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) { } if (haystack[i] == needle) { - return &haystack[i]; + return (char *)&haystack[i]; } } From 0f5dc110957a7f075ee5683742af85882dd7b575 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 9 Dec 2021 21:14:42 -0500 Subject: [PATCH 02/22] TD-11819 Parsing insert statement and assembling binary objects. --- include/libs/parser/parser.h | 10 ++++++++-- source/libs/parser/src/dataBlockMgt.c | 15 ++++----------- source/libs/parser/src/insertParser.c | 1 - 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index ea8caa691d..e2817baeae 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -158,9 +158,15 @@ typedef enum { PAYLOAD_TYPE_RAW = 1, } EPayloadType; +typedef struct SVgDataBlocks { + int64_t vgId; // virtual group id + int32_t numOfTables; // number of tables in current submit block + uint32_t size; + char *pData; +} SVgDataBlocks; + typedef struct SInsertStmtInfo { - // SHashObj* pTableBlockHashList; // data block for each table - SArray* pDataBlocks; // SArray. Merged submit block for each vgroup + SArray* pDataBlocks; // data block for each vgroup, SArray. int8_t schemaAttache; // denote if submit block is built with table schema or not uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint32_t insertType; // insert data from [file|sql statement| bound statement] diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index a08f4fb15b..e8bf9da793 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -507,22 +507,16 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); STableDataBlocks** p = taosHashIterate(pHashObj, NULL); - STableDataBlocks* pOneTableBlock = *p; - SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - - while(pOneTableBlock) { + while (pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; STableDataBlocks* dataBuf = NULL; - int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { @@ -532,22 +526,21 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t return ret; } + // the maximum expanded size in byte when a row-wise data is converted to SDataRow format + int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); if (tmp != NULL) { dataBuf->pData = tmp; - //memset(dataBuf->pData + dataBuf->size, 0, dataBuf->nAllocSize - dataBuf->size); } else { // failed to allocate memory, free already allocated memory and return error code taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); tfree(dataBuf->pData); tfree(blkKeyInfo.pKeyTuple); - return TSDB_CODE_TSC_OUT_OF_MEMORY; } } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 61f92eebe0..1748588427 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -233,7 +233,6 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { pDataBlocks->ordered = false; - // tscWarn("NOT ordered input timestamp"); } pDataBlocks->prevTS = k; From 2164b8ef20692d55251a886639feb581964982a5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 10 Dec 2021 15:20:04 +0800 Subject: [PATCH 03/22] TD-10431 create stable --- include/common/taosmsg.h | 31 +- include/dnode/mnode/sdb/sdb.h | 2 +- include/util/taoserror.h | 14 +- source/dnode/mgmt/impl/src/dndTransport.c | 20 +- source/dnode/mnode/impl/inc/mndDef.h | 7 +- source/dnode/mnode/impl/inc/mndStable.h | 4 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndStable.c | 421 ++++++++++++++-------- source/dnode/mnode/impl/src/mnode.c | 2 +- source/libs/parser/inc/astGenerator.h | 2 +- source/libs/parser/inc/sql.y | 4 +- source/libs/parser/src/astGenerator.c | 2 +- source/libs/parser/src/sql.c | 4 +- source/libs/transport/src/rpcMain.c | 2 +- source/util/src/terror.c | 4 +- src/client/inc/tscParseLine.h | 2 +- 16 files changed, 328 insertions(+), 195 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 2ce6da9806..5fbbbb2855 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -74,10 +74,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_DB, "compact-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE, "create-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STABLE_VGROUP, "stable-vgroup" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STB_VGROUP, "stb-vgroup" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) @@ -94,9 +94,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) // message from vnode to dnode // message from mnode to vnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE_IN, "create-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE_IN, "alter-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE_IN, "drop-stable" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-in" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-in" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-in" ) // message from mnode to mnode // message from mnode to qnode // message from mnode to dnode @@ -159,7 +159,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_DNODE, TSDB_MGMT_TABLE_MNODE, TSDB_MGMT_TABLE_VGROUP, - TSDB_MGMT_TABLE_STABLE, + TSDB_MGMT_TABLE_STB, TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, @@ -294,7 +294,7 @@ typedef struct { uint64_t superTableUid; uint64_t createdTime; char tableFname[TSDB_TABLE_FNAME_LEN]; - char stableFname[TSDB_TABLE_FNAME_LEN]; + char stbFname[TSDB_TABLE_FNAME_LEN]; char data[]; } SMDCreateTableMsg; @@ -311,9 +311,12 @@ typedef struct { } SCreateTableMsg; typedef struct { - int32_t numOfTables; - int32_t contLen; -} SCMCreateTableMsg; + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igExists; + int32_t numOfTags; + int32_t numOfColumns; + SSchema pSchema[]; +} SCreateStbMsg; typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -765,7 +768,7 @@ typedef struct { typedef struct { char name[TSDB_TABLE_FNAME_LEN]; -} SStableInfoMsg; +} SStbInfoMsg; typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; @@ -798,7 +801,7 @@ typedef struct { typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; // table id - char stableFname[TSDB_TABLE_FNAME_LEN]; + char stbFname[TSDB_TABLE_FNAME_LEN]; int32_t numOfTags; int32_t numOfColumns; int8_t precision; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d1f7d9a958..36b1e41978 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -159,7 +159,7 @@ typedef enum { SDB_AUTH = 6, SDB_ACCT = 7, SDB_VGROUP = 9, - SDB_STABLE = 9, + SDB_STB = 9, SDB_DB = 10, SDB_FUNC = 11, SDB_MAX = 12 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 36301466f8..1232e2e31f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -183,8 +183,18 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0348) //"Mnode already exists") #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0349) //"Mnode not there") -// mnode-table -#define TSDB_CODE_MND_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) //"Table already exists") +// mnode-stable +#define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_IGEXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COLS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_TYPE TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_ID TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_BYTES TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_STB_INVALID_COL_NAME TAOS_DEF_ERROR_CODE(0, 0x0360) + + #define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") #define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") #define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) //"Invalid table type in tsdb") diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index a36c56ea3d..09c207bac7 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -69,10 +69,10 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_RETRIEVE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_STB_VGROUP] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; @@ -84,12 +84,12 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq; // message from mnode to vnode - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB_IN_RSP] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB_IN_RSP] = dndProcessMnodeWriteMsg; // message from mnode to dnode pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_VNODE_IN] = dndProcessVnodeMgmtMsg; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7a7bc161da..b2e3ecff55 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -241,7 +241,7 @@ typedef struct SVgObj { SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; -typedef struct SStableObj { +typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_FULL_DB_NAME_LEN]; int64_t createdTime; @@ -251,9 +251,8 @@ typedef struct SStableObj { int32_t numOfColumns; int32_t numOfTags; SRWLatch lock; - SSchema *columnSchema; - SSchema *tagSchema; -} SStableObj; + SSchema *pSchema; +} SStbObj; typedef struct SFuncObj { char name[TSDB_FUNC_NAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndStable.h b/source/dnode/mnode/impl/inc/mndStable.h index c7767a59e4..71d9483044 100644 --- a/source/dnode/mnode/impl/inc/mndStable.h +++ b/source/dnode/mnode/impl/inc/mndStable.h @@ -22,8 +22,8 @@ extern "C" { #endif -int32_t mndInitStable(SMnode *pMnode); -void mndCleanupStable(SMnode *pMnode); +int32_t mndInitStb(SMnode *pMnode); +void mndCleanupStb(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index cfecaf2775..a610eaeeb9 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -270,7 +270,7 @@ char *mndShowStr(int32_t showType) { return "show mnodes"; case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; - case TSDB_MGMT_TABLE_STABLE: + case TSDB_MGMT_TABLE_STB: return "show stables"; case TSDB_MGMT_TABLE_MODULE: return "show modules"; diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c index b57b05f299..aaa44a14c1 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStable.c @@ -15,62 +15,62 @@ #define _DEFAULT_SOURCE #include "mndStable.h" +#include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" -#include "mndDb.h" #include "tname.h" -#define TSDB_STABLE_VER_NUM 1 -#define TSDB_STABLE_RESERVE_SIZE 64 +#define TSDB_STB_VER_NUM 1 +#define TSDB_STB_RESERVE_SIZE 64 -static SSdbRaw *mndStableActionEncode(SStableObj *pStb); -static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw); -static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb); -static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb); -static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb); -static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg); -static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg); -static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static void mndCancelGetNextStable(SMnode *pMnode, void *pIter); +static SSdbRaw *mndStbActionEncode(SStbObj *pStb); +static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); +static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); +static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); +static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb); +static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg); +static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); -int32_t mndInitStable(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_STABLE, +int32_t mndInitStb(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_STB, .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndStableActionEncode, - .decodeFp = (SdbDecodeFp)mndStableActionDecode, - .insertFp = (SdbInsertFp)mndStableActionInsert, - .updateFp = (SdbUpdateFp)mndStableActionUpdate, - .deleteFp = (SdbDeleteFp)mndStableActionDelete}; + .encodeFp = (SdbEncodeFp)mndStbActionEncode, + .decodeFp = (SdbDecodeFp)mndStbActionDecode, + .insertFp = (SdbInsertFp)mndStbActionInsert, + .updateFp = (SdbUpdateFp)mndStbActionUpdate, + .deleteFp = (SdbDeleteFp)mndStbActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE, mndProcessCreateStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE, mndProcessAlterStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE, mndProcessDropStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP, mndProcessCreateStableInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP, mndProcessAlterStableInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE_IN_RSP, mndProcessDropStableInRsp); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStableMetaMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB, mndProcessCreateStbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB, mndProcessAlterStbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB, mndProcessDropStbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STB_IN_RSP, mndProcessCreateStbInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STB_IN_RSP, mndProcessAlterStbInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STB_IN_RSP, mndProcessDropStbInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStbMetaMsg); - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndGetStableMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndRetrieveStables); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndCancelGetNextStable); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb); return sdbSetTable(pMnode->pSdb, table); } -void mndCleanupStable(SMnode *pMnode) {} +void mndCleanupStb(SMnode *pMnode) {} -static SSdbRaw *mndStableActionEncode(SStableObj *pStb) { - int32_t size = sizeof(SStableObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); - SSdbRaw *pRaw = sdbAllocRaw(SDB_STABLE, TSDB_STABLE_VER_NUM, size); +static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -82,41 +82,34 @@ static SSdbRaw *mndStableActionEncode(SStableObj *pStb) { SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pStb->columnSchema[i]; + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pStb->pSchema[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type); SDB_SET_INT32(pRaw, dataPos, pSchema->colId); SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); } - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pStb->tagSchema[i]; - SDB_SET_INT8(pRaw, dataPos, pSchema->type); - SDB_SET_INT32(pRaw, dataPos, pSchema->colId); - SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); - SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); - } - - SDB_SET_RESERVE(pRaw, dataPos, TSDB_STABLE_RESERVE_SIZE) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; } -static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { +static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_STABLE_VER_NUM) { + if (sver != TSDB_STB_VER_NUM) { mError("failed to decode stable since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; } - int32_t size = sizeof(SStableObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); - SSdbRow *pRow = sdbAllocRow(size); - SStableObj *pStb = sdbGetRowObj(pRow); + int32_t size = sizeof(SStbObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); + SSdbRow *pRow = sdbAllocRow(size); + SStbObj *pStb = sdbGetRowObj(pRow); if (pStb == NULL) return NULL; int32_t dataPos = 0; @@ -128,87 +121,61 @@ static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns) SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags) - pStb->columnSchema = calloc(pStb->numOfColumns, sizeof(SSchema)); - pStb->tagSchema = calloc(pStb->numOfTags, sizeof(SSchema)); + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + pStb->pSchema = calloc(totalCols, sizeof(SSchema)); - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pStb->columnSchema[i]; + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pStb->pSchema[i]; SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); } - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pStb->tagSchema[i]; - SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); - SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); - SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); - SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); - } - - SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STABLE_RESERVE_SIZE) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STB_RESERVE_SIZE) return pRow; } -static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb) { - mTrace("stable:%s, perform insert action", pStb->name); +static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { + mTrace("stb:%s, perform insert action", pStb->name); return 0; } -static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb) { - mTrace("stable:%s, perform delete action", pStb->name); +static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { + mTrace("stb:%s, perform delete action", pStb->name); return 0; } -static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb) { - mTrace("stable:%s, perform update action", pOldStb->name); +static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb) { + mTrace("stb:%s, perform update action", pOldStb->name); atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime); atomic_exchange_32(&pOldStb->version, pNewStb->version); taosWLockLatch(&pOldStb->lock); - int32_t numOfTags = pNewStb->numOfTags; - int32_t tagSize = numOfTags * sizeof(SSchema); - int32_t numOfColumns = pNewStb->numOfColumns; - int32_t columnSize = numOfColumns * sizeof(SSchema); + int32_t totalCols = pNewStb->numOfTags + pNewStb->numOfColumns; + int32_t totalSize = totalCols * sizeof(SSchema); - if (pOldStb->numOfTags < numOfTags) { - pOldStb->tagSchema = malloc(tagSize); - } - if (pOldStb->numOfColumns < numOfColumns) { - pOldStb->columnSchema = malloc(columnSize); + if (pOldStb->numOfTags + pOldStb->numOfColumns < totalCols) { + pOldStb->pSchema = malloc(totalSize); } - memcpy(pOldStb->tagSchema, pNewStb->tagSchema, tagSize); - memcpy(pOldStb->columnSchema, pNewStb->columnSchema, columnSize); + memcpy(pOldStb->pSchema, pNewStb->pSchema, totalSize); taosWUnLockLatch(&pOldStb->lock); return 0; } -SStableObj *mndAcquireStb(SMnode *pMnode, char *stbName) { +SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) { SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_STABLE, stbName); + return sdbAcquire(pSdb, SDB_STB, stbName); } -void mndReleaseStb(SMnode *pMnode, SStableObj *pStb) { +void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pStb); } -static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg) { return 0; } - -static SDbObj *mndGetDbByStbName(SMnode *pMnode, char *stbName) { +static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) { SName name = {0}; tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -218,24 +185,187 @@ static SDbObj *mndGetDbByStbName(SMnode *pMnode, char *stbName) { return mndAcquireDb(pMnode, db); } -static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SStableInfoMsg *pInfo = pMsg->rpcMsg.pCont; +static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { + pCreate->numOfColumns = htonl(pCreate->numOfColumns); + pCreate->numOfTags = htonl(pCreate->numOfTags); + int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags; + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pCreate->pSchema[i]; + pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = htonl(pSchema->bytes); + } - mDebug("stable:%s, start to retrieve meta", pInfo->name); - - SDbObj *pDb = mndGetDbByStbName(pMnode, pInfo->name); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stable:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); + if (pCreate->igExists < 0 || pCreate->igExists > 1) { + terrno = TSDB_CODE_MND_STB_INVALID_IGEXIST; return -1; } - SStableObj *pStb = mndAcquireStb(pMnode, pInfo->name); + if (pCreate->numOfColumns < TSDB_MIN_COLUMNS || pCreate->numOfColumns > TSDB_MAX_COLUMNS) { + terrno = TSDB_CODE_MND_STB_INVALID_COLS_NUM; + return -1; + } + + if (pCreate->numOfTags <= 0 || pCreate->numOfTags > TSDB_MAX_TAGS) { + terrno = TSDB_CODE_MND_STB_INVALID_TAGS_NUM; + return -1; + } + + int32_t maxColId = (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS); + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pCreate->pSchema[i]; + if (pSchema->type <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + return -1; + } + if (pSchema->colId < 0 || pSchema->colId >= maxColId) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + return -1; + } + if (pSchema->bytes <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + return -1; + } + if (pSchema->name[0] == 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + return -1; + } + } + + return 0; +} + +static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) { + SStbObj stbObj = {0}; + tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); + tstrncpy(stbObj.db, pDb->name, TSDB_FULL_DB_NAME_LEN); + stbObj.createdTime = taosGetTimestampMs(); + stbObj.updateTime = stbObj.createdTime; + stbObj.uid = 1234; + stbObj.version = 1; + stbObj.numOfColumns = pCreate->numOfColumns; + stbObj.numOfTags = pCreate->numOfTags; + + int32_t totalCols = stbObj.numOfColumns + stbObj.numOfTags; + int32_t totalSize = totalCols * sizeof(SSchema); + stbObj.pSchema = malloc(totalSize); + if (stbObj.pSchema == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(stbObj.pSchema, pCreate->pSchema, totalSize); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); + + SSdbRaw *pRedoRaw = mndStbActionEncode(&stbObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndStbActionEncode(&stbObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateStbMsg *pCreate = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to create", pCreate->name); + + if (mndCheckStbMsg(pCreate) != 0) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); + if (pStb != NULL) { + sdbRelease(pMnode->pSdb, pStb); + if (pCreate->igExists) { + mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); + return 0; + } else { + terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; + mError("db:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } + + SDbObj *pDb = mndAcquireDbByStb(pMnode, pCreate->name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + int32_t code = mndCreateStb(pMnode, pMsg, pCreate, pDb); + mndReleaseDb(pMnode, pDb); + + if (code != 0) { + terrno = code; + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SStbInfoMsg *pInfo = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to retrieve meta", pInfo->name); + + SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, pInfo->name); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; - mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); + mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); return -1; } @@ -243,11 +373,11 @@ static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { STableMetaMsg *pMeta = rpcMallocCont(contLen); if (pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); + mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); return -1; } - memcpy(pMeta->stableFname, pStb->name, TSDB_TABLE_FNAME_LEN); + memcpy(pMeta->stbFname, pStb->name, TSDB_TABLE_FNAME_LEN); pMeta->numOfTags = htonl(pStb->numOfTags); pMeta->numOfColumns = htonl(pStb->numOfColumns); pMeta->precision = pDb->cfg.precision; @@ -258,30 +388,21 @@ static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pMeta->pSchema[i]; - SSchema *pColumn = &pStb->columnSchema[i]; - memcpy(pSchema->name, pColumn->name, TSDB_COL_NAME_LEN); - pSchema->type = pColumn->type; - pSchema->colId = htonl(pColumn->colId); - pSchema->bytes = htonl(pColumn->bytes); - } - - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns]; - SSchema *pTag = &pStb->tagSchema[i]; - memcpy(pSchema->name, pTag->name, TSDB_COL_NAME_LEN); - pSchema->type = pTag->type; - pSchema->colId = htons(pTag->colId); - pSchema->bytes = htonl(pTag->bytes); + SSchema *pSrcSchema = &pStb->pSchema[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); } pMsg->pCont = pMeta; pMsg->contLen = contLen; - mDebug("stable:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); + mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); return 0; } -static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfStables) { +static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); @@ -290,29 +411,29 @@ static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfS return -1; } - int32_t numOfStables = 0; + int32_t numOfStbs = 0; void *pIter = NULL; while (1) { - SStableObj *pStb = NULL; + SStbObj *pStb = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb); if (pIter == NULL) break; if (strcmp(pStb->db, dbName) == 0) { - numOfStables++; + numOfStbs++; } sdbRelease(pSdb, pStb); } - *pNumOfStables = numOfStables; + *pNumOfStbs = numOfStbs; return 0; } -static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; - if (mndGetNumOfStables(pMnode, pShow->db, &pShow->numOfRows) != 0) { + if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) { return -1; } @@ -357,7 +478,7 @@ static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg return 0; } -static void mnodeExtractTableName(char* tableId, char* name) { +static void mnodeExtractTableName(char *tableId, char *name) { int pos = -1; int num = 0; for (pos = 0; tableId[pos] != 0; ++pos) { @@ -370,21 +491,21 @@ static void mnodeExtractTableName(char* tableId, char* name) { } } -static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SStableObj *pStb = NULL; - int32_t cols = 0; - char *pWrite; - char prefix[64] = {0}; +static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SStbObj *pStb = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[64] = {0}; tstrncpy(prefix, pShow->db, 64); strcat(prefix, TS_PATH_DELIMITER); int32_t prefixLen = (int32_t)strlen(prefix); while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_STABLE, pShow->pIter, (void **)&pStb); + pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); if (pShow->pIter == NULL) break; if (strncmp(pStb->name, prefix, prefixLen) != 0) { @@ -394,10 +515,10 @@ static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, cols = 0; - char stableName[TSDB_TABLE_FNAME_LEN] = {0}; - memcpy(stableName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); + char stbName[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(stbName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, stableName); + STR_TO_VARSTR(pWrite, stbName); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -421,7 +542,7 @@ static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, return numOfRows; } -static void mndCancelGetNextStable(SMnode *pMnode, void *pIter) { +static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3e0ac746fc..a13c0bb5af 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -131,7 +131,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-stable", mndInitStable, mndCleanupStable) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (pMnode->clusterId <= 0) { if (mndAllocStep(pMnode, "mnode-sdb-deploy", mndDeploySdb, NULL) != 0) return -1; diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index cb3d459de6..954bc29e62 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -121,7 +121,7 @@ typedef struct SRelationInfo { typedef struct SCreatedTableInfo { SToken name; // table name token - SToken stableName; // super table name token , for using clause + SToken stbName; // super table name token , for using clause SArray *pTagNames; // create by using super table, tag name SArray *pTagVals; // create by using super table, tag value char *fullname; // table full name diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index ca281c9f1c..da975b966e 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -112,13 +112,13 @@ cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { } cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &X, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &X, 0); } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SToken token; tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &Y); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index 3b7d1cbc29..8d357f2fe5 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -634,7 +634,7 @@ SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, info.name = *pToken; info.pTagNames = pTagNames; info.pTagVals = pTagVals; - info.stableName = *pTableName; + info.stbName = *pTableName; info.igExist = (igExists->n > 0)? 1:0; return info; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2d6ec3f864..53a7ba48f8 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2312,14 +2312,14 @@ static void yy_reduce( break; case 26: /* cmd ::= SHOW dbPrefix STABLES */ { - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &yymsp[-1].minor.yy0, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &yymsp[-1].minor.yy0, 0); } break; case 27: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SToken token; tSetDbName(&token, &yymsp[-3].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &yymsp[0].minor.yy0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STB, &token, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= SHOW dbPrefix VGROUPS */ diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 911e8472ab..ce110ede32 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -406,7 +406,7 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_SHOW_RETRIEVE - || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_STABLE_VGROUP + || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_STB_VGROUP || type == TSDB_MSG_TYPE_TABLES_META || type == TSDB_MSG_TYPE_TABLE_META || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_STATUS || type == TSDB_MSG_TYPE_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 5110c8ba22..b0960ed3b0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -193,8 +193,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST, "Table already exists") +// mnode-stable +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") diff --git a/src/client/inc/tscParseLine.h b/src/client/inc/tscParseLine.h index 401dcafdfb..e7c5e54841 100644 --- a/src/client/inc/tscParseLine.h +++ b/src/client/inc/tscParseLine.h @@ -28,7 +28,7 @@ typedef struct { } TAOS_SML_KV; typedef struct { - char* stableName; + char* stbName; char* childTableName; TAOS_SML_KV* tags; From 9c1827f93dff3687da7465c4393c0c2aee6f5a36 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 10 Dec 2021 15:21:34 +0800 Subject: [PATCH 04/22] TD-10431 rename file --- source/dnode/mnode/impl/inc/{mndStable.h => mndStb.h} | 6 +++--- source/dnode/mnode/impl/src/{mndStable.c => mndStb.c} | 2 +- source/dnode/mnode/impl/src/mnode.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename source/dnode/mnode/impl/inc/{mndStable.h => mndStb.h} (90%) rename source/dnode/mnode/impl/src/{mndStable.c => mndStb.c} (99%) diff --git a/source/dnode/mnode/impl/inc/mndStable.h b/source/dnode/mnode/impl/inc/mndStb.h similarity index 90% rename from source/dnode/mnode/impl/inc/mndStable.h rename to source/dnode/mnode/impl/inc/mndStb.h index 71d9483044..58cae73c7f 100644 --- a/source/dnode/mnode/impl/inc/mndStable.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_STABLE_H_ -#define _TD_MND_STABLE_H_ +#ifndef _TD_MND_STB_H_ +#define _TD_MND_STB_H_ #include "mndInt.h" @@ -29,4 +29,4 @@ void mndCleanupStb(SMnode *pMnode); } #endif -#endif /*_TD_MND_STABLE_H_*/ +#endif /*_TD_MND_STB_H_*/ diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStb.c similarity index 99% rename from source/dnode/mnode/impl/src/mndStable.c rename to source/dnode/mnode/impl/src/mndStb.c index aaa44a14c1..9f9308b2fa 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "mndStable.h" +#include "mndStb.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index a13c0bb5af..c9310809d8 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -24,7 +24,7 @@ #include "mndMnode.h" #include "mndProfile.h" #include "mndShow.h" -#include "mndStable.h" +#include "mndStb.h" #include "mndSync.h" #include "mndTelem.h" #include "mndTrans.h" From 77b8bdd56b5933518c1894e7cfa859a874c94a26 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 11 Dec 2021 00:35:26 -0500 Subject: [PATCH 05/22] TD-11819 Parsing insert statement and assembling binary objects. --- include/common/tdataformat.h | 2 +- source/libs/parser/inc/dataBlockMgt.h | 33 +- source/libs/parser/src/dataBlockMgt.c | 39 +- source/libs/parser/src/insertParser.c | 834 +++++++------------------- 4 files changed, 252 insertions(+), 656 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index a3de2452ac..c6ef6c513f 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -544,7 +544,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) { +static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx* pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index d163d0b741..350610ec06 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -42,7 +42,7 @@ typedef enum ERowCompareStat { typedef struct SBoundColumn { int32_t offset; // all column offset value int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future - uint8_t valStat; // denote if current column bound or not(0 means has val, 1 means no val) + uint8_t valStat; // EValStat. denote if current column bound or not(0 means has val, 1 means no val) } SBoundColumn; typedef struct { @@ -63,15 +63,7 @@ typedef struct SParsedDataColInfo { int8_t orderStatus; // bound columns } SParsedDataColInfo; -typedef struct SParamInfo { - int32_t idx; - uint8_t type; - uint8_t timePrec; - int16_t bytes; - uint32_t offset; -} SParamInfo; - -typedef struct { +typedef struct SMemRowInfo { int32_t dataLen; // len of SDataRow int32_t kvLen; // len of SKVRow } SMemRowInfo; @@ -83,15 +75,13 @@ typedef struct { SMemRowInfo *rowInfo; } SMemRowBuilder; -typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; -} SBlockKeyTuple; - -typedef struct SBlockKeyInfo { - int32_t maxBytesAlloc; - SBlockKeyTuple* pKeyTuple; -} SBlockKeyInfo; +typedef struct SParamInfo { + int32_t idx; + uint8_t type; + uint8_t timePrec; + int16_t bytes; + uint32_t offset; +} SParamInfo; typedef struct STableDataBlocks { SName tableName; @@ -147,7 +137,7 @@ static FORCE_INLINE void appendMemRowColValEx(SMemRow row, const void *value, bo } static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd, - int32_t idx, int32_t *toffset, int16_t *colId) { + int32_t idx, int32_t *toffset) { int32_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { schemaIdx = spd->boundedColumns[idx]; @@ -165,10 +155,9 @@ static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t memRowTyp *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); } } - *colId = pSchema[schemaIdx].colId; } -static FORCE_INLINE void checkAndConvertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) { +static FORCE_INLINE void convertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) { if (isDataRow(row)) { if (kvLen < (dataLen * KVRatioConvert)) { memRowSetConvert(row); diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index e8bf9da793..4ece848888 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -15,20 +15,24 @@ #include "dataBlockMgt.h" -// #include "astGenerator.h" -// #include "parserInt.h" #include "catalog.h" #include "parserUtil.h" #include "queryInfoUtil.h" -// #include "ttoken.h" -// #include "function.h" -// #include "ttime.h" -// #include "tglobal.h" #include "taosmsg.h" #define IS_RAW_PAYLOAD(t) \ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert +typedef struct SBlockKeyTuple { + TSKEY skey; + void* payloadAddr; +} SBlockKeyTuple; + +typedef struct SBlockKeyInfo { + int32_t maxBytesAlloc; + SBlockKeyTuple* pKeyTuple; +} SBlockKeyInfo; + static int32_t rowDataCompar(const void *lhs, const void *rhs) { TSKEY left = *(TSKEY *)lhs; TSKEY right = *(TSKEY *)rhs; @@ -189,8 +193,7 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { * TODO: Move to tdataformat.h and refactor when STSchema available. * - fetch flen and toffset from STSChema and remove param spd */ -static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, - SParsedDataColInfo *spd) { +static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, SParsedDataColInfo *spd) { ASSERT(isKvRow(src)); SKVRow kvRow = memRowKvBody(src); SDataRow dataRow = memRowDataBody(dest); @@ -209,8 +212,7 @@ static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *p } // TODO: Move to tdataformat.h and refactor when STSchema available. -static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, - SParsedDataColInfo *spd) { +static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, SParsedDataColInfo *spd) { ASSERT(isDataRow(src)); SDataRow dataRow = memRowDataBody(src); @@ -485,22 +487,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } static void extractTableNameList(SHashObj* pHashObj, bool freeBlockMap) { - // pInsertParam->numOfTables = (int32_t) taosHashGetSize(pInsertParam->pTableBlockHashList); - // if (pInsertParam->pTableNameList == NULL) { - // pInsertParam->pTableNameList = malloc(pInsertParam->numOfTables * POINTER_BYTES); - // } - - // STableDataBlocks **p1 = taosHashIterate(pInsertParam->pTableBlockHashList, NULL); - // int32_t i = 0; - // while(p1) { - // STableDataBlocks* pBlocks = *p1; - // pInsertParam->pTableNameList[i++] = tNameDup(&pBlocks->tableName); - // p1 = taosHashIterate(pInsertParam->pTableBlockHashList, p1); - // } - - // if (freeBlockMap) { - // pInsertParam->pTableBlockHashList = tscDestroyBlockHashTable(pInsertParam->pTableBlockHashList, false); - // } + // todo } int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap) { diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 1748588427..0d9d6d7580 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -15,7 +15,6 @@ #include "insertParser.h" -// #include "astGenerator.h" #include "dataBlockMgt.h" #include "parserInt.h" #include "parserUtil.h" @@ -23,7 +22,6 @@ #include "tglobal.h" #include "ttime.h" #include "ttoken.h" -// #include "function.h" #include "ttypes.h" #define NEXT_TOKEN(pSql, sToken) \ @@ -308,518 +306,249 @@ static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec return TSDB_CODE_SUCCESS; } -static int32_t parseOneColumn(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, bool primaryKey, int16_t timePrec, char* payload) { - int64_t iv; - int32_t ret; - char *endptr = NULL; +typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param); + +typedef struct SKvParam { + char buf[TSDB_MAX_TAGS_LEN]; + SKVRowBuilder* builder; + SSchema* schema; +} SKvParam; + +static FORCE_INLINE int32_t KvRowAppend(const void *value, int32_t len, void *param) { + SKvParam* pa = (SKvParam*)param; + if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { + STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); + tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf); + } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + if (!taosMbsToUcs4(value, len, varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } + varDataSetLen(pa->buf, output); + tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, pa->buf); + } else { + tdAddColToKVRow(pa->builder, pa->schema->colId, pa->schema->type, value); + } + return TSDB_CODE_SUCCESS; +} + +typedef struct SMemParam { + SMemRow row; + SSchema* schema; + int32_t toffset; + uint8_t compareStat; + int32_t dataLen; + int32_t kvLen; +} SMemParam; + +static FORCE_INLINE int32_t MemRowAppend(const void *value, int32_t len, void *param) { + SMemParam* pa = (SMemParam*)param; + if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { + char *rowEnd = memRowEnd(pa->row); + STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); + appendMemRowColValEx(pa->row, rowEnd, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat); + } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t output = 0; + char * rowEnd = memRowEnd(pa->row); + if (!taosMbsToUcs4(value, len, (char *)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } + varDataSetLen(rowEnd, output); + appendMemRowColValEx(pa->row, rowEnd, false, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat); + } else { + appendMemRowColValEx(pa->row, value, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat); + } + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t checkAndTrimValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, char* tmpTokenBuf) { + int16_t type = pToken->type; + if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && + type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || + (pToken->n == 0) || (type == TK_RP)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pToken->z); + } if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); } - switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_BOOL_NULL; - } else { - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_TRUE; - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - *(uint8_t *)payload = TSDB_FALSE; - } else { - return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - *(uint8_t *)payload = (int8_t)((iv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - *(uint8_t *)payload = (int8_t)((dv == 0) ? TSDB_FALSE : TSDB_TRUE); - } else { - return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); - } - } - break; + // Remove quotation marks + if (TK_STRING == type) { + if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { + return buildSyntaxErrMsg(&pCxt->msg, "too long string", pToken->z); } - case TSDB_DATA_TYPE_TINYINT: - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_TINYINT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z); - } - *((uint8_t *)payload) = (uint8_t)iv; + // delete escape character: \\, \', \" + char delim = pToken->z[0]; + int32_t cnt = 0; + int32_t j = 0; + for (uint32_t k = 1; k < pToken->n - 1; ++k) { + if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { + tmpTokenBuf[j] = pToken->z[k + 1]; + cnt++; + j++; + k++; + continue; } - break; - case TSDB_DATA_TYPE_UTINYINT: - if (isNullStr(pToken)) { - *((uint8_t *)payload) = TSDB_DATA_UTINYINT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); - } - *((uint8_t *)payload) = (uint8_t)iv; - } - break; - case TSDB_DATA_TYPE_SMALLINT: - if (isNullStr(pToken)) { - *((int16_t *)payload) = TSDB_DATA_SMALLINT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); - } - *((int16_t *)payload) = (int16_t)iv; - } - break; - case TSDB_DATA_TYPE_USMALLINT: - if (isNullStr(pToken)) { - *((uint16_t *)payload) = TSDB_DATA_USMALLINT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); - } - *((uint16_t *)payload) = (uint16_t)iv; - } - break; - case TSDB_DATA_TYPE_INT: - if (isNullStr(pToken)) { - *((int32_t *)payload) = TSDB_DATA_INT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); - } - - *((int32_t *)payload) = (int32_t)iv; - } - - break; - - case TSDB_DATA_TYPE_UINT: - if (isNullStr(pToken)) { - *((uint32_t *)payload) = TSDB_DATA_UINT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); - } - - *((uint32_t *)payload) = (uint32_t)iv; - } - - break; - - case TSDB_DATA_TYPE_BIGINT: - if (isNullStr(pToken)) { - *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z); - } - - *((int64_t *)payload) = iv; - } - break; - - case TSDB_DATA_TYPE_UBIGINT: - if (isNullStr(pToken)) { - *((uint64_t *)payload) = TSDB_DATA_UBIGINT_NULL; - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z); - } - - *((uint64_t *)payload) = iv; - } - break; - - case TSDB_DATA_TYPE_FLOAT: - if (isNullStr(pToken)) { - *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; - } else { - double dv; - if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); - } - - SET_FLOAT_VAL(payload, dv); - } - break; - - case TSDB_DATA_TYPE_DOUBLE: - if (isNullStr(pToken)) { - *((int64_t *)payload) = TSDB_DATA_DOUBLE_NULL; - } else { - double dv; - if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); - } - - *((double *)payload) = dv; - } - break; - - case TSDB_DATA_TYPE_BINARY: - // binary data cannot be null-terminated char string, otherwise the last char of the string is lost - if (pToken->type == TK_NULL) { - setVardataNull(payload, TSDB_DATA_TYPE_BINARY); - } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { //todo refactor - return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); - } - - STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); - } - - break; - - case TSDB_DATA_TYPE_NCHAR: - if (pToken->type == TK_NULL) { - setVardataNull(payload, TSDB_DATA_TYPE_NCHAR); - } else { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); - } - - varDataSetLen(payload, output); - } - break; - - case TSDB_DATA_TYPE_TIMESTAMP: { - if (pToken->type == TK_NULL) { - if (primaryKey) { - *((int64_t *)payload) = 0; - } else { - *((int64_t *)payload) = TSDB_DATA_BIGINT_NULL; - } - } else { - int64_t temp; - if (parseTime(pCxt, pToken, timePrec, &temp) != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); - } - - *((int64_t *)payload) = temp; - } - - break; + tmpTokenBuf[j] = pToken->z[k]; + j++; } + tmpTokenBuf[j] = 0; + pToken->z = tmpTokenBuf; + pToken->n -= 2 + cnt; } return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t parseOneColumnKV(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, bool primaryKey, int16_t timePrec, - SMemRow row, int32_t toffset, int16_t colId, int32_t* dataLen, int32_t* kvLen, uint8_t compareStat) { +static FORCE_INLINE int32_t parseOneValue(SInsertParseContext* pCxt, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, FRowAppend func, void* param) { int64_t iv; int32_t ret; char * endptr = NULL; - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); + CHECK_CODE(checkAndTrimValue(pCxt, pToken, pSchema, tmpTokenBuf)); + + if (isNullStr(pToken)) { + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + int64_t tmpVal = 0; + return func(&tmpVal, pSchema->bytes, param); + } + return func(getNullValue(pSchema->type), 0, param); } switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { // bool - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - appendMemRowColValEx(row, &TRUE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - appendMemRowColValEx(row, &FALSE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_INTEGER) { - iv = strtoll(pToken->z, NULL, 10); - appendMemRowColValEx(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, - dataLen, kvLen, compareStat); - } else if (pToken->type == TK_FLOAT) { - double dv = strtod(pToken->z, NULL); - appendMemRowColValEx(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset, - dataLen, kvLen, compareStat); + case TSDB_DATA_TYPE_BOOL: { + if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + return func(&TRUE_VALUE, pSchema->bytes, param); + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + return func(&FALSE_VALUE, pSchema->bytes, param); } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } + } else if (pToken->type == TK_INTEGER) { + return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + } else if (pToken->type == TK_FLOAT) { + return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } break; } - - case TSDB_DATA_TYPE_TINYINT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + case TSDB_DATA_TYPE_TINYINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "data overflow", pToken->z); } - - break; - - case TSDB_DATA_TYPE_UTINYINT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + uint8_t tmpVal = (uint8_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_UTINYINT:{ + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); } - - break; - - case TSDB_DATA_TYPE_SMALLINT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); - } - - int16_t tmpVal = (int16_t)iv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + uint8_t tmpVal = (uint8_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_SMALLINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); } - - break; - - case TSDB_DATA_TYPE_USMALLINT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); - } - - uint16_t tmpVal = (uint16_t)iv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + int16_t tmpVal = (int16_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_USMALLINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); } - - break; - - case TSDB_DATA_TYPE_INT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); - } - - int32_t tmpVal = (int32_t)iv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + uint16_t tmpVal = (uint16_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_INT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); } - - break; - - case TSDB_DATA_TYPE_UINT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); - } - - uint32_t tmpVal = (uint32_t)iv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + int32_t tmpVal = (int32_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_UINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); } - - break; - - case TSDB_DATA_TYPE_BIGINT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, true); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); - } else if (!IS_VALID_BIGINT(iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z); - } - - appendMemRowColValEx(row, &iv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + uint32_t tmpVal = (uint32_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_BIGINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, true)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "bigint data overflow", pToken->z); } - break; - - case TSDB_DATA_TYPE_UBIGINT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - ret = toInt64(pToken->z, pToken->type, pToken->n, &iv, false); - if (ret != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { - return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z); - } - - uint64_t tmpVal = (uint64_t)iv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + return func(&iv, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_UBIGINT: { + if (TSDB_CODE_SUCCESS != toInt64(pToken->z, pToken->type, pToken->n, &iv, false)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned bigint data overflow", pToken->z); } - break; - - case TSDB_DATA_TYPE_FLOAT: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - double dv; - if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); - } - - float tmpVal = (float)dv; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + uint64_t tmpVal = (uint64_t)iv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_FLOAT: { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - break; - - case TSDB_DATA_TYPE_DOUBLE: - if (isNullStr(pToken)) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - double dv; - if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); - } - - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); - } - - appendMemRowColValEx(row, &dv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - break; - - case TSDB_DATA_TYPE_BINARY: - // binary data cannot be null-terminated char string, otherwise the last char of the string is lost - if (pToken->type == TK_NULL) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor - return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); - } - // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); - char *rowEnd = memRowEnd(row); - STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n); - appendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + float tmpVal = (float)dv; + return func(&tmpVal, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_DOUBLE: { + double dv; + if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - break; - - case TSDB_DATA_TYPE_NCHAR: - if (pToken->type == TK_NULL) { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } else { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - char * rowEnd = memRowEnd(row); - if (!taosMbsToUcs4(pToken->z, pToken->n, (char *)varDataVal(rowEnd), pSchema->bytes - VARSTR_HEADER_SIZE, - &output)) { - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); - } - varDataSetLen(rowEnd, output); - appendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - break; - + return func(&dv, pSchema->bytes, param); + } + case TSDB_DATA_TYPE_BINARY: { + // too long values will return invalid sql, not be truncated automatically + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); + } + return func(pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_NCHAR: { + return func(pToken->z, pToken->n, param); + } case TSDB_DATA_TYPE_TIMESTAMP: { - if (pToken->type == TK_NULL) { - if (primaryKey) { - // When building SKVRow primaryKey, we should not skip even with NULL value. - int64_t tmpVal = 0; - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); - } else { - appendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen, - compareStat); - } - } else { - int64_t tmpVal; - if (parseTime(pCxt, pToken, timePrec, &tmpVal) != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); - } - appendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat); + int64_t tmpVal; + if (parseTime(pCxt, pToken, timePrec, &tmpVal) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); } - - break; + return func(&tmpVal, pSchema->bytes, param); } } - return TSDB_CODE_SUCCESS; + return TSDB_CODE_FAILED; } // pSql -> tag1_name, ...) @@ -892,35 +621,17 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pS return TSDB_CODE_TSC_OUT_OF_MEMORY; } + SKvParam param = {.builder = &kvRowBuilder}; SToken sToken; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pSpd->numOfBound; ++i) { - SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]]; - NEXT_TOKEN(pCxt->pSql, sToken); - - if (TK_ILLEGAL == sToken.type) { - tdDestroyKVRowBuilder(&kvRowBuilder); - destroyBoundColumnInfo(pSpd); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - if (sToken.n == 0 || sToken.type == TK_RP) { - break; - } - - // Remove quotation marks - if (TK_STRING == sToken.type) { - sToken.z++; - sToken.n -= 2; - } - - char tagVal[TSDB_MAX_TAGS_LEN]; - CHECK_CODE_2(parseOneColumn(pCxt, &sToken, pSchema, false, precision, tagVal), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd)); - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); + SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]]; + param.schema = pSchema; + CHECK_CODE_2(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd)); } - - destroyBoundColumnInfo(pSpd); + destroyBoundColumnInfo(pSpd); SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder); if (NULL == row) { @@ -928,19 +639,9 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pS } tdSortKVRowByColIdx(row); - // pInsertParam->tagData.dataLen = kvRowLen(row); - // if (pInsertParam->tagData.dataLen <= 0){ - // return buildInvalidOperationMsg(msgBuf, "tag value expected"); - // } - - // char* pTag = realloc(pInsertParam->tagData.data, pInsertParam->tagData.dataLen); - // if (pTag == NULL) { - // return TSDB_CODE_TSC_OUT_OF_MEMORY; - // } + // todo construct payload - // kvRowCpy(pTag, row); tfree(row); - // pInsertParam->tagData.data = pTag; } // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) @@ -978,99 +679,27 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) return TSDB_CODE_SUCCESS; } -static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) { - int32_t index = 0; - SToken sToken = {0}; - +static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) { + SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo; + SMemRowBuilder* pBuilder = &pDataBlocks->rowBuilder; char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header - - SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; - STableMeta * pTableMeta = pDataBlocks->pTableMeta; - SSchema * schema = getTableColumnSchema(pTableMeta); - SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder; - int32_t dataLen = spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE; - int32_t kvLen = pBuilder->kvRowInitLen; - bool isParseBindParam = false; - initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); + bool isParseBindParam = false; + SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta); + SMemParam param = {.row = row}; + SToken sToken = {0}; // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { - // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundedColumns[i]; - - char *start = row + spd->cols[colIndex].offset; - - SSchema *pSchema = &schema[colIndex]; // get colId here - NEXT_TOKEN(pCxt->pSql, sToken); + // todo bind param + SSchema *pSchema = &schema[spd->boundedColumns[i]]; + param.schema = pSchema; + param.compareStat = pBuilder->compareStat; + getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, ¶m.toffset); + CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, ¶m)); - // if (sToken.type == TK_QUESTION) { - // if (!isParseBindParam) { - // isParseBindParam = true; - // } - // if (pInsertParam->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { - // return buildSyntaxErrMsg(pInsertParam->msg, "? only allowed in binding insertion", *str); - // } - - // uint32_t offset = (uint32_t)(start - pDataBlocks->pData); - // if (tscAddParamToDataBlock(pDataBlocks, pSchema->type, (uint8_t)timePrec, pSchema->bytes, offset) != NULL) { - // continue; - // } - - // strcpy(pInsertParam->msg, "client out of memory"); - // return TSDB_CODE_TSC_OUT_OF_MEMORY; - // } - - int16_t type = sToken.type; - if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && - type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || - (sToken.n == 0) || (type == TK_RP)) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", sToken.z); - } - - // Remove quotation marks - if (TK_STRING == sToken.type) { - // delete escape character: \\, \', \" - char delim = sToken.z[0]; - - int32_t cnt = 0; - int32_t j = 0; - if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) { - return buildSyntaxErrMsg(&pCxt->msg, "too long string", sToken.z); - } - - for (uint32_t k = 1; k < sToken.n - 1; ++k) { - if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { - tmpTokenBuf[j] = sToken.z[k + 1]; - - cnt++; - j++; - k++; - continue; - } - - tmpTokenBuf[j] = sToken.z[k]; - j++; - } - - tmpTokenBuf[j] = 0; - sToken.z = tmpTokenBuf; - sToken.n -= 2 + cnt; - } - - bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_ID); - int32_t toffset = -1; - int16_t colId = -1; - getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); - - int32_t ret = parseOneColumnKV(pCxt, &sToken, pSchema, isPrimaryKey, timePrec, row, toffset, - colId, &dataLen, &kvLen, pBuilder->compareStat); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (isPrimaryKey) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { TSKEY tsKey = memRowKey(row); if (checkTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) { buildSyntaxErrMsg(&pCxt->msg, "client time/server time can not be mixed up", sToken.z); @@ -1082,7 +711,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, if (!isParseBindParam) { // 2. check and set convert flag if (pBuilder->compareStat == ROW_COMPARE_NEED) { - checkAndConvertMemRow(row, dataLen, kvLen); + convertMemRow(row, spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE, pBuilder->kvRowInitLen); } // 3. set the null value for the columns that do not assign values @@ -1097,17 +726,17 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } *len = getExtendedRowSize(pDataBlocks); - return TSDB_CODE_SUCCESS; } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows, char* tmpTokenBuf) { +static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) { STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); int32_t extendedRowSize = getExtendedRowSize(pDataBlock); CHECK_CODE(initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound, pDataBlock->boundColumnInfo.allNullLen)); (*numOfRows) = 0; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" SToken sToken; while (1) { NEXT_TOKEN(pCxt->pSql, sToken); @@ -1145,8 +774,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows)); int32_t numOfRows = 0; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows, tmpTokenBuf)); + CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { SParamInfo *param = dataBuf->params + i; @@ -1186,10 +814,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } SToken tbnameToken = sToken; - NEXT_TOKEN(pCxt->pSql, sToken); - // USING cluase + // USING cluase if (TK_USING == sToken.type) { CHECK_CODE(parseUsingClause(pCxt, &tbnameToken)); NEXT_TOKEN(pCxt->pSql, sToken); @@ -1203,12 +830,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_LP == sToken.type) { // pSql -> field1_name, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - // todo col_list - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_RP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); - } + CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -1222,11 +844,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_FILE == sToken.type) { // pSql -> csv_file_path NEXT_TOKEN(pCxt->pSql, sToken); - if (0 == sToken.n || (TK_STRING != sToken.type && TK_ID != sToken.type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); } - // todo continue; } From cd653e56abd613729ac557124c6349ffcda71b57 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 11 Dec 2021 15:42:09 +0800 Subject: [PATCH 06/22] update fst search frame --- source/libs/index/inc/index_fst.h | 16 ++--- source/libs/index/inc/index_fst_automation.h | 30 ++++---- source/libs/index/src/index_fst.c | 31 ++++---- source/libs/index/src/index_fst_automation.c | 74 ++++++++++++++++++++ source/libs/index/test/indexTests.cpp | 46 +++++++----- 5 files changed, 146 insertions(+), 51 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index eb288e0aa2..23da009b22 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -291,11 +291,11 @@ typedef struct StreamState { void streamStateDestroy(void *s); typedef struct StreamWithState { - Fst *fst; - Automation *aut; - SArray *inp; - FstOutput emptyOutput; - SArray *stack; // + Fst *fst; + AutomationCtx *aut; + SArray *inp; + FstOutput emptyOutput; + SArray *stack; // FstBoundWithData *endAt; } StreamWithState ; @@ -310,19 +310,19 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta void swsResultDestroy(StreamWithStateResult *result); typedef void* (*StreamCallback)(void *); -StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) ; +StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ; void streamWithStateDestroy(StreamWithState *sws); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); typedef struct FstStreamBuilder { Fst *fst; - Automation *aut; + AutomationCtx *aut; FstBoundWithData *min; FstBoundWithData *max; } FstStreamBuilder; -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut); +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); // set up bound range // refator, simple code by marco diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 480d3110c4..4943599104 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -19,33 +19,39 @@ extern "C" { #endif +#include "index_fst_util.h" typedef struct AutomationCtx AutomationCtx; +typedef enum AutomationType { + AUTOMATION_PREFIX, + AUTMMATION_MATCH +} AutomationType; + typedef struct StartWith { AutomationCtx *autoSelf; } StartWith; typedef struct Complement { AutomationCtx *autoSelf; - } Complement; // automation typedef struct AutomationCtx { -// automation interface - void *data; + AutomationType type; } AutomationCtx; -typedef struct Automation { - void* (*start)() ; - bool (*isMatch)(void *); - bool (*canMatch)(void *data); - bool (*willAlwaysMatch)(void *state); - void* (*accept)(void *state, uint8_t byte); - void* (*acceptEof)(void *state); - void *data; -} Automation; +typedef struct AutomationFunc { + void* (*start)(AutomationCtx *ctx) ; + bool (*isMatch)(AutomationCtx *ctx, void *); + bool (*canMatch)(AutomationCtx *ctx, void *data); + bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); + void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte); + void* (*acceptEof)(AutomationCtx *ct, void *state); +} AutomationFunc; +AutomationCtx *automCtxCreate(void *data, AutomationType type); + +extern AutomationFunc automFuncs[]; #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 6a81e888d2..b1401c30fe 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1177,7 +1177,7 @@ void fstBoundDestroy(FstBoundWithData *bound) { free(bound); } -StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) { +StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { StreamWithState *sws = calloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; } @@ -1204,6 +1204,8 @@ void streamWithStateDestroy(StreamWithState *sws) { } bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { + + AutomationCtx *aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); @@ -1211,7 +1213,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { StreamState s = {.node = fstGetRoot(sws->fst), .trans = 0, .out = {.null = false, .out = 0}, - .autState = sws->aut->start()}; // auto.start callback + .autState = automFuncs[aut->type].start(aut)}; // auto.start callback taosArrayPush(sws->stack, &s); return true; } @@ -1229,7 +1231,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { FstNode *node = fstGetRoot(sws->fst); Output out = 0; - void* autState = sws->aut->start(); + //void* autState = sws->aut->start(); + void* autState = automFuncs[aut->type].start(aut); int32_t len; uint8_t *data = fstSliceData(key, &len); @@ -1241,7 +1244,8 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { FstTransition trn; fstNodeGetTransitionAt(node, res, &trn); void *preState = autState; - autState = sws->aut->accept(preState, b); + // autState = sws->aut->accept(preState, b); + autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); StreamState s = {.node = node, .trans = res + 1, @@ -1298,6 +1302,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { } StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { + AutomationCtx *aut = sws->aut; FstOutput output = sws->emptyOutput; if (output.null == false) { FstSlice emptySlice = fstSliceCreate(NULL, 0); @@ -1306,15 +1311,15 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); return NULL; } - void* start = sws->aut->start(); - if (sws->aut->isMatch(start)) { + void *start = automFuncs[aut->type].start(aut); + if (automFuncs[aut->type].isMatch(aut, start)) { FstSlice s = fstSliceCreate(NULL, 0); return swsResultCreate(&s, output, callback(start)); } } while (taosArrayGetSize(sws->stack) > 0) { StreamState *p = (StreamState *)taosArrayPop(sws->stack); - if (p->trans >= FST_NODE_LEN(p->node) || !sws->aut->canMatch(p->autState)) { + if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } @@ -1324,16 +1329,18 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb FstTransition trn; fstNodeGetTransitionAt(p->node, p->trans, &trn); Output out = p->out.out + trn.out; - void* nextState = sws->aut->accept(p->autState, trn.inp); + void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); void* tState = callback(nextState); - bool isMatch = sws->aut->isMatch(nextState); + bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); + //bool isMatch = sws->aut->isMatch(nextState); FstNode *nextNode = fstGetNode(sws->fst, trn.addr); taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { - void *eofState = sws->aut->acceptEof(nextState); + //void *eofState = sws->aut->acceptEof(nextState); + void *eofState = automFuncs[aut->type].acceptEof(aut, nextState); if (eofState != NULL) { - isMatch = sws->aut->isMatch(eofState); + isMatch = automFuncs[aut->type].isMatch(aut, eofState); } } StreamState s1 = { .node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; @@ -1391,7 +1398,7 @@ void streamStateDestroy(void *s) { //free(s->autoState); } -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); if (NULL == b) { return NULL; } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 3d5efd30f3..c777f57b02 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -13,3 +13,77 @@ * along with this program. If not, see . */ +#include "index_fst_automation.h" + + +// prefix query, impl later +static void* prefixStart(AutomationCtx *ctx) { + return NULL; +}; +static bool prefixIsMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool prefixCanMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { + return true; +} +static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { + return NULL; +} +static void* prefixAcceptEof(AutomationCtx *ctx, void *state) { + return NULL; +} + +// pattern query, impl later + +static void* patternStart(AutomationCtx *ctx) { + return NULL; +} +static bool patternIsMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool patternCanMatch(AutomationCtx *ctx, void *data) { + return true; +} +static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { + return true; +} + +static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { + return NULL; +} + +static void* patternAcceptEof(AutomationCtx *ctx, void *state) { + return NULL; +} + +AutomationFunc automFuncs[] = {{ + prefixStart, + prefixIsMatch, + prefixCanMatch, + prefixWillAlwaysMatch, + prefixAccept, + prefixAcceptEof + }, + { + patternStart, + patternIsMatch, + patternCanMatch, + patternWillAlwaysMatch, + patternAccept, + patternAcceptEof + } +}; + +AutomationCtx* automCtxCreate(void *data, AutomationType type) { + AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); + if (ctx == NULL) { return NULL; } + + ctx->type = type; + if (type == AUTOMATION_PREFIX) { + + } +} + diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 928c3875b0..0cfb0fedc3 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -65,6 +65,7 @@ class FstReadMemory { ~FstReadMemory() { fstCountingWriterDestroy(_w); + fstDestroy(_fst); fstSliceDestroy(&_s); } @@ -129,10 +130,12 @@ class FstReadMemory { //} +#define L 100 +#define M 100 +#define N 100 int Performance_fstWriteRecords(FstWriter *b) { std::string str("aa"); - int L = 100, M = 100, N = 10; for (int i = 0; i < L; i++) { str[0] = 'a' + i; str.resize(2); @@ -150,22 +153,29 @@ int Performance_fstWriteRecords(FstWriter *b) { } void Performance_fstReadRecords(FstReadMemory *m) { - std::string str("a"); - for (int i = 0; i < 50; i++) { - //std::string str("aa"); - str.push_back('a'); - uint64_t out, cost; - bool ok = m->GetWithTimeCostUs(str, &out, &cost); - if (ok == true) { - printf("success to get (%s, %" PRId64"), time cost: %" PRId64")\n", str.c_str(), out, cost); - } else { - printf("failed to get(%s)\n", str.c_str()); - } - } + std::string str("aa"); + for (int i = 0; i < M; i++) { + str[0] = 'a' + i; + str.resize(2); + for(int j = 0; j < N; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < L; k++) { + str.push_back('a'); + uint64_t val, cost; + if (m->GetWithTimeCostUs(str, &val, &cost)) { + printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost); + } else { + printf("failed to get key: %s\n", str.c_str()); + } + } + } + } } void checkFstPerf() { FstWriter *fw = new FstWriter; int64_t s = taosGetTimestampUs(); + int num = Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); printf("write %d record cost %" PRId64"us\n", num, e - s); @@ -173,13 +183,11 @@ void checkFstPerf() { FstReadMemory *m = new FstReadMemory(1024 * 64); if (m->init()) { - uint64_t val; - if(m->Get("aaaaaaa", &val)) { - std::cout << "succes to Get val: " << val << std::endl; - } else { - std::cout << "failed to Get " << std::endl; - } + printf("success to init fst read"); } + Performance_fstReadRecords(m); + + delete m; } From a9a068410b71b30212733f8a0dc70689624b7cd7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 11 Dec 2021 16:50:05 +0800 Subject: [PATCH 07/22] update fst search frame --- source/libs/index/inc/index_fst.h | 1 + source/libs/index/inc/index_fst_automation.h | 1 + source/libs/index/src/index_fst.c | 1 - source/libs/index/src/index_fst_automation.c | 14 +++++++++++--- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 23da009b22..b26bd227f8 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -275,6 +275,7 @@ FstNode* fstGetRoot(Fst *fst); FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); Output fstEmptyFinalOutput(Fst *fst, bool *null); + bool fstVerify(Fst *fst); diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 4943599104..4a29f2e3a6 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -50,6 +50,7 @@ typedef struct AutomationFunc { } AutomationFunc; AutomationCtx *automCtxCreate(void *data, AutomationType type); +void autoCtxDestroy(AutomationCtx *ctx); extern AutomationFunc automFuncs[]; #ifdef __cplusplus diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index b1401c30fe..18e617f6ae 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1072,7 +1072,6 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { tOut += trn.out; root = fstGetNode(fst, trn.addr); taosArrayPush(nodes, &root); - //fstNodeDestroy(root); } if (!FST_NODE_IS_FINAL(root)) { return false; diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index c777f57b02..748c55c29d 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -75,15 +75,23 @@ AutomationFunc automFuncs[] = {{ patternAccept, patternAcceptEof } + // add more search type }; AutomationCtx* automCtxCreate(void *data, AutomationType type) { - AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); + AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; - if (type == AUTOMATION_PREFIX) { + if (ctx->type == AUTOMATION_PREFIX) { + } else if (ctx->type == AUTMMATION_MATCH) { + + } else { + // add more search type } + return ctx; } - +void autoCtxDestroy(AutomationCtx *ctx) { + free(ctx); +} From 4f2cdd223b860b15b2ad62b505e78d3161469431 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 11 Dec 2021 17:17:21 +0800 Subject: [PATCH 08/22] add test for wal snapshot --- include/libs/wal/wal.h | 47 +-- include/util/tarray.h | 15 + include/util/tchecksum.h | 2 +- source/libs/wal/inc/walInt.h | 30 ++ source/libs/wal/src/walMeta.c | 12 + source/libs/wal/src/walMgmt.c | 12 +- source/libs/wal/src/walRead.c | 32 +- source/libs/wal/src/{walIndex.c => walSeek.c} | 45 ++- source/libs/wal/src/walWrite.c | 307 +++++++++++------- source/libs/wal/test/walMetaTest.cpp | 128 ++++++-- source/util/src/tarray.c | 11 +- 11 files changed, 440 insertions(+), 201 deletions(-) rename source/libs/wal/src/{walIndex.c => walSeek.c} (76%) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index f84297670b..fe3ee17d00 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -38,6 +38,24 @@ typedef enum { TAOS_WAL_FSYNC = 2 } EWalType; +typedef struct SWalReadHead { + int8_t sver; + uint8_t msgType; + int8_t reserved[2]; + int32_t len; + int64_t version; + char cont[]; +} SWalReadHead; + +typedef struct { + int32_t vgId; + int32_t fsyncPeriod; // millisecond + int32_t retentionPeriod; // secs + int32_t rollPeriod; // secs + int64_t segSize; + EWalType walLevel; // wal level +} SWalCfg; + typedef struct { //union { //uint32_t info; @@ -47,25 +65,11 @@ typedef struct { //uint32_t reserved : 24; //}; //}; - int8_t sver; - uint8_t msgType; - int8_t reserved[2]; - int32_t len; - int64_t version; - uint32_t signature; uint32_t cksumHead; uint32_t cksumBody; - char cont[]; + SWalReadHead head; } SWalHead; -typedef struct { - int32_t vgId; - int32_t fsyncPeriod; // millisecond - int32_t rollPeriod; - int64_t segSize; - EWalType walLevel; // wal level -} SWalCfg; - #define WAL_PREFIX "wal" #define WAL_PREFIX_LEN 3 #define WAL_NOSUFFIX_LEN 20 @@ -80,7 +84,7 @@ typedef struct { //#define WAL_FILE_NUM 1 // 3 #define WAL_FILESET_MAX 128 -#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) +#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) #define WAL_CUR_POS_WRITABLE 1 #define WAL_CUR_FILE_WRITABLE 2 #define WAL_CUR_FAILED 4 @@ -103,21 +107,17 @@ typedef struct SWal { //write tfd int64_t writeLogTfd; int64_t writeIdxTfd; - //read tfd - int64_t readLogTfd; - int64_t readIdxTfd; - //current version - int64_t curVersion; //wal lifecycle int64_t firstVersion; int64_t snapshotVersion; int64_t commitVersion; int64_t lastVersion; + //snapshotting version + int64_t snapshottingVer; //roll status int64_t lastRollSeq; //file set int32_t writeCur; - int32_t readCur; SArray* fileInfoSet; //ctl int32_t curStatus; @@ -148,7 +148,8 @@ int32_t walCommit(SWal *, int64_t ver); // truncate after int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely -int32_t walTakeSnapshot(SWal *, int64_t ver); +int32_t walBeginTakeSnapshot(SWal *, int64_t ver); +int32_t walEndTakeSnapshot(SWal *); //int32_t walDataCorrupted(SWal*); // read diff --git a/include/util/tarray.h b/include/util/tarray.h index 25862a7119..ad02b20868 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -153,6 +153,13 @@ void taosArraySet(SArray* pArray, size_t index, void* pData); */ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt); +/** + * remove some data entry from front + * @param pArray + * @param cnt + */ +void taosArrayPopTailBatch(SArray* pArray, size_t cnt); + /** * remove data entry of the given index * @param pArray @@ -213,6 +220,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); */ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); +/** + * search the array, return index of the element + * @param pArray + * @param compar + * @param key + */ +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); + /** * search the array * @param pArray diff --git a/include/util/tchecksum.h b/include/util/tchecksum.h index c1907b0c46..fb6c85edcd 100644 --- a/include/util/tchecksum.h +++ b/include/util/tchecksum.h @@ -39,7 +39,7 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin } static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { - return (checksum == (*crc32c)(0, stream, (size_t)ssize)); + return (checksum != (*crc32c)(0, stream, (size_t)ssize)); } static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) { diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index f56f240904..f8f2e2eadd 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -18,6 +18,7 @@ #include "wal.h" #include "compare.h" +#include "tchecksum.h" #ifdef __cplusplus extern "C" { @@ -32,6 +33,11 @@ typedef struct WalFileInfo { int64_t fileSize; } WalFileInfo; +typedef struct WalIdxEntry { + int64_t ver; + int64_t offset; +} WalIdxEntry; + static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; WalFileInfo* pInfoRight = (WalFileInfo*)pRight; @@ -79,6 +85,26 @@ static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); } +static inline int walValidHeadCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)&pHead->head, sizeof(SWalReadHead), pHead->cksumHead); +} + +static inline int walValidBodyCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)pHead->head.cont, pHead->head.len, pHead->cksumBody); +} + +static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { + return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); +} + +static inline uint32_t walCalcHeadCksum(SWalHead *pHead) { + return taosCalcChecksum(0, (uint8_t*)&pHead->head, sizeof(SWalReadHead)); +} + +static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) { + return taosCalcChecksum(0, (uint8_t*)body, len); +} + int walReadMeta(SWal* pWal); int walWriteMeta(SWal* pWal); int walRollFileInfo(SWal* pWal); @@ -87,6 +113,10 @@ char* walMetaSerialize(SWal* pWal); int walMetaDeserialize(SWal* pWal, const char* bytes); //meta section end +//seek section +int walChangeFile(SWal *pWal, int64_t ver); +//seek section end + int64_t walGetSeq(); int walSeekVer(SWal *pWal, int64_t ver); int walRoll(SWal *pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 65085bb96d..89588b1ddf 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -24,6 +24,18 @@ #include #include +int64_t walGetFirstVer(SWal *pWal) { + return pWal->firstVersion; +} + +int64_t walGetSnaphostVer(SWal *pWal) { + return pWal->snapshotVersion; +} + +int64_t walGetLastVer(SWal *pWal) { + return pWal->lastVersion; +} + int walRollFileInfo(SWal* pWal) { int64_t ts = taosGetTimestampSec(); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 6bf9008917..9090348c04 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -82,6 +82,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; + pWal->writeCur = -1; //set config pWal->vgId = pCfg->vgId; @@ -90,13 +91,20 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->segSize = pCfg->segSize; pWal->level = pCfg->walLevel; - //init status + //init version info + pWal->firstVersion = -1; + pWal->commitVersion = -1; + pWal->snapshotVersion = -1; pWal->lastVersion = -1; + + pWal->snapshottingVer = -1; + + //init status pWal->lastRollSeq = -1; //init write buffer memset(&pWal->head, 0, sizeof(SWalHead)); - pWal->head.sver = 0; + pWal->head.head.sver = 0; tstrncpy(pWal->path, path, sizeof(pWal->path)); pthread_mutex_init(&pWal->mutex, NULL); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e9f5bcbc5d..27dffddf83 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -15,19 +15,6 @@ #include "walInt.h" #include "tfile.h" -#include "tchecksum.h" - -static inline int walValidHeadCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead, sizeof(SWalHead) - sizeof(uint32_t)*2, pHead->cksumHead); -} - -static inline int walValidBodyCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead->cont, pHead->len, pHead->cksumBody); -} - -static int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { - return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); -} int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int code; @@ -49,13 +36,13 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { if(walValidHeadCksum(*ppHead) != 0) { return -1; } - void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->len); + void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); if(ptr == NULL) { free(*ppHead); *ppHead = NULL; return -1; } - if(tfRead(pWal->writeLogTfd, (*ppHead)->cont, (*ppHead)->len) != (*ppHead)->len) { + if(tfRead(pWal->writeLogTfd, (*ppHead)->head.cont, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } //TODO: endian compatibility processing after read @@ -69,18 +56,3 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) { return 0; } - -int64_t walGetFirstVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->firstVersion; -} - -int64_t walGetSnaphostVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->snapshotVersion; -} - -int64_t walGetLastVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->lastVersion; -} diff --git a/source/libs/wal/src/walIndex.c b/source/libs/wal/src/walSeek.c similarity index 76% rename from source/libs/wal/src/walIndex.c rename to source/libs/wal/src/walSeek.c index bf51be4346..48272f8f32 100644 --- a/source/libs/wal/src/walIndex.c +++ b/source/libs/wal/src/walSeek.c @@ -43,12 +43,35 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { if (code != 0) { return -1; } - /*pWal->curLogOffset = readBuf[1];*/ - pWal->curVersion = ver; return code; } -static int walChangeFile(SWal *pWal, int64_t ver) { +int walChangeFileToLast(SWal *pWal) { + int64_t idxTfd, logTfd; + WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); + ASSERT(pRet != NULL); + int64_t fileFirstVer = pRet->firstVer; + + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, fileFirstVer, fnameStr); + idxTfd = tfOpenReadWrite(fnameStr); + if(idxTfd < 0) { + return -1; + } + walBuildLogName(pWal, fileFirstVer, fnameStr); + logTfd = tfOpenReadWrite(fnameStr); + if(logTfd < 0) { + return -1; + } + //switch file + pWal->writeIdxTfd = idxTfd; + pWal->writeLogTfd = logTfd; + //change status + pWal->curStatus = WAL_CUR_FILE_WRITABLE; + return 0; +} + +int walChangeFile(SWal *pWal, int64_t ver) { int code = 0; int64_t idxTfd, logTfd; char fnameStr[WAL_FILE_LEN]; @@ -86,21 +109,21 @@ static int walChangeFile(SWal *pWal, int64_t ver) { return code; } +int walGetVerOffset(SWal* pWal, int64_t ver) { + int code; + return 0; +} + int walSeekVer(SWal *pWal, int64_t ver) { int code; - if((!(pWal->curStatus & WAL_CUR_FAILED)) && ver == pWal->curVersion) { + if(ver == pWal->lastVersion) { return 0; } - if(ver > pWal->lastVersion) { - //TODO: some records are skipped - return -1; - } - if(ver < pWal->firstVersion) { - //TODO: try to seek pruned log + if(ver > pWal->lastVersion || ver < pWal->firstVersion) { return -1; } if(ver < pWal->snapshotVersion) { - //TODO: seek snapshotted log, invalid in some cases + //TODO: set flag to prevent roll back } if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeFile(pWal, ver); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 0c4989300f..87d52ad236 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -21,65 +21,6 @@ #include "tfile.h" #include "walInt.h" -static void walFtruncate(SWal *pWal, int64_t ver); - -int32_t walCommit(SWal *pWal, int64_t ver) { - ASSERT(pWal->snapshotVersion <= pWal->commitVersion); - ASSERT(pWal->commitVersion <= pWal->lastVersion); - ASSERT(ver >= pWal->commitVersion); - ASSERT(ver <= pWal->lastVersion); - pWal->commitVersion = ver; - return 0; -} - -int32_t walRollback(SWal *pWal, int64_t ver) { - //TODO: ftruncate - ASSERT(ver > pWal->commitVersion); - ASSERT(ver <= pWal->lastVersion); - //seek position - walSeekVer(pWal, ver); - walFtruncate(pWal, ver); - return 0; -} - -int32_t walTakeSnapshot(SWal *pWal, int64_t ver) { - pWal->snapshotVersion = ver; - int ts = taosGetTimestampSec(); - - int deleteCnt = 0; - int64_t newTotSize = pWal->totSize; - WalFileInfo tmp; - tmp.firstVer = ver; - //mark files safe to delete - WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); - //iterate files, until the searched result - for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if(pWal->totSize > pWal->retentionSize || - iter->closeTs + pWal->retentionPeriod > ts) { - //delete according to file size or close time - deleteCnt++; - newTotSize -= iter->fileSize; - } - } - char fnameStr[WAL_FILE_LEN]; - //remove file - for(int i = 0; i < deleteCnt; i++) { - WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); - walBuildLogName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); - walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); - } - - //save snapshot ver, commit ver - - - //make new array, remove files - taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); - pWal->totSize = newTotSize; - - return 0; -} #if 0 static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId); @@ -172,6 +113,185 @@ void walRemoveAllOldFiles(void *handle) { } #endif +int32_t walCommit(SWal *pWal, int64_t ver) { + ASSERT(pWal->commitVersion >= pWal->snapshotVersion); + ASSERT(pWal->commitVersion <= pWal->lastVersion); + if(ver < pWal->commitVersion || ver > pWal->lastVersion) { + return -1; + } + pWal->commitVersion = ver; + return 0; +} + +int32_t walRollback(SWal *pWal, int64_t ver) { + int code; + char fnameStr[WAL_FILE_LEN]; + if(ver == pWal->lastVersion) { + return 0; + } + if(ver > pWal->lastVersion || ver < pWal->commitVersion) { + return -1; + } + pthread_mutex_lock(&pWal->mutex); + + //find correct file + if(ver < walGetLastFileFirstVer(pWal)) { + //close current files + tfClose(pWal->writeIdxTfd); + tfClose(pWal->writeLogTfd); + //open old files + code = walChangeFile(pWal, ver); + if(code != 0) { + return -1; + } + + //delete files + int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); + for(int i = pWal->writeCur; i < fileSetSize; i++) { + walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + remove(fnameStr); + walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + remove(fnameStr); + } + //pop from fileInfoSet + taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); + } + + walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + int64_t idxTfd = tfOpenReadWrite(fnameStr); + + //change to deserialize function + + if(idxTfd < 0) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + int idxOff = (ver - walGetCurFileFirstVer(pWal)) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, idxOff, SEEK_SET); + if(code < 0) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + //read idx file and get log file pos + //TODO:change to deserialize function + WalIdxEntry entry; + if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + ASSERT(entry.ver == ver); + + walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + int64_t logTfd = tfOpenReadWrite(fnameStr); + if(logTfd < 0) { + //TODO + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + code = tfLseek(logTfd, entry.offset, SEEK_SET); + if(code < 0) { + //TODO + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + //validate offset + SWalHead head; + ASSERT(tfValid(logTfd)); + int size = tfRead(logTfd, &head, sizeof(SWalHead)); + if(size != sizeof(SWalHead)) { + return -1; + } + code = walValidHeadCksum(&head); + + ASSERT(code == 0); + if(code != 0) { + return -1; + } + if(head.head.version != ver) { + //TODO + return -1; + } + //truncate old files + code = tfFtruncate(logTfd, entry.offset); + if(code < 0) { + return -1; + } + code = tfFtruncate(idxTfd, idxOff); + if(code < 0) { + return -1; + } + pWal->lastVersion = ver - 1; + ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; + ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + + //unlock + pthread_mutex_unlock(&pWal->mutex); + return 0; +} +int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { + pWal->snapshottingVer = ver; + //check file rolling + if(pWal->retentionPeriod == 0) { + walRoll(pWal); + } + + return 0; +} + +int32_t walEndTakeSnapshot(SWal *pWal) { + int64_t ver = pWal->snapshottingVer; + if(ver == -1) return -1; + + pWal->snapshotVersion = ver; + int ts = taosGetTimestampSec(); + + int deleteCnt = 0; + int64_t newTotSize = pWal->totSize; + WalFileInfo tmp; + tmp.firstVer = ver; + //find files safe to delete + WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); + if(ver >= pInfo->lastVer) { + pInfo++; + } + //iterate files, until the searched result + for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { + if(pWal->totSize > pWal->retentionSize || + iter->closeTs + pWal->retentionPeriod > ts) { + //delete according to file size or close time + deleteCnt++; + newTotSize -= iter->fileSize; + } + } + char fnameStr[WAL_FILE_LEN]; + //remove file + for(int i = 0; i < deleteCnt; i++) { + WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); + walBuildLogName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); + walBuildIdxName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); + } + + //make new array, remove files + taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); + if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + pWal->firstVersion = -1; + } else { + pWal->firstVersion = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + } + pWal->totSize = newTotSize; + pWal->snapshottingVer = -1; + + //save snapshot ver, commit ver + int code = walWriteMeta(pWal); + if(code != 0) { + return -1; + } + + return 0; +} + int walRoll(SWal *pWal) { int code = 0; if(pWal->writeIdxTfd != -1) { @@ -211,6 +331,7 @@ int walRoll(SWal *pWal) { //switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; //change status pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE; @@ -218,32 +339,6 @@ int walRoll(SWal *pWal) { return 0; } -int walChangeFileToLast(SWal *pWal) { - int64_t idxTfd, logTfd; - WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); - ASSERT(pRet != NULL); - int64_t fileFirstVer = pRet->firstVer; - - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenReadWrite(fnameStr); - if(idxTfd < 0) { - return -1; - } - walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenReadWrite(fnameStr); - if(logTfd < 0) { - return -1; - } - //switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; - //change status - pWal->curVersion = fileFirstVer; - pWal->curStatus = WAL_CUR_FILE_WRITABLE; - return 0; -} - static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { int code = 0; //get index file @@ -253,9 +348,11 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); return code; } - int64_t writeBuf[2] = { ver, offset }; - int size = tfWrite(pWal->writeIdxTfd, writeBuf, sizeof(writeBuf)); - if(size != sizeof(writeBuf)) { + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + WalIdxEntry entry = { .ver = ver, .offset = offset }; + int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); + if(size != sizeof(WalIdxEntry)) { return -1; } return 0; @@ -270,13 +367,14 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i if (index == pWal->lastVersion + 1) { if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + pWal->firstVersion = index; code = walRoll(pWal); ASSERT(code == 0); } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(pWal->rollPeriod != -1 && passed > pWal->rollPeriod) { + if(pWal->rollPeriod != -1 && pWal->rollPeriod != 0 && passed > pWal->rollPeriod) { walRoll(pWal); - } else if(pWal->segSize != -1 && walGetLastFileSize(pWal) > pWal->segSize) { + } else if(pWal->segSize != -1 && pWal->segSize != 0 && walGetLastFileSize(pWal) > pWal->segSize) { walRoll(pWal); } } @@ -287,16 +385,13 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } /*if (!tfValid(pWal->curLogTfd)) return 0;*/ - pWal->head.version = index; - - pWal->head.signature = WAL_SIGNATURE; - pWal->head.len = bodyLen; - pWal->head.msgType = msgType; - - pWal->head.cksumHead = taosCalcChecksum(0, (const uint8_t*)&pWal->head, sizeof(SWalHead)- sizeof(uint32_t)*2); - pWal->head.cksumBody = taosCalcChecksum(0, (const uint8_t*)&body, bodyLen); - pthread_mutex_lock(&pWal->mutex); + pWal->head.head.version = index; + + pWal->head.head.len = bodyLen; + pWal->head.head.msgType = msgType; + pWal->head.cksumHead = walCalcHeadCksum(&pWal->head); + pWal->head.cksumBody = walCalcBodyCksum(body, bodyLen); if (tfWrite(pWal->writeLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) { //ftruncate @@ -312,6 +407,7 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); if(code != 0) { //TODO + return -1; } //set status @@ -326,8 +422,6 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } void walFsync(SWal *pWal, bool forceFsync) { - if (pWal == NULL || !tfValid(pWal->writeLogTfd)) return; - if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, walGetCurFileFirstVer(pWal)); if (tfFsync(pWal->writeLogTfd) < 0) { @@ -408,7 +502,7 @@ static int walValidateOffset(SWal* pWal, int64_t ver) { int code = 0; SWalHead *pHead = NULL; code = (int)walRead(pWal, &pHead, ver); - if(pHead->version != ver) { + if(pHead->head.version != ver) { return -1; } return 0; @@ -428,15 +522,6 @@ static int64_t walGetOffset(SWal* pWal, int64_t ver) { return 0; } -static void walFtruncate(SWal *pWal, int64_t ver) { - int64_t tfd = pWal->writeLogTfd; - tfFtruncate(tfd, ver); - tfFsync(tfd); - tfd = pWal->writeIdxTfd; - tfFtruncate(tfd, ver * WAL_IDX_ENTRY_SIZE); - tfFsync(tfd); -} - #if 0 static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { int64_t pos = *offset; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 96258662a1..f244917f77 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -36,6 +36,36 @@ class WalCleanEnv : public ::testing::Test { const char* pathName = "/tmp/wal_test"; }; +class WalCleanDeleteEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } + + static void TearDownTestCase() { + walCleanUp(); + } + + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->walLevel = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; +}; + class WalKeepEnv : public ::testing::Test { protected: static void SetUpTestCase() { @@ -110,40 +140,94 @@ TEST_F(WalCleanEnv, removeOldMeta) { ASSERT(code == 0); } -TEST_F(WalKeepEnv, readOldMeta) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walWriteMeta(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walWriteMeta(pWal); - ASSERT(code == 0); - char*oldss = walMetaSerialize(pWal); +//TEST_F(WalKeepEnv, readOldMeta) { + //int code = walRollFileInfo(pWal); + //ASSERT(code == 0); + //code = walWriteMeta(pWal); + //ASSERT(code == 0); + //code = walRollFileInfo(pWal); + //ASSERT(code == 0); + //code = walWriteMeta(pWal); + //ASSERT(code == 0); + //char*oldss = walMetaSerialize(pWal); - TearDown(); - SetUp(); - code = walReadMeta(pWal); - ASSERT(code == 0); - char* newss = walMetaSerialize(pWal); + //TearDown(); + //SetUp(); + //code = walReadMeta(pWal); + //ASSERT(code == 0); + //char* newss = walMetaSerialize(pWal); - int len = strlen(oldss); - ASSERT_EQ(len, strlen(newss)); - for(int i = 0; i < len; i++) { - EXPECT_EQ(oldss[i], newss[i]); - } -} + //int len = strlen(oldss); + //ASSERT_EQ(len, strlen(newss)); + //for(int i = 0; i < len; i++) { + //EXPECT_EQ(oldss[i], newss[i]); + //} +//} -TEST_F(WalKeepEnv, write) { +TEST_F(WalCleanEnv, write) { const char* ranStr = "tvapq02tcp"; const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { code = walWrite(pWal, i, i+1, (void*)ranStr, len); ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, i); code = walWrite(pWal, i+2, i, (void*)ranStr, len); ASSERT_EQ(code, -1); + ASSERT_EQ(pWal->lastVersion, i); } code = walWriteMeta(pWal); ASSERT_EQ(code, 0); } + +TEST_F(WalCleanEnv, rollback) { + const char* ranStr = "tvapq02tcp"; + const int len = strlen(ranStr); + int code; + for(int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i+1, (void*)ranStr, len); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, i); + } + code = walRollback(pWal, 5); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, 4); + code = walRollback(pWal, 3); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, 2); + code = walWriteMeta(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalCleanDeleteEnv, roll) { + const char* ranStr = "tvapq02tcp"; + const int len = strlen(ranStr); + int code; + int i; + for(i = 0; i < 100; i++) { + code = walWrite(pWal, i, 0, (void*)ranStr, len); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, i); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->commitVersion, i); + } + + walBeginTakeSnapshot(pWal, i-1); + ASSERT_EQ(pWal->snapshottingVer, i-1); + walEndTakeSnapshot(pWal); + ASSERT_EQ(pWal->snapshotVersion, i-1); + ASSERT_EQ(pWal->snapshottingVer, -1); + + code = walWrite(pWal, 5, 0, (void*)ranStr, len); + ASSERT_NE(code, 0); + + for(; i < 200; i++) { + code = walWrite(pWal, i, 0, (void*)ranStr, len); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->commitVersion, i); + } + + code = walWriteMeta(pWal); + ASSERT_EQ(code, 0); +} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index ff52477a6f..2191419670 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -241,12 +241,16 @@ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { assert(cnt <= pArray->size); pArray->size = pArray->size - cnt; if(pArray->size == 0) { - pArray->size = 0; return; } memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size); } +void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { + assert(cnt <= pArray->size); + pArray->size = pArray->size - cnt; +} + void taosArrayRemove(SArray* pArray, size_t index) { assert(index < pArray->size); @@ -329,6 +333,11 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); } +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { + void* item = taosArraySearch(pArray, key, comparFn, flags); + return (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; +} + void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { assert(pArray != NULL); qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); From 9e4b8907273584992635fa1a4a56f4aa99570e33 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 11 Dec 2021 17:17:21 +0800 Subject: [PATCH 09/22] add test for wal snapshot --- include/libs/wal/wal.h | 47 +-- include/util/tarray.h | 15 + include/util/tchecksum.h | 2 +- source/libs/wal/inc/walInt.h | 30 ++ source/libs/wal/src/walMeta.c | 12 + source/libs/wal/src/walMgmt.c | 12 +- source/libs/wal/src/walRead.c | 32 +- source/libs/wal/src/{walIndex.c => walSeek.c} | 45 ++- source/libs/wal/src/walWrite.c | 307 +++++++++++------- source/libs/wal/test/walMetaTest.cpp | 128 ++++++-- source/util/src/tarray.c | 11 +- 11 files changed, 440 insertions(+), 201 deletions(-) rename source/libs/wal/src/{walIndex.c => walSeek.c} (76%) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index f84297670b..fe3ee17d00 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -38,6 +38,24 @@ typedef enum { TAOS_WAL_FSYNC = 2 } EWalType; +typedef struct SWalReadHead { + int8_t sver; + uint8_t msgType; + int8_t reserved[2]; + int32_t len; + int64_t version; + char cont[]; +} SWalReadHead; + +typedef struct { + int32_t vgId; + int32_t fsyncPeriod; // millisecond + int32_t retentionPeriod; // secs + int32_t rollPeriod; // secs + int64_t segSize; + EWalType walLevel; // wal level +} SWalCfg; + typedef struct { //union { //uint32_t info; @@ -47,25 +65,11 @@ typedef struct { //uint32_t reserved : 24; //}; //}; - int8_t sver; - uint8_t msgType; - int8_t reserved[2]; - int32_t len; - int64_t version; - uint32_t signature; uint32_t cksumHead; uint32_t cksumBody; - char cont[]; + SWalReadHead head; } SWalHead; -typedef struct { - int32_t vgId; - int32_t fsyncPeriod; // millisecond - int32_t rollPeriod; - int64_t segSize; - EWalType walLevel; // wal level -} SWalCfg; - #define WAL_PREFIX "wal" #define WAL_PREFIX_LEN 3 #define WAL_NOSUFFIX_LEN 20 @@ -80,7 +84,7 @@ typedef struct { //#define WAL_FILE_NUM 1 // 3 #define WAL_FILESET_MAX 128 -#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) +#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) #define WAL_CUR_POS_WRITABLE 1 #define WAL_CUR_FILE_WRITABLE 2 #define WAL_CUR_FAILED 4 @@ -103,21 +107,17 @@ typedef struct SWal { //write tfd int64_t writeLogTfd; int64_t writeIdxTfd; - //read tfd - int64_t readLogTfd; - int64_t readIdxTfd; - //current version - int64_t curVersion; //wal lifecycle int64_t firstVersion; int64_t snapshotVersion; int64_t commitVersion; int64_t lastVersion; + //snapshotting version + int64_t snapshottingVer; //roll status int64_t lastRollSeq; //file set int32_t writeCur; - int32_t readCur; SArray* fileInfoSet; //ctl int32_t curStatus; @@ -148,7 +148,8 @@ int32_t walCommit(SWal *, int64_t ver); // truncate after int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely -int32_t walTakeSnapshot(SWal *, int64_t ver); +int32_t walBeginTakeSnapshot(SWal *, int64_t ver); +int32_t walEndTakeSnapshot(SWal *); //int32_t walDataCorrupted(SWal*); // read diff --git a/include/util/tarray.h b/include/util/tarray.h index 25862a7119..ad02b20868 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -153,6 +153,13 @@ void taosArraySet(SArray* pArray, size_t index, void* pData); */ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt); +/** + * remove some data entry from front + * @param pArray + * @param cnt + */ +void taosArrayPopTailBatch(SArray* pArray, size_t cnt); + /** * remove data entry of the given index * @param pArray @@ -213,6 +220,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); */ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); +/** + * search the array, return index of the element + * @param pArray + * @param compar + * @param key + */ +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); + /** * search the array * @param pArray diff --git a/include/util/tchecksum.h b/include/util/tchecksum.h index c1907b0c46..fb6c85edcd 100644 --- a/include/util/tchecksum.h +++ b/include/util/tchecksum.h @@ -39,7 +39,7 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin } static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { - return (checksum == (*crc32c)(0, stream, (size_t)ssize)); + return (checksum != (*crc32c)(0, stream, (size_t)ssize)); } static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) { diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index f56f240904..f8f2e2eadd 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -18,6 +18,7 @@ #include "wal.h" #include "compare.h" +#include "tchecksum.h" #ifdef __cplusplus extern "C" { @@ -32,6 +33,11 @@ typedef struct WalFileInfo { int64_t fileSize; } WalFileInfo; +typedef struct WalIdxEntry { + int64_t ver; + int64_t offset; +} WalIdxEntry; + static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; WalFileInfo* pInfoRight = (WalFileInfo*)pRight; @@ -79,6 +85,26 @@ static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); } +static inline int walValidHeadCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)&pHead->head, sizeof(SWalReadHead), pHead->cksumHead); +} + +static inline int walValidBodyCksum(SWalHead* pHead) { + return taosCheckChecksum((uint8_t*)pHead->head.cont, pHead->head.len, pHead->cksumBody); +} + +static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { + return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); +} + +static inline uint32_t walCalcHeadCksum(SWalHead *pHead) { + return taosCalcChecksum(0, (uint8_t*)&pHead->head, sizeof(SWalReadHead)); +} + +static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) { + return taosCalcChecksum(0, (uint8_t*)body, len); +} + int walReadMeta(SWal* pWal); int walWriteMeta(SWal* pWal); int walRollFileInfo(SWal* pWal); @@ -87,6 +113,10 @@ char* walMetaSerialize(SWal* pWal); int walMetaDeserialize(SWal* pWal, const char* bytes); //meta section end +//seek section +int walChangeFile(SWal *pWal, int64_t ver); +//seek section end + int64_t walGetSeq(); int walSeekVer(SWal *pWal, int64_t ver); int walRoll(SWal *pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 65085bb96d..89588b1ddf 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -24,6 +24,18 @@ #include #include +int64_t walGetFirstVer(SWal *pWal) { + return pWal->firstVersion; +} + +int64_t walGetSnaphostVer(SWal *pWal) { + return pWal->snapshotVersion; +} + +int64_t walGetLastVer(SWal *pWal) { + return pWal->lastVersion; +} + int walRollFileInfo(SWal* pWal) { int64_t ts = taosGetTimestampSec(); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 6bf9008917..9090348c04 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -82,6 +82,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; + pWal->writeCur = -1; //set config pWal->vgId = pCfg->vgId; @@ -90,13 +91,20 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->segSize = pCfg->segSize; pWal->level = pCfg->walLevel; - //init status + //init version info + pWal->firstVersion = -1; + pWal->commitVersion = -1; + pWal->snapshotVersion = -1; pWal->lastVersion = -1; + + pWal->snapshottingVer = -1; + + //init status pWal->lastRollSeq = -1; //init write buffer memset(&pWal->head, 0, sizeof(SWalHead)); - pWal->head.sver = 0; + pWal->head.head.sver = 0; tstrncpy(pWal->path, path, sizeof(pWal->path)); pthread_mutex_init(&pWal->mutex, NULL); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e9f5bcbc5d..27dffddf83 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -15,19 +15,6 @@ #include "walInt.h" #include "tfile.h" -#include "tchecksum.h" - -static inline int walValidHeadCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead, sizeof(SWalHead) - sizeof(uint32_t)*2, pHead->cksumHead); -} - -static inline int walValidBodyCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead->cont, pHead->len, pHead->cksumBody); -} - -static int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { - return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); -} int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int code; @@ -49,13 +36,13 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { if(walValidHeadCksum(*ppHead) != 0) { return -1; } - void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->len); + void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); if(ptr == NULL) { free(*ppHead); *ppHead = NULL; return -1; } - if(tfRead(pWal->writeLogTfd, (*ppHead)->cont, (*ppHead)->len) != (*ppHead)->len) { + if(tfRead(pWal->writeLogTfd, (*ppHead)->head.cont, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } //TODO: endian compatibility processing after read @@ -69,18 +56,3 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) { return 0; } - -int64_t walGetFirstVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->firstVersion; -} - -int64_t walGetSnaphostVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->snapshotVersion; -} - -int64_t walGetLastVer(SWal *pWal) { - if (pWal == NULL) return 0; - return pWal->lastVersion; -} diff --git a/source/libs/wal/src/walIndex.c b/source/libs/wal/src/walSeek.c similarity index 76% rename from source/libs/wal/src/walIndex.c rename to source/libs/wal/src/walSeek.c index bf51be4346..48272f8f32 100644 --- a/source/libs/wal/src/walIndex.c +++ b/source/libs/wal/src/walSeek.c @@ -43,12 +43,35 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { if (code != 0) { return -1; } - /*pWal->curLogOffset = readBuf[1];*/ - pWal->curVersion = ver; return code; } -static int walChangeFile(SWal *pWal, int64_t ver) { +int walChangeFileToLast(SWal *pWal) { + int64_t idxTfd, logTfd; + WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); + ASSERT(pRet != NULL); + int64_t fileFirstVer = pRet->firstVer; + + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, fileFirstVer, fnameStr); + idxTfd = tfOpenReadWrite(fnameStr); + if(idxTfd < 0) { + return -1; + } + walBuildLogName(pWal, fileFirstVer, fnameStr); + logTfd = tfOpenReadWrite(fnameStr); + if(logTfd < 0) { + return -1; + } + //switch file + pWal->writeIdxTfd = idxTfd; + pWal->writeLogTfd = logTfd; + //change status + pWal->curStatus = WAL_CUR_FILE_WRITABLE; + return 0; +} + +int walChangeFile(SWal *pWal, int64_t ver) { int code = 0; int64_t idxTfd, logTfd; char fnameStr[WAL_FILE_LEN]; @@ -86,21 +109,21 @@ static int walChangeFile(SWal *pWal, int64_t ver) { return code; } +int walGetVerOffset(SWal* pWal, int64_t ver) { + int code; + return 0; +} + int walSeekVer(SWal *pWal, int64_t ver) { int code; - if((!(pWal->curStatus & WAL_CUR_FAILED)) && ver == pWal->curVersion) { + if(ver == pWal->lastVersion) { return 0; } - if(ver > pWal->lastVersion) { - //TODO: some records are skipped - return -1; - } - if(ver < pWal->firstVersion) { - //TODO: try to seek pruned log + if(ver > pWal->lastVersion || ver < pWal->firstVersion) { return -1; } if(ver < pWal->snapshotVersion) { - //TODO: seek snapshotted log, invalid in some cases + //TODO: set flag to prevent roll back } if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeFile(pWal, ver); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 0c4989300f..87d52ad236 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -21,65 +21,6 @@ #include "tfile.h" #include "walInt.h" -static void walFtruncate(SWal *pWal, int64_t ver); - -int32_t walCommit(SWal *pWal, int64_t ver) { - ASSERT(pWal->snapshotVersion <= pWal->commitVersion); - ASSERT(pWal->commitVersion <= pWal->lastVersion); - ASSERT(ver >= pWal->commitVersion); - ASSERT(ver <= pWal->lastVersion); - pWal->commitVersion = ver; - return 0; -} - -int32_t walRollback(SWal *pWal, int64_t ver) { - //TODO: ftruncate - ASSERT(ver > pWal->commitVersion); - ASSERT(ver <= pWal->lastVersion); - //seek position - walSeekVer(pWal, ver); - walFtruncate(pWal, ver); - return 0; -} - -int32_t walTakeSnapshot(SWal *pWal, int64_t ver) { - pWal->snapshotVersion = ver; - int ts = taosGetTimestampSec(); - - int deleteCnt = 0; - int64_t newTotSize = pWal->totSize; - WalFileInfo tmp; - tmp.firstVer = ver; - //mark files safe to delete - WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); - //iterate files, until the searched result - for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if(pWal->totSize > pWal->retentionSize || - iter->closeTs + pWal->retentionPeriod > ts) { - //delete according to file size or close time - deleteCnt++; - newTotSize -= iter->fileSize; - } - } - char fnameStr[WAL_FILE_LEN]; - //remove file - for(int i = 0; i < deleteCnt; i++) { - WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); - walBuildLogName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); - walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); - } - - //save snapshot ver, commit ver - - - //make new array, remove files - taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); - pWal->totSize = newTotSize; - - return 0; -} #if 0 static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId); @@ -172,6 +113,185 @@ void walRemoveAllOldFiles(void *handle) { } #endif +int32_t walCommit(SWal *pWal, int64_t ver) { + ASSERT(pWal->commitVersion >= pWal->snapshotVersion); + ASSERT(pWal->commitVersion <= pWal->lastVersion); + if(ver < pWal->commitVersion || ver > pWal->lastVersion) { + return -1; + } + pWal->commitVersion = ver; + return 0; +} + +int32_t walRollback(SWal *pWal, int64_t ver) { + int code; + char fnameStr[WAL_FILE_LEN]; + if(ver == pWal->lastVersion) { + return 0; + } + if(ver > pWal->lastVersion || ver < pWal->commitVersion) { + return -1; + } + pthread_mutex_lock(&pWal->mutex); + + //find correct file + if(ver < walGetLastFileFirstVer(pWal)) { + //close current files + tfClose(pWal->writeIdxTfd); + tfClose(pWal->writeLogTfd); + //open old files + code = walChangeFile(pWal, ver); + if(code != 0) { + return -1; + } + + //delete files + int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); + for(int i = pWal->writeCur; i < fileSetSize; i++) { + walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + remove(fnameStr); + walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + remove(fnameStr); + } + //pop from fileInfoSet + taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); + } + + walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + int64_t idxTfd = tfOpenReadWrite(fnameStr); + + //change to deserialize function + + if(idxTfd < 0) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + int idxOff = (ver - walGetCurFileFirstVer(pWal)) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, idxOff, SEEK_SET); + if(code < 0) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + //read idx file and get log file pos + //TODO:change to deserialize function + WalIdxEntry entry; + if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + ASSERT(entry.ver == ver); + + walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + int64_t logTfd = tfOpenReadWrite(fnameStr); + if(logTfd < 0) { + //TODO + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + code = tfLseek(logTfd, entry.offset, SEEK_SET); + if(code < 0) { + //TODO + pthread_mutex_unlock(&pWal->mutex); + return -1; + } + //validate offset + SWalHead head; + ASSERT(tfValid(logTfd)); + int size = tfRead(logTfd, &head, sizeof(SWalHead)); + if(size != sizeof(SWalHead)) { + return -1; + } + code = walValidHeadCksum(&head); + + ASSERT(code == 0); + if(code != 0) { + return -1; + } + if(head.head.version != ver) { + //TODO + return -1; + } + //truncate old files + code = tfFtruncate(logTfd, entry.offset); + if(code < 0) { + return -1; + } + code = tfFtruncate(idxTfd, idxOff); + if(code < 0) { + return -1; + } + pWal->lastVersion = ver - 1; + ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; + ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + + //unlock + pthread_mutex_unlock(&pWal->mutex); + return 0; +} +int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { + pWal->snapshottingVer = ver; + //check file rolling + if(pWal->retentionPeriod == 0) { + walRoll(pWal); + } + + return 0; +} + +int32_t walEndTakeSnapshot(SWal *pWal) { + int64_t ver = pWal->snapshottingVer; + if(ver == -1) return -1; + + pWal->snapshotVersion = ver; + int ts = taosGetTimestampSec(); + + int deleteCnt = 0; + int64_t newTotSize = pWal->totSize; + WalFileInfo tmp; + tmp.firstVer = ver; + //find files safe to delete + WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); + if(ver >= pInfo->lastVer) { + pInfo++; + } + //iterate files, until the searched result + for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { + if(pWal->totSize > pWal->retentionSize || + iter->closeTs + pWal->retentionPeriod > ts) { + //delete according to file size or close time + deleteCnt++; + newTotSize -= iter->fileSize; + } + } + char fnameStr[WAL_FILE_LEN]; + //remove file + for(int i = 0; i < deleteCnt; i++) { + WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); + walBuildLogName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); + walBuildIdxName(pWal, pInfo->firstVer, fnameStr); + remove(fnameStr); + } + + //make new array, remove files + taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); + if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + pWal->firstVersion = -1; + } else { + pWal->firstVersion = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + } + pWal->totSize = newTotSize; + pWal->snapshottingVer = -1; + + //save snapshot ver, commit ver + int code = walWriteMeta(pWal); + if(code != 0) { + return -1; + } + + return 0; +} + int walRoll(SWal *pWal) { int code = 0; if(pWal->writeIdxTfd != -1) { @@ -211,6 +331,7 @@ int walRoll(SWal *pWal) { //switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; //change status pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE; @@ -218,32 +339,6 @@ int walRoll(SWal *pWal) { return 0; } -int walChangeFileToLast(SWal *pWal) { - int64_t idxTfd, logTfd; - WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); - ASSERT(pRet != NULL); - int64_t fileFirstVer = pRet->firstVer; - - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenReadWrite(fnameStr); - if(idxTfd < 0) { - return -1; - } - walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenReadWrite(fnameStr); - if(logTfd < 0) { - return -1; - } - //switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; - //change status - pWal->curVersion = fileFirstVer; - pWal->curStatus = WAL_CUR_FILE_WRITABLE; - return 0; -} - static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { int code = 0; //get index file @@ -253,9 +348,11 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); return code; } - int64_t writeBuf[2] = { ver, offset }; - int size = tfWrite(pWal->writeIdxTfd, writeBuf, sizeof(writeBuf)); - if(size != sizeof(writeBuf)) { + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); + WalIdxEntry entry = { .ver = ver, .offset = offset }; + int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); + if(size != sizeof(WalIdxEntry)) { return -1; } return 0; @@ -270,13 +367,14 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i if (index == pWal->lastVersion + 1) { if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + pWal->firstVersion = index; code = walRoll(pWal); ASSERT(code == 0); } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(pWal->rollPeriod != -1 && passed > pWal->rollPeriod) { + if(pWal->rollPeriod != -1 && pWal->rollPeriod != 0 && passed > pWal->rollPeriod) { walRoll(pWal); - } else if(pWal->segSize != -1 && walGetLastFileSize(pWal) > pWal->segSize) { + } else if(pWal->segSize != -1 && pWal->segSize != 0 && walGetLastFileSize(pWal) > pWal->segSize) { walRoll(pWal); } } @@ -287,16 +385,13 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } /*if (!tfValid(pWal->curLogTfd)) return 0;*/ - pWal->head.version = index; - - pWal->head.signature = WAL_SIGNATURE; - pWal->head.len = bodyLen; - pWal->head.msgType = msgType; - - pWal->head.cksumHead = taosCalcChecksum(0, (const uint8_t*)&pWal->head, sizeof(SWalHead)- sizeof(uint32_t)*2); - pWal->head.cksumBody = taosCalcChecksum(0, (const uint8_t*)&body, bodyLen); - pthread_mutex_lock(&pWal->mutex); + pWal->head.head.version = index; + + pWal->head.head.len = bodyLen; + pWal->head.head.msgType = msgType; + pWal->head.cksumHead = walCalcHeadCksum(&pWal->head); + pWal->head.cksumBody = walCalcBodyCksum(body, bodyLen); if (tfWrite(pWal->writeLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) { //ftruncate @@ -312,6 +407,7 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); if(code != 0) { //TODO + return -1; } //set status @@ -326,8 +422,6 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } void walFsync(SWal *pWal, bool forceFsync) { - if (pWal == NULL || !tfValid(pWal->writeLogTfd)) return; - if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, walGetCurFileFirstVer(pWal)); if (tfFsync(pWal->writeLogTfd) < 0) { @@ -408,7 +502,7 @@ static int walValidateOffset(SWal* pWal, int64_t ver) { int code = 0; SWalHead *pHead = NULL; code = (int)walRead(pWal, &pHead, ver); - if(pHead->version != ver) { + if(pHead->head.version != ver) { return -1; } return 0; @@ -428,15 +522,6 @@ static int64_t walGetOffset(SWal* pWal, int64_t ver) { return 0; } -static void walFtruncate(SWal *pWal, int64_t ver) { - int64_t tfd = pWal->writeLogTfd; - tfFtruncate(tfd, ver); - tfFsync(tfd); - tfd = pWal->writeIdxTfd; - tfFtruncate(tfd, ver * WAL_IDX_ENTRY_SIZE); - tfFsync(tfd); -} - #if 0 static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { int64_t pos = *offset; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 96258662a1..f244917f77 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -36,6 +36,36 @@ class WalCleanEnv : public ::testing::Test { const char* pathName = "/tmp/wal_test"; }; +class WalCleanDeleteEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } + + static void TearDownTestCase() { + walCleanUp(); + } + + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->walLevel = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; +}; + class WalKeepEnv : public ::testing::Test { protected: static void SetUpTestCase() { @@ -110,40 +140,94 @@ TEST_F(WalCleanEnv, removeOldMeta) { ASSERT(code == 0); } -TEST_F(WalKeepEnv, readOldMeta) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walWriteMeta(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walWriteMeta(pWal); - ASSERT(code == 0); - char*oldss = walMetaSerialize(pWal); +//TEST_F(WalKeepEnv, readOldMeta) { + //int code = walRollFileInfo(pWal); + //ASSERT(code == 0); + //code = walWriteMeta(pWal); + //ASSERT(code == 0); + //code = walRollFileInfo(pWal); + //ASSERT(code == 0); + //code = walWriteMeta(pWal); + //ASSERT(code == 0); + //char*oldss = walMetaSerialize(pWal); - TearDown(); - SetUp(); - code = walReadMeta(pWal); - ASSERT(code == 0); - char* newss = walMetaSerialize(pWal); + //TearDown(); + //SetUp(); + //code = walReadMeta(pWal); + //ASSERT(code == 0); + //char* newss = walMetaSerialize(pWal); - int len = strlen(oldss); - ASSERT_EQ(len, strlen(newss)); - for(int i = 0; i < len; i++) { - EXPECT_EQ(oldss[i], newss[i]); - } -} + //int len = strlen(oldss); + //ASSERT_EQ(len, strlen(newss)); + //for(int i = 0; i < len; i++) { + //EXPECT_EQ(oldss[i], newss[i]); + //} +//} -TEST_F(WalKeepEnv, write) { +TEST_F(WalCleanEnv, write) { const char* ranStr = "tvapq02tcp"; const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { code = walWrite(pWal, i, i+1, (void*)ranStr, len); ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, i); code = walWrite(pWal, i+2, i, (void*)ranStr, len); ASSERT_EQ(code, -1); + ASSERT_EQ(pWal->lastVersion, i); } code = walWriteMeta(pWal); ASSERT_EQ(code, 0); } + +TEST_F(WalCleanEnv, rollback) { + const char* ranStr = "tvapq02tcp"; + const int len = strlen(ranStr); + int code; + for(int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i+1, (void*)ranStr, len); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, i); + } + code = walRollback(pWal, 5); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, 4); + code = walRollback(pWal, 3); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, 2); + code = walWriteMeta(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalCleanDeleteEnv, roll) { + const char* ranStr = "tvapq02tcp"; + const int len = strlen(ranStr); + int code; + int i; + for(i = 0; i < 100; i++) { + code = walWrite(pWal, i, 0, (void*)ranStr, len); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, i); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->commitVersion, i); + } + + walBeginTakeSnapshot(pWal, i-1); + ASSERT_EQ(pWal->snapshottingVer, i-1); + walEndTakeSnapshot(pWal); + ASSERT_EQ(pWal->snapshotVersion, i-1); + ASSERT_EQ(pWal->snapshottingVer, -1); + + code = walWrite(pWal, 5, 0, (void*)ranStr, len); + ASSERT_NE(code, 0); + + for(; i < 200; i++) { + code = walWrite(pWal, i, 0, (void*)ranStr, len); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + ASSERT_EQ(pWal->commitVersion, i); + } + + code = walWriteMeta(pWal); + ASSERT_EQ(code, 0); +} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index ff52477a6f..2191419670 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -241,12 +241,16 @@ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { assert(cnt <= pArray->size); pArray->size = pArray->size - cnt; if(pArray->size == 0) { - pArray->size = 0; return; } memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size); } +void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { + assert(cnt <= pArray->size); + pArray->size = pArray->size - cnt; +} + void taosArrayRemove(SArray* pArray, size_t index) { assert(index < pArray->size); @@ -329,6 +333,11 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); } +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { + void* item = taosArraySearch(pArray, key, comparFn, flags); + return (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; +} + void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { assert(pArray != NULL); qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); From f07045c1d8dbc29712f791153d5f72ba391a2b18 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 11 Dec 2021 17:28:58 +0800 Subject: [PATCH 10/22] update fst search frame --- source/libs/index/inc/index_fst.h | 39 ++++++++++---------- source/libs/index/inc/index_fst_automation.h | 2 +- source/libs/index/src/index_fst.c | 8 ++++ source/libs/index/src/index_fst_automation.c | 2 +- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index b26bd227f8..a1d4962e8b 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -26,10 +26,24 @@ extern "C" { #include "index_fst_counting_writer.h" #include "index_fst_automation.h" - -typedef struct FstNode FstNode; #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) +typedef struct Fst Fst; +typedef struct FstNode FstNode; + +typedef enum { Included, Excluded, Unbounded} FstBound; + +typedef struct FstBoundWithData { + FstSlice data; + FstBound type; +} FstBoundWithData; + +typedef struct FstStreamBuilder { + Fst *fst; + AutomationCtx *aut; + FstBoundWithData *min; + FstBoundWithData *max; +} FstStreamBuilder, FstStreamWithStateBuilder; typedef struct FstRange { uint64_t start; @@ -39,16 +53,9 @@ typedef struct FstRange { typedef enum {GE, GT, LE, LT} RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State; - typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType; -typedef enum { Included, Excluded, Unbounded} FstBound; -typedef struct FstBoundWithData { - FstSlice data; - FstBound type; -} FstBoundWithData; - FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data); bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); @@ -60,8 +67,6 @@ typedef struct FstOutput { Output out; } FstOutput; - - /* * * UnFinished node and helper function @@ -275,6 +280,8 @@ FstNode* fstGetRoot(Fst *fst); FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); Output fstEmptyFinalOutput(Fst *fst, bool *null); +FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); +FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); bool fstVerify(Fst *fst); @@ -298,7 +305,7 @@ typedef struct StreamWithState { FstOutput emptyOutput; SArray *stack; // FstBoundWithData *endAt; -} StreamWithState ; +} StreamWithState; typedef struct StreamWithStateResult { FstSlice data; @@ -314,14 +321,8 @@ typedef void* (*StreamCallback)(void *); StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ; void streamWithStateDestroy(StreamWithState *sws); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); -StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); -typedef struct FstStreamBuilder { - Fst *fst; - AutomationCtx *aut; - FstBoundWithData *min; - FstBoundWithData *max; -} FstStreamBuilder; +StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); // set up bound range diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 4a29f2e3a6..6deeb1878a 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -50,7 +50,7 @@ typedef struct AutomationFunc { } AutomationFunc; AutomationCtx *automCtxCreate(void *data, AutomationType type); -void autoCtxDestroy(AutomationCtx *ctx); +void automCtxDestroy(AutomationCtx *ctx); extern AutomationFunc automFuncs[]; #ifdef __cplusplus diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 18e617f6ae..07f1e343bd 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1090,6 +1090,12 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { return true; } +FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { + return fstStreamBuilderCreate(fst, ctx); +} +FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { + return fstStreamBuilderCreate(fst, ctx); +} FstNode *fstGetRoot(Fst *fst) { if (fst->root != NULL) { @@ -1440,3 +1446,5 @@ FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, Rang + + diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 748c55c29d..392d8e6e8d 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -92,6 +92,6 @@ AutomationCtx* automCtxCreate(void *data, AutomationType type) { } return ctx; } -void autoCtxDestroy(AutomationCtx *ctx) { +void automCtxDestroy(AutomationCtx *ctx) { free(ctx); } From e29bb8b63cc6d8d2e6e80fa178042193eb3cc1a5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 12 Dec 2021 10:46:49 +0800 Subject: [PATCH 11/22] TD-10431 alter stb --- include/common/taosmsg.h | 10 +- source/dnode/mgmt/impl/test/CMakeLists.txt | 1 + .../dnode/mgmt/impl/test/dnode/CMakeLists.txt | 29 ++++ source/dnode/mnode/impl/inc/mndInt.h | 2 + source/dnode/mnode/impl/src/mndStb.c | 133 +++++++++++++++++- source/dnode/mnode/impl/src/mnode.c | 7 + 6 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/dnode/CMakeLists.txt diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 5fbbbb2855..456b06b925 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -320,10 +320,14 @@ typedef struct { typedef struct { char name[TSDB_TABLE_FNAME_LEN]; - // if user specify DROP STABLE, this flag will be set. And an error will be returned if it is not a super table - int8_t supertable; int8_t igNotExists; -} SCMDropTableMsg; +} SDropStbMsg; + +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + int8_t alterType; + SSchema schema; +} SAlterStbMsg; typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index b3b6818a5c..b340029044 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,5 +1,6 @@ # add_subdirectory(acct) # add_subdirectory(cluster) +add_subdirectory(dnode) # add_subdirectory(profile) # add_subdirectory(show) add_subdirectory(user) diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt new file mode 100644 index 0000000000..e118cb8725 --- /dev/null +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTestDnode "") + +target_sources(dndTestDnode + PRIVATE + "dnode.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dndTestDnode + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTestDnode + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dndTestDnode + COMMAND dndTestDnode +) diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index b44d6570fe..e9913803bd 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -88,6 +88,8 @@ void mndSendMsgToMnode(SMnode *pMnode, SRpcMsg *pMsg); void mndSendRedirectMsg(SMnode *pMnode, SRpcMsg *pMsg); void mndSetMsgHandle(SMnode *pMnode, int32_t msgType, MndMsgFp fp); +uint64_t mndGenerateUid(char *name, int32_t len) ; + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9f9308b2fa..ccd7fda24b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -240,7 +240,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre tstrncpy(stbObj.db, pDb->name, TSDB_FULL_DB_NAME_LEN); stbObj.createdTime = taosGetTimestampMs(); stbObj.updateTime = stbObj.createdTime; - stbObj.uid = 1234; + stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); stbObj.version = 1; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; @@ -340,11 +340,138 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { + SSchema *pSchema = &pAlter->schema; + pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = htonl(pSchema->bytes); + + if (pSchema->type <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + return -1; + } + if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + return -1; + } + if (pSchema->bytes <= 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + return -1; + } + if (pSchema->name[0] == 0) { + terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + return -1; + } + + return 0; +} + +static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pOldStb, SStbObj *pNewStb) { return 0; } + +static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SAlterStbMsg *pAlter = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to alter", pAlter->name); + + if (mndCheckAlterStbMsg(pAlter) != 0) { + mError("stb:%s, failed to alter since %s", pAlter->name, terrstr()); + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, pAlter->name); + if (pStb == NULL) { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + mError("stb:%s, failed to alter since %s", pAlter->name, terrstr()); + return -1; + } + + SStbObj stbObj = {0}; + memcpy(&stbObj, pStb, sizeof(SStbObj)); + + int32_t code = mndUpdateStb(pMnode, pMsg, pStb, &stbObj); + mndReleaseStb(pMnode, pStb); + + if (code != 0) { + mError("stb:%s, failed to alter since %s", pAlter->name, tstrerror(code)); + return code; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("stb:%s, failed to drop since %s", pStb->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); + + SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropStbMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("stb:%s, start to drop", pDrop->name); + + SStbObj *pStb = mndAcquireStb(pMnode, pDrop->name); + if (pStb == NULL) { + if (pDrop->igNotExists) { + mDebug("stb:%s, not exist, ignore not exist is set", pDrop->name); + return 0; + } else { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + } + + int32_t code = mndDropStb(pMnode, pMsg, pStb); + mndReleaseStb(pMnode, pStb); + + if (code != 0) { + terrno = code; + mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index c9310809d8..f599c4e3ed 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -430,3 +430,10 @@ void mndProcessWriteMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } void mndProcessSyncMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); } void mndProcessApplyMsg(SMnodeMsg *pMsg) {} + +uint64_t mndGenerateUid(char *name, int32_t len) { + int64_t us = taosGetTimestampUs(); + int32_t hashval = MurmurHash3_32(name, len); + uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; + return x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); +} \ No newline at end of file From 890c4ec831c2ec1d18d021463c344c91ce68bdbb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 12 Dec 2021 12:14:26 +0800 Subject: [PATCH 12/22] TD-10431 dnode mange in mnode module --- include/common/taosmsg.h | 3 +- include/common/tglobal.h | 1 - include/dnode/mgmt/dnode.h | 1 - include/dnode/mnode/mnode.h | 1 - include/util/taoserror.h | 23 ++--- include/util/tdef.h | 2 + source/common/src/tglobal.c | 12 --- source/dnode/mgmt/daemon/src/daemon.c | 1 - source/dnode/mgmt/impl/src/dndDnode.c | 1 - source/dnode/mgmt/impl/src/dndMnode.c | 1 - source/dnode/mgmt/impl/test/sut/deploy.cpp | 1 - source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 107 +++++++++++---------- source/dnode/mnode/impl/src/mnode.c | 3 +- source/util/src/terror.c | 19 ++-- 15 files changed, 82 insertions(+), 96 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 456b06b925..ed3742cac7 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -670,7 +670,6 @@ typedef struct { typedef struct { int32_t statusInterval; - int32_t mnodeEqualVnodeNum; int64_t checkTime; // 1970-01-01 00:00:00.000 char timezone[TSDB_TIMEZONE_LEN]; // tsTimezone char locale[TSDB_LOCALE_LEN]; // tsLocale @@ -869,7 +868,7 @@ typedef struct { typedef struct { int32_t dnodeId; - char config[128]; + char config[TSDB_DNODE_CONFIG_LEN]; } SCfgDnodeMsg; typedef struct { diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e62083e999..aedf4122bb 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -108,7 +108,6 @@ extern int8_t tsEnableBalance; extern int8_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; -extern int32_t tsMnodeEqualVnodeNum; extern int8_t tsEnableFlowCtrl; extern int8_t tsEnableSlaveQuery; extern int8_t tsEnableAdjustMaster; diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 540a501f3a..fbe447baf9 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -33,7 +33,6 @@ typedef struct { int16_t numOfSupportQnodes; int8_t enableTelem; int32_t statusInterval; - int32_t mnodeEqualVnodeNum; float numOfThreadsPerCore; float ratioOfQueryCores; int32_t maxShellConns; diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 8a8346fbc0..f4a6f7a4c9 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -47,7 +47,6 @@ typedef struct SMnodeCfg { int32_t sver; int8_t enableTelem; int32_t statusInterval; - int32_t mnodeEqualVnodeNum; int32_t shellActivityTimer; char *timezone; char *locale; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 1232e2e31f..64a1fa84ad 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -152,22 +152,23 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x032A) #define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x032B) -#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) //"DNode already exists") -#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) //"DNode does not exist") +// mnode-dnode +#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) +#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) +#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0332) +#define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0333) +#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0334) +#define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0335) +#define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0336) +#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0337) + +// mnode-vgroup #define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0332) //"VGroup does not exist") -#define TSDB_CODE_MND_NO_REMOVE_MASTER TAOS_DEF_ERROR_CODE(0, 0x0333) //"Master DNode cannot be removed") -#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0334) //"Out of DNodes") -#define TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT TAOS_DEF_ERROR_CODE(0, 0x0335) //"Cluster cfg inconsistent") -#define TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION TAOS_DEF_ERROR_CODE(0, 0x0336) //"Invalid dnode cfg option") -#define TSDB_CODE_MND_BALANCE_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0337) //"Balance already enabled") #define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0338) //"Vgroup not in dnode") #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0339) //"Vgroup already in dnode") -#define TSDB_CODE_MND_DNODE_NOT_FREE TAOS_DEF_ERROR_CODE(0, 0x033A) //"Dnode not avaliable") -#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x033B) //"Cluster id not match") #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) //"Cluster not ready") -#define TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033D) //"Dnode Id not configured") -#define TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033E) //"Dnode Ep not configured") +// mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) //"Account already exists") #define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341) //"Invalid account") #define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0342) //"Invalid account options") diff --git a/include/util/tdef.h b/include/util/tdef.h index 3ab515671a..28a1e24581 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -209,6 +209,8 @@ do { \ #define TSDB_STEP_NAME_LEN 32 #define TSDB_STEP_DESC_LEN 128 +#define TSDB_DNODE_CONFIG_LEN 128 + #define TSDB_MQTT_HOSTNAME_LEN 64 #define TSDB_MQTT_PORT_LEN 8 #define TSDB_MQTT_USER_LEN 24 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3cf5e52c44..b385db181f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -156,7 +156,6 @@ int8_t tsEnableBalance = 1; int8_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds int32_t tsOfflineThreshold = 86400 * 10; // seconds of 10 days -int32_t tsMnodeEqualVnodeNum = 4; int8_t tsEnableFlowCtrl = 1; int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; @@ -1072,17 +1071,6 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); - - // module configs - cfg.option = "mnodeEqualVnodeNum"; - cfg.ptr = &tsMnodeEqualVnodeNum; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); // module configs diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index cf9a960062..82201df072 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -142,7 +142,6 @@ void dmnInitOption(SDnodeOpt *pOption) { pOption->numOfSupportVnodes = 1; pOption->numOfSupportQnodes = 1; pOption->statusInterval = tsStatusInterval; - pOption->mnodeEqualVnodeNum = tsMnodeEqualVnodeNum; pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; pOption->ratioOfQueryCores = tsRatioOfQueryCores; pOption->maxShellConns = tsMaxShellConns; diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index c89741d03b..9786381b7c 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -357,7 +357,6 @@ static void dndSendStatusMsg(SDnode *pDnode) { tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN); pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval); - pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(pDnode->opt.mnodeEqualVnodeNum); pStatus->clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index d3336fb079..7eb6ee980a 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -334,7 +334,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->cfg.sver = pDnode->opt.sver; pOption->cfg.enableTelem = pDnode->opt.enableTelem; pOption->cfg.statusInterval = pDnode->opt.statusInterval; - pOption->cfg.mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum; pOption->cfg.shellActivityTimer = pDnode->opt.shellActivityTimer; pOption->cfg.timezone = pDnode->opt.timezone; pOption->cfg.charset = pDnode->opt.charset; diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 92b0967345..565f7a21ba 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -57,7 +57,6 @@ void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t pOption->numOfSupportVnodes = 1; pOption->numOfSupportQnodes = 1; pOption->statusInterval = 1; - pOption->mnodeEqualVnodeNum = 1; pOption->numOfThreadsPerCore = 1; pOption->ratioOfQueryCores = 1; pOption->maxShellConns = 1000; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b2e3ecff55..aace3f4ee3 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -92,7 +92,6 @@ typedef enum { DND_REASON_VERSION_NOT_MATCH, DND_REASON_DNODE_ID_NOT_MATCH, DND_REASON_CLUSTER_ID_NOT_MATCH, - DND_REASON_MN_EQUAL_VN_NOT_MATCH, DND_REASON_STATUS_INTERVAL_NOT_MATCH, DND_REASON_TIME_ZONE_NOT_MATCH, DND_REASON_LOCALE_NOT_MATCH, @@ -125,6 +124,7 @@ typedef struct SDnodeObj { int64_t createdTime; int64_t updateTime; int64_t rebootTime; + int64_t lastAccessTime; int32_t accessTimes; int16_t numOfMnodes; int16_t numOfVnodes; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index dc9f54888f..5d7cf8c700 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -22,10 +22,13 @@ #include "tutil.h" #define TSDB_DNODE_VER 1 +#define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_CONFIG_OPTION_LEN 16 #define TSDB_CONIIG_VALUE_LEN 48 #define TSDB_CONFIG_NUMBER 8 +static int32_t id = 2; + static const char *offlineReason[] = { "", "status msg timeout", @@ -33,7 +36,6 @@ static const char *offlineReason[] = { "version not match", "dnodeId not match", "clusterId not match", - "mnEqualVn not match", "interval not match", "timezone not match", "locale not match", @@ -117,6 +119,7 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { SDB_SET_INT64(pRaw, dataPos, pDnode->updateTime) SDB_SET_INT16(pRaw, dataPos, pDnode->port) SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -142,28 +145,28 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDnode->updateTime) SDB_GET_INT16(pRaw, pRow, dataPos, &pDnode->port) SDB_GET_BINARY(pRaw, pRow, dataPos, pDnode->fqdn, TSDB_FQDN_LEN) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_DNODE_RESERVE_SIZE) return pRow; } -static void mnodeResetDnode(SDnodeObj *pDnode) { +static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { + mTrace("dnode:%d, perform insert action", pDnode->id); + pDnode->rebootTime = 0; + pDnode->lastAccessTime = 0; pDnode->accessTimes = 0; - pDnode->numOfCores = 0; pDnode->numOfMnodes = 0; pDnode->numOfVnodes = 0; pDnode->numOfQnodes = 0; pDnode->numOfSupportMnodes = 0; pDnode->numOfSupportVnodes = 0; pDnode->numOfSupportQnodes = 0; + pDnode->numOfCores = 0; pDnode->status = DND_STATUS_OFFLINE; pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); -} -static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { - mTrace("dnode:%d, perform insert action", pDnode->id); - mnodeResetDnode(pDnode); return 0; } @@ -174,11 +177,6 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { mTrace("dnode:%d, perform update action", pOldDnode->id); - pOldDnode->id = pNewDnode->id; - pOldDnode->createdTime = pNewDnode->createdTime; - pOldDnode->updateTime = pNewDnode->updateTime; - pOldDnode->port = pNewDnode->port; - memcpy(pOldDnode->fqdn, pNewDnode->fqdn, TSDB_FQDN_LEN); return 0; } @@ -232,6 +230,7 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { if (pIter == NULL) break; if (i >= numOfEps) { sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pDnode); break; } @@ -244,20 +243,15 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { pEp->isMnode = 1; } i++; + sdbRelease(pSdb, pDnode); } pEps->num = htonl(i); } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { - if (pCfg->mnodeEqualVnodeNum != pMnode->cfg.mnodeEqualVnodeNum) { - mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg inconsistent", pCfg->mnodeEqualVnodeNum, - pMnode->cfg.mnodeEqualVnodeNum); - return DND_REASON_MN_EQUAL_VN_NOT_MATCH; - } - if (pCfg->statusInterval != pMnode->cfg.statusInterval) { - mError("\"statusInterval\"[%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); + mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } @@ -265,18 +259,18 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) { - mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, + mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, pCfg->checkTime, checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) { - mError("\"locale\"[%s - %s] cfg parameters inconsistent", pCfg->locale, pMnode->cfg.locale); + mError("locale [%s - %s] cfg inconsistent", pCfg->locale, pMnode->cfg.locale); return DND_REASON_LOCALE_NOT_MATCH; } if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) { - mError("\"charset\"[%s - %s] cfg parameters inconsistent.", pCfg->charset, pMnode->cfg.charset); + mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, pMnode->cfg.charset); return DND_REASON_CHARSET_NOT_MATCH; } @@ -292,9 +286,7 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->numOfSupportMnodes = htons(pStatus->numOfSupportMnodes); pStatus->numOfSupportVnodes = htons(pStatus->numOfSupportVnodes); pStatus->numOfSupportQnodes = htons(pStatus->numOfSupportQnodes); - pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval); - pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(pStatus->clusterCfg.mnodeEqualVnodeNum); pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); } @@ -308,7 +300,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp); if (pDnode == NULL) { mDebug("dnode:%s, not created yet", pStatus->dnodeEp); - return TSDB_CODE_MND_DNODE_NOT_EXIST; + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; } } else { pDnode = mndAcquireDnode(pMnode, pStatus->dnodeId); @@ -319,7 +312,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_MND_DNODE_NOT_EXIST; + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; } } @@ -329,7 +323,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mndReleaseDnode(pMnode, pDnode); mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); - return TSDB_CODE_MND_INVALID_MSG_VERSION; + terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; + return -1; } if (pStatus->dnodeId == 0) { @@ -341,7 +336,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mError("dnode:%d, clusterId %d not match exist %d", pDnode->id, pStatus->clusterId, pMnode->clusterId); mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_MND_INVALID_CLUSTER_ID; + terrno != TSDB_CODE_MND_INVALID_CLUSTER_ID; + return -1; } else { pDnode->accessTimes++; mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); @@ -355,7 +351,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode->offlineReason = ret; mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[ret]); mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT; + terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG; + return -1; } mInfo("dnode:%d, from offline to online", pDnode->id); @@ -366,6 +363,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode->numOfSupportMnodes = pStatus->numOfSupportMnodes; pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes; pDnode->numOfSupportQnodes = pStatus->numOfSupportQnodes; + pDnode->lastAccessTime = taosGetTimestampMs(); pDnode->status = DND_STATUS_READY; int32_t numOfEps = mndGetDnodeSize(pMnode); @@ -373,7 +371,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { SStatusRsp *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { mndReleaseDnode(pMnode, pDnode); - return TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); @@ -390,13 +389,13 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { SDnodeObj dnodeObj = {0}; - dnodeObj.id = 1; // todo + dnodeObj.id = id++; dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); if (dnodeObj.fqdn[0] == 0 || dnodeObj.port <= 0) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_MND_INVALID_DNODE_EP; mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); return terrno; } @@ -449,7 +448,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { mDebug("dnode:%s, start to create", pCreate->ep); if (pCreate->ep[0] == 0) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_MND_INVALID_DNODE_EP; mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); return -1; } @@ -457,7 +456,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, pCreate->ep); if (pDnode != NULL) { mError("dnode:%d, already exist", pDnode->id); - sdbRelease(pMnode->pSdb, pDnode); + mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; return -1; } @@ -478,7 +477,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); return -1; } - mDebug("trans:%d, used to drop user:%d", pTrans->id, pDnode->id); + mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { @@ -522,26 +521,26 @@ static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { mDebug("dnode:%d, start to drop", pDrop->dnodeId); if (pDrop->dnodeId <= 0) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_MND_INVALID_DNODE_ID; mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDrop->dnodeId); if (pDnode == NULL) { - mError("dnode:%d, not exist", pDrop->dnodeId); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; - } - - int32_t code = mndDropDnode(pMnode, pMsg, pDnode); - - if (code != 0) { mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } - sdbRelease(pMnode->pSdb, pDnode); + int32_t code = mndDropDnode(pMnode, pMsg, pDnode); + if (code != 0) { + mndReleaseDnode(pMnode, pDnode); + mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + mndReleaseDnode(pMnode, pDnode); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -553,7 +552,7 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - mError("dnode:%d, failed to cfg since %s ", pCfg->dnodeId, terrstr()); + mError("dnode:%d, failed to config since %s ", pCfg->dnodeId, terrstr()); return -1; } @@ -562,17 +561,22 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { SCfgDnodeMsg *pCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg)); pCfgDnode->dnodeId = htonl(pCfg->dnodeId); - memcpy(pCfgDnode->config, pCfg->config, 128); + memcpy(pCfgDnode->config, pCfg->config, TSDB_DNODE_CONFIG_LEN); - SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, .pCont = pCfgDnode, .contLen = sizeof(SCfgDnodeMsg)}; + SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, + .pCont = pCfgDnode, + .contLen = sizeof(SCfgDnodeMsg), + .ahandle = pMsg->rpcMsg.ahandle}; - mInfo("dnode:%d, is configured", pCfg->dnodeId); + mInfo("dnode:%d, app:%p config:%s req send to dnode", pCfg->dnodeId, rpcMsg.ahandle, pCfg->config); mndSendMsgToDnode(pMnode, &epSet, &rpcMsg); return 0; } -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { mInfo("cfg dnode rsp is received"); } +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { + mInfo("app:%p config rsp from dnode", pMsg->rpcMsg.ahandle); +} static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { int32_t cols = 0; @@ -600,7 +604,6 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->numOfRows = TSDB_CONFIG_NUMBER; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pIter = NULL; strcpy(pMeta->tableFname, mndShowStr(pShow->type)); return 0; @@ -676,7 +679,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "cores"); + strcpy(pSchema[cols].name, "max vnodes"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -740,7 +743,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDnode->numOfCores; + *(int16_t *)pWrite = pDnode->numOfSupportVnodes; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index f599c4e3ed..c83fd9124c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -205,7 +205,6 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->cfg.sver = pOption->cfg.sver; pMnode->cfg.enableTelem = pOption->cfg.enableTelem; pMnode->cfg.statusInterval = pOption->cfg.statusInterval; - pMnode->cfg.mnodeEqualVnodeNum = pOption->cfg.mnodeEqualVnodeNum; pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer; pMnode->cfg.timezone = strdup(pOption->cfg.timezone); pMnode->cfg.locale = strdup(pOption->cfg.locale); @@ -215,7 +214,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { if (pMnode->sendMsgToDnodeFp == NULL || pMnode->sendMsgToMnodeFp == NULL || pMnode->sendRedirectMsgFp == NULL || pMnode->putMsgToApplyMsgFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || - pMnode->cfg.statusInterval < 1 || pOption->cfg.mnodeEqualVnodeNum < 0) { + pMnode->cfg.statusInterval < 1) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b0960ed3b0..ceb344ffe8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -162,22 +162,23 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_VER, "Invalid raw data vers TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_LEN, "Invalid raw data len") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_CONTENT, "Invalid raw data content") +// mnode-dnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "DNode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "DNode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, "Master DNode cannot be removed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of DNodes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, "Cluster cfg inconsistent") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, "Invalid dnode cfg option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, "Balance already enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_CFG, "Cluster cfg inconsistent") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG, "Invalid dnode cfg") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP, "Invalid dnode end point") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID, "Invalid dnode id") + +// mnode-vgroup +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, "Dnode not avaliable") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED, "Dnode Id not configured") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED, "Dnode Ep not configured") +// mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_NOT_EXIST, "Invalid account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, "Invalid account options") From 442a8461adb9ee6b19c4f1ea4d0bcc723d9fbae8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 12 Dec 2021 14:46:31 +0800 Subject: [PATCH 13/22] TD-10431 dnode mange in dnode module --- include/common/taosmsg.h | 4 +- source/dnode/mgmt/impl/inc/dndInt.h | 3 +- source/dnode/mgmt/impl/src/dndDnode.c | 29 +- source/dnode/mgmt/impl/src/dnode.c | 4 +- .../dnode/mgmt/impl/test/cluster/cluster.cpp | 2 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 440 ++++++++++++++++++ .../dnode/mgmt/impl/test/profile/profile.cpp | 6 +- source/dnode/mgmt/impl/test/show/show.cpp | 2 +- source/dnode/mgmt/impl/test/sut/deploy.cpp | 9 +- source/dnode/mgmt/impl/test/sut/deploy.h | 2 +- source/dnode/mgmt/impl/test/user/user.cpp | 5 +- source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 6 +- source/dnode/mnode/impl/src/mndFunc.c | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 2 +- source/dnode/mnode/impl/src/mndUser.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- 19 files changed, 490 insertions(+), 38 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/dnode/dnode.cpp diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index ed3742cac7..6f25d0fb87 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -695,7 +695,7 @@ typedef struct SStatusMsg { int32_t sver; int32_t dnodeId; int32_t clusterId; - uint32_t rebootTime; // time stamp for last reboot + int64_t rebootTime; // time stamp for last reboot int16_t numOfCores; int16_t numOfSupportMnodes; int16_t numOfSupportVnodes; @@ -803,7 +803,7 @@ typedef struct { } SVgroupsMsg, SVgroupsInfo; typedef struct { - char tableFname[TSDB_TABLE_FNAME_LEN]; // table id + char tbFname[TSDB_TABLE_FNAME_LEN]; // table id char stbFname[TSDB_TABLE_FNAME_LEN]; int32_t numOfTags; int32_t numOfColumns; diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 136f6eee0c..f65810feab 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -58,7 +58,8 @@ typedef struct { int32_t dnodeId; int32_t dropped; int32_t clusterId; - uint32_t rebootTime; + int64_t rebootTime; + int8_t statusSent; SEpSet mnodeEpSet; char *file; SHashObj *dnodeHash; diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 9786381b7c..b948dc8ce4 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -349,7 +349,7 @@ static void dndSendStatusMsg(SDnode *pDnode) { pStatus->sver = htonl(pDnode->opt.sver); pStatus->dnodeId = htonl(pMgmt->dnodeId); pStatus->clusterId = htonl(pMgmt->clusterId); - pStatus->rebootTime = htonl(pMgmt->rebootTime); + pStatus->rebootTime = htobe64(pMgmt->rebootTime); pStatus->numOfCores = htons(pDnode->opt.numOfCores); pStatus->numOfSupportMnodes = htons(pDnode->opt.numOfCores); pStatus->numOfSupportVnodes = htons(pDnode->opt.numOfCores); @@ -370,6 +370,7 @@ static void dndSendStatusMsg(SDnode *pDnode) { contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad); SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS}; + pMgmt->statusSent = 1; dndSendMsgToMnode(pDnode, &rpcMsg); } @@ -382,7 +383,7 @@ static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { pMgmt->dnodeId = pCfg->dnodeId; pMgmt->clusterId = pCfg->clusterId; pMgmt->dropped = pCfg->dropped; - (void)dndWriteDnodes(pDnode); + dndWriteDnodes(pDnode); taosWUnLockLatch(&pMgmt->latch); } } @@ -408,11 +409,16 @@ static void dndUpdateDnodeEps(SDnode *pDnode, SDnodeEps *pDnodeEps) { } static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + SDnodeMgmt *pMgmt = &pDnode->dmgmt; + if (pEpSet && pEpSet->numOfEps > 0) { dndUpdateMnodeEpSet(pDnode, pEpSet); } - if (pMsg->code != TSDB_CODE_SUCCESS) return; + if (pMsg->code != TSDB_CODE_SUCCESS) { + pMgmt->statusSent = 0; + return; + } SStatusRsp *pRsp = pMsg->pCont; SDnodeCfg *pCfg = &pRsp->dnodeCfg; @@ -420,7 +426,10 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { pCfg->clusterId = htonl(pCfg->clusterId); dndUpdateDnodeCfg(pDnode, pCfg); - if (pCfg->dropped) return; + if (pCfg->dropped) { + pMgmt->statusSent = 0; + return; + } SDnodeEps *pDnodeEps = &pRsp->dnodeEps; pDnodeEps->num = htonl(pDnodeEps->num); @@ -430,6 +439,7 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { } dndUpdateDnodeEps(pDnode, pDnodeEps); + pMgmt->statusSent = 0; } static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); } @@ -460,16 +470,17 @@ static void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) { } static void *dnodeThreadRoutine(void *param) { - SDnode *pDnode = param; - int32_t ms = pDnode->opt.statusInterval * 1000; + SDnode *pDnode = param; + SDnodeMgmt *pMgmt = &pDnode->dmgmt; + int32_t ms = pDnode->opt.statusInterval * 1000; while (true) { - taosMsleep(ms); pthread_testcancel(); - if (dndGetStat(pDnode) == DND_STAT_RUNNING) { + if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent) { dndSendStatusMsg(pDnode); } + taosMsleep(ms); } } @@ -477,7 +488,7 @@ int32_t dndInitDnode(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; pMgmt->dnodeId = 0; - pMgmt->rebootTime = taosGetTimestampSec(); + pMgmt->rebootTime = taosGetTimestampMs(); pMgmt->dropped = 0; pMgmt->clusterId = 0; diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 1159bbb4c5..90e8fe7d54 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -61,15 +61,15 @@ static int32_t dndCheckRunning(char *dataDir) { FileFd fd = taosOpenFileCreateWriteTrunc(filepath); if (fd < 0) { - dError("failed to lock file:%s since %s, quit", filepath, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); return -1; } int32_t ret = taosLockFile(fd); if (ret != 0) { - dError("failed to lock file:%s since %s, quit", filepath, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s, quit", filepath, terrstr()); taosCloseFile(fd); return -1; } diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index 4e71a8016c..99614ec921 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -80,7 +80,7 @@ TEST_F(DndTestCluster, ShowCluster) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, "show cluster"); + EXPECT_STREQ(pMeta->tbFname, "show cluster"); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp new file mode 100644 index 0000000000..1632438b5c --- /dev/null +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestDnode : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} + + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } + + static void SetUpTestSuite() { + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9521"; + pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); + pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp); + pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); + pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); + pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9521); + } + + static void TearDownTestSuite() { + dropServer(pServer1); + dropServer(pServer2); + dropServer(pServer3); + dropServer(pServer4); + dropServer(pServer5); + dropClient(pClient); + } + + static SServer* pServer1; + static SServer* pServer2; + static SServer* pServer3; + static SServer* pServer4; + static SServer* pServer5; + static SClient* pClient; + + void CheckShowMsg(int8_t msgType) { + + } +}; + +SServer* DndTestDnode::pServer1; +SServer* DndTestDnode::pServer2; +SServer* DndTestDnode::pServer3; +SServer* DndTestDnode::pServer4; +SServer* DndTestDnode::pServer5; +SClient* DndTestDnode::pClient; + +TEST_F(DndTestDnode, ShowDnode) { + int32_t showId = 0; + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_DNODE; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htons(pMeta->numOfTags); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, "show dnodes"); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, 7); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + + pSchema = &pMeta->pSchema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pSchema->bytes, 2); + EXPECT_STREQ(pSchema->name, "id"); + + pSchema = &pMeta->pSchema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_EP_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "end point"); + + pSchema = &pMeta->pSchema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pSchema->bytes, 2); + EXPECT_STREQ(pSchema->name, "vnodes"); + + pSchema = &pMeta->pSchema[3]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pSchema->bytes, 2); + EXPECT_STREQ(pSchema->name, "max vnodes"); + + pSchema = &pMeta->pSchema[4]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, 10 + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "status"); + + pSchema = &pMeta->pSchema[5]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "create time"); + + pSchema = &pMeta->pSchema[6]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "offline reason"); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, 1); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + EXPECT_EQ(pRetrieveRsp->completed, 1); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + int64_t int64Val = 0; + int16_t int16Val = 0; + + //--- name --- + { + int16Val = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(int16Val, 1); + } + + // //--- privilege --- + // { + // pos += sizeof(VarDataLenT); + // strVal = (char*)(pData + pos); + // pos += 10; + // EXPECT_STREQ(strVal, "super"); + + // pos += sizeof(VarDataLenT); + // strVal = (char*)(pData + pos); + // pos += 10; + // EXPECT_STREQ(strVal, "writable"); + // } + + // //--- create_time --- + // { + // int64Val = *((int64_t*)(pData + pos)); + // pos += sizeof(int64_t); + // EXPECT_GT(int64Val, 0); + + // int64Val = *((int64_t*)(pData + pos)); + // pos += sizeof(int64_t); + // EXPECT_GT(int64Val, 0); + // } + + // //--- account --- + // { + // pos += sizeof(VarDataLenT); + // strVal = (char*)(pData + pos); + // pos += TSDB_USER_LEN; + // EXPECT_STREQ(strVal, "root"); + + // pos += sizeof(VarDataLenT); + // strVal = (char*)(pData + pos); + // pos += TSDB_USER_LEN; + // EXPECT_STREQ(strVal, "root"); + // } +} + +#if 0 +TEST_F(DndTestDnode, CreateUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- create user --- + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 3); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "u1"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} + +TEST_F(DndTestDnode, AlterUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- drop user --- + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p2"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 3); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "u1"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} + +TEST_F(DndTestDnode, DropUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- drop user --- + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + strcpy(pReq->user, "u1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 2); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} + +#endif \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index a4632547a2..cb93f1572f 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -139,7 +139,7 @@ TEST_F(DndTestProfile, SConnectMsg_03) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); @@ -480,7 +480,7 @@ TEST_F(DndTestProfile, SKillQueryMsg_03) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); @@ -667,7 +667,7 @@ TEST_F(DndTestProfile, SKillStreamMsg_03) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index 266f5858e4..2208de1b59 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -141,7 +141,7 @@ TEST_F(DndTestShow, SShowMsg_04) { EXPECT_NE(pRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, ""); + EXPECT_STREQ(pMeta->tbFname, ""); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 565f7a21ba..d0431313cd 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -50,7 +50,7 @@ void* runServer(void* param) { } } -void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port) { +void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port, const char* firstEp) { pOption->sver = 1; pOption->numOfCores = 1; pOption->numOfSupportMnodes = 1; @@ -65,16 +65,16 @@ void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t strcpy(pOption->dataDir, path); snprintf(pOption->localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); snprintf(pOption->localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(pOption->firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(pOption->firstEp, TSDB_EP_LEN, "%s", firstEp); } -SServer* createServer(const char* path, const char* fqdn, uint16_t port) { +SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { taosRemoveDir(path); taosMkDir(path); initLog(path); SDnodeOpt option = {0}; - initOption(&option, path, fqdn, port); + initOption(&option, path, fqdn, port, firstEp); SDnode* pDnode = dndInit(&option); ASSERT(pDnode); @@ -98,7 +98,6 @@ void dropServer(SServer* pServer) { void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SClient* pClient = (SClient*)parent; pClient->pRsp = pMsg; - // taosMsleep(1000000); tsem_post(&pClient->sem); } diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index 93188458ec..5c804b56d3 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -39,7 +39,7 @@ typedef struct { tsem_t sem; } SClient; -SServer* createServer(const char* path, const char* fqdn, uint16_t port); +SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); void dropServer(SServer* pServer); SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); void dropClient(SClient* pClient); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 36f801fec2..9e75e620e7 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -26,8 +26,9 @@ class DndTestUser : public ::testing::Test { const char* path = "/tmp/dndTestUser"; const char* fqdn = "localhost"; uint16_t port = 9524; + const char* firstEp = "localhost:9524"; - pServer = createServer(path, fqdn, port); + pServer = createServer(path, fqdn, port, firstEp); ASSERT(pServer); pClient = createClient(user, pass, fqdn, port); } @@ -79,7 +80,7 @@ TEST_F(DndTestUser, ShowUser) { EXPECT_NE(pShowRsp->showId, 0); EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tableFname, "show users"); + EXPECT_STREQ(pMeta->tbFname, "show users"); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->precision, 0); EXPECT_EQ(pMeta->tableType, 0); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index c5c71d8751..7edeb73647 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -159,7 +159,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg cols++; pMeta->numOfColumns = htons(cols); - strcpy(pMeta->tableFname, "show cluster"); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 7477213890..fd02c6e251 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -767,7 +767,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe pShow->numOfRows = sdbGetSize(pSdb, SDB_DB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5d7cf8c700..ebf9d0e083 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -281,7 +281,7 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->sver = htonl(pStatus->sver); pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->clusterId = htonl(pStatus->clusterId); - pStatus->rebootTime = htonl(pStatus->rebootTime); + pStatus->rebootTime = htobe64(pStatus->rebootTime); pStatus->numOfCores = htons(pStatus->numOfCores); pStatus->numOfSupportMnodes = htons(pStatus->numOfSupportMnodes); pStatus->numOfSupportVnodes = htons(pStatus->numOfSupportVnodes); @@ -604,7 +604,7 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->numOfRows = TSDB_CONFIG_NUMBER; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -711,7 +711,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index d2d538b973..e407b271fd 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -429,7 +429,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, "show funcs"); + strcpy(pMeta->tbFname, "show funcs"); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 8a05eb02e8..6278e3ffef 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -405,7 +405,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ccd7fda24b..c3afbf37c8 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -600,7 +600,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index fc1f441a90..2e4719cceb 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -489,7 +489,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a0af5f7f5e..abd0d6a96b 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -238,7 +238,7 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -337,7 +337,7 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->replica = dnodeId; pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } From 6941b148780303140d701abe42b1710ec561f8c4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 12 Dec 2021 20:43:29 +0800 Subject: [PATCH 14/22] TD-10431 fix crash while test dnode create --- include/dnode/mnode/sdb/sdb.h | 2 +- source/common/src/tglobal.c | 4 ++-- source/dnode/mgmt/impl/inc/dndDnode.h | 1 + source/dnode/mgmt/impl/src/dndDnode.c | 2 +- source/dnode/mgmt/impl/src/dndTransport.c | 7 +++---- source/dnode/mgmt/impl/src/dnode.c | 1 + source/dnode/mgmt/impl/test/dnode/dnode.cpp | 3 +++ source/dnode/mgmt/impl/test/sut/deploy.cpp | 7 +++++-- source/dnode/mnode/impl/src/mndVgroup.c | 2 +- source/libs/transport/src/rpcMain.c | 17 ++++++++++++----- 10 files changed, 30 insertions(+), 16 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 36b1e41978..7e7afc9774 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -158,7 +158,7 @@ typedef enum { SDB_USER = 5, SDB_AUTH = 6, SDB_ACCT = 7, - SDB_VGROUP = 9, + SDB_VGROUP = 8, SDB_STB = 9, SDB_DB = 10, SDB_FUNC = 11, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b385db181f..7e71c6bfc7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -46,7 +46,7 @@ int64_t tsDnodeStartTime = 0; // common int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 0; //disable this, means query, show command use udp protocol as default +int32_t tsRpcForceTcp = 1; //disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second @@ -1583,7 +1583,7 @@ static void doInitGlobalConfig(void) { taosInitConfigOption(cfg); assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); #else - assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); + //assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif } diff --git a/source/dnode/mgmt/impl/inc/dndDnode.h b/source/dnode/mgmt/impl/inc/dndDnode.h index c21c6a0b86..27cc99c27c 100644 --- a/source/dnode/mgmt/impl/inc/dndDnode.h +++ b/source/dnode/mgmt/impl/inc/dndDnode.h @@ -31,6 +31,7 @@ int32_t dndGetClusterId(SDnode *pDnode); void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet); void dndSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg); +void dndSendStatusMsg(SDnode *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index b948dc8ce4..2cdebab6bf 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -335,7 +335,7 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { return 0; } -static void dndSendStatusMsg(SDnode *pDnode) { +void dndSendStatusMsg(SDnode *pDnode) { int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); SStatusMsg *pStatus = rpcMallocCont(contLen); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 09c207bac7..245a1e41f6 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -130,7 +130,7 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (dndGetStat(pDnode) == DND_STAT_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; - dTrace("RPC %p, rsp:%s app:%p is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pMsg->handle, taosMsg[msgType]); rpcFreeCont(pMsg->pCont); return; } @@ -138,10 +138,9 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { DndMsgFp fp = pMgmt->msgFp[msgType]; if (fp != NULL) { (*fp)(pDnode, pMsg, pEpSet); - dTrace("RPC %p, rsp:%s app:%p is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->ahandle, - pMsg->code & 0XFFFF); + dTrace("RPC %p, rsp:%s is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->code & 0XFFFF); } else { - dError("RPC %p, rsp:%s app:%p not processed", pMsg->handle, taosMsg[msgType], pMsg->ahandle); + dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]); } rpcFreeCont(pMsg->pCont); } diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 90e8fe7d54..a5b118e67b 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -194,6 +194,7 @@ SDnode *dndInit(SDnodeOpt *pOption) { } dndSetStat(pDnode, DND_STAT_RUNNING); + dndSendStatusMsg(pDnode); dndReportStartup(pDnode, "TDengine", "initialized successfully"); dInfo("TDengine is initialized successfully"); diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 1632438b5c..d2a242d797 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -80,6 +80,8 @@ TEST_F(DndTestDnode, ShowDnode) { sendMsg(pClient, &showRpcMsg); ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; ASSERT_NE(pShowRsp, nullptr); @@ -170,6 +172,7 @@ TEST_F(DndTestDnode, ShowDnode) { sendMsg(pClient, &retrieveRpcMsg); ASSERT_NE(pClient->pRsp, nullptr); ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index d0431313cd..0048b7c4ad 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -16,7 +16,7 @@ #include "deploy.h" void initLog(const char* path) { - dDebugFlag = 0; + dDebugFlag = 207; vDebugFlag = 0; mDebugFlag = 207; cDebugFlag = 0; @@ -90,6 +90,7 @@ SServer* createServer(const char* path, const char* fqdn, uint16_t port, const c } void dropServer(SServer* pServer) { + if (pServer == NULL) return; if (pServer->threadId != NULL) { taosDestoryThread(pServer->threadId); } @@ -98,6 +99,8 @@ void dropServer(SServer* pServer) { void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SClient* pClient = (SClient*)parent; pClient->pRsp = pMsg; + uInfo("response:%s from dnode, pCont:%p contLen:%d code:0x%X", taosMsg[pMsg->msgType], pMsg->pCont, pMsg->contLen, + pMsg->code); tsem_post(&pClient->sem); } @@ -143,7 +146,7 @@ void sendMsg(SClient* pClient, SRpcMsg* pMsg) { epSet.inUse = 0; epSet.numOfEps = 1; epSet.port[0] = pClient->port; - strcpy(epSet.fqdn[0], pClient->fqdn); + memcpy(epSet.fqdn[0], pClient->fqdn, TSDB_FQDN_LEN); rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); tsem_wait(&pClient->sem); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index abd0d6a96b..44d13d1fb4 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -45,7 +45,7 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_VGROUP, - .keyType = SDB_KEY_BINARY, + .keyType = SDB_KEY_INT32, .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, .insertFp = (SdbInsertFp)mndVgroupActionInsert, diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index ce110ede32..fb69a74876 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -126,6 +126,8 @@ typedef struct SRpcConn { SRpcReqContext *pContext; // request context } SRpcConn; +static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; + int tsRpcMaxUdpSize = 15000; // bytes int tsProgressTimer = 100; // not configurable @@ -220,17 +222,22 @@ static void rpcFree(void *p) { free(p); } -int32_t rpcInit(void) { - tsProgressTimer = tsRpcTimer/2; - tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer; - tsRpcHeadSize = RPC_MSG_OVERHEAD; +static void rpcInitImp(void) { + tsProgressTimer = tsRpcTimer / 2; + tsRpcMaxRetry = tsRpcMaxTime * 1000 / tsProgressTimer; + tsRpcHeadSize = RPC_MSG_OVERHEAD; tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); return 0; } - + +int32_t rpcInit(void) { + pthread_once(&tsRpcInitOnce, rpcInitImp); + return 0; +} + void rpcCleanup(void) { taosCloseRef(tsRpcRefId); tsRpcRefId = -1; From 6be2787796f12a60fe14d95a6c0e99c973e67817 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 12 Dec 2021 21:55:43 +0800 Subject: [PATCH 15/22] TD-10431 refact dnode test --- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 403 ++++++++------------ source/dnode/mgmt/impl/test/sut/deploy.cpp | 1 - source/dnode/mgmt/impl/test/sut/deploy.h | 3 + source/dnode/mnode/impl/src/mndDnode.c | 6 +- 4 files changed, 173 insertions(+), 240 deletions(-) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index d2a242d797..dd8ba40b4f 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -16,10 +16,7 @@ #include "deploy.h" class DndTestDnode : public ::testing::Test { - protected: - void SetUp() override {} - void TearDown() override {} - + public: static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { SServer* pServer = createServer(path, fqdn, port, firstEp); ASSERT(pServer); @@ -27,14 +24,17 @@ class DndTestDnode : public ::testing::Test { } static void SetUpTestSuite() { + initLog("/tmp/dndTestDnode1"); + const char* fqdn = "localhost"; const char* firstEp = "localhost:9521"; pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp); - pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); - pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); - pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + // pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); + // pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); + // pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); pClient = createClient("root", "taosdata", fqdn, 9521); + taosMsleep(300); } static void TearDownTestSuite() { @@ -53,9 +53,127 @@ class DndTestDnode : public ::testing::Test { static SServer* pServer5; static SClient* pClient; - void CheckShowMsg(int8_t msgType) { + public: + void SetUp() override {} + void TearDown() override {} + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htons(pMeta->numOfTags); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + EXPECT_EQ(pRetrieveRsp->completed, 1); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; }; SServer* DndTestDnode::pServer1; @@ -66,250 +184,59 @@ SServer* DndTestDnode::pServer5; SClient* DndTestDnode::pClient; TEST_F(DndTestDnode, ShowDnode) { - int32_t showId = 0; + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max vnodes"); + CheckSchema(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); + CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline reason"); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_DNODE; - strcpy(pShow->db, ""); - - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, "show dnodes"); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, 7); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_SMALLINT); - EXPECT_EQ(pSchema->bytes, 2); - EXPECT_STREQ(pSchema->name, "id"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_EP_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "end point"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_SMALLINT); - EXPECT_EQ(pSchema->bytes, 2); - EXPECT_STREQ(pSchema->name, "vnodes"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_SMALLINT); - EXPECT_EQ(pSchema->bytes, 2); - EXPECT_STREQ(pSchema->name, "max vnodes"); - - pSchema = &pMeta->pSchema[4]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, 10 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "status"); - - pSchema = &pMeta->pSchema[5]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "create time"); - - pSchema = &pMeta->pSchema[6]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "offline reason"); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, 1); - EXPECT_EQ(pRetrieveRsp->offset, 0); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - EXPECT_EQ(pRetrieveRsp->completed, 1); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - int64_t int64Val = 0; - int16_t int16Val = 0; - - //--- name --- - { - int16Val = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(int16Val, 1); - } - - // //--- privilege --- - // { - // pos += sizeof(VarDataLenT); - // strVal = (char*)(pData + pos); - // pos += 10; - // EXPECT_STREQ(strVal, "super"); - - // pos += sizeof(VarDataLenT); - // strVal = (char*)(pData + pos); - // pos += 10; - // EXPECT_STREQ(strVal, "writable"); - // } - - // //--- create_time --- - // { - // int64Val = *((int64_t*)(pData + pos)); - // pos += sizeof(int64_t); - // EXPECT_GT(int64Val, 0); - - // int64Val = *((int64_t*)(pData + pos)); - // pos += sizeof(int64_t); - // EXPECT_GT(int64Val, 0); - // } - - // //--- account --- - // { - // pos += sizeof(VarDataLenT); - // strVal = (char*)(pData + pos); - // pos += TSDB_USER_LEN; - // EXPECT_STREQ(strVal, "root"); - - // pos += sizeof(VarDataLenT); - // strVal = (char*)(pData + pos); - // pos += TSDB_USER_LEN; - // EXPECT_STREQ(strVal, "root"); - // } + SendThenCheckShowRetrieveMsg(1); + CheckInt16(1); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(1); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckBinary("", 24); } -#if 0 -TEST_F(DndTestDnode, CreateUser_01) { - ASSERT_NE(pClient, nullptr); - - //--- create user --- - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p1"); +TEST_F(DndTestDnode, CreateDnode_01) { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9522"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; sendMsg(pClient, &rpcMsg); SRpcMsg* pMsg = pClient->pRsp; ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 3); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "u1"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } + taosMsleep(1300); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + SendThenCheckShowRetrieveMsg(2); + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9522", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(0); + CheckInt16(1); + CheckInt16(1); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("", 24); + CheckBinary("", 24); } +#if 0 TEST_F(DndTestDnode, AlterUser_01) { ASSERT_NE(pClient, nullptr); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 0048b7c4ad..0aa01ed0d0 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -71,7 +71,6 @@ void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { taosRemoveDir(path); taosMkDir(path); - initLog(path); SDnodeOpt option = {0}; initOption(&option, path, fqdn, port, firstEp); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index 5c804b56d3..d3b911640d 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -39,8 +39,11 @@ typedef struct { tsem_t sem; } SClient; +void initLog(const char* path); SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); void dropServer(SServer* pServer); SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); void dropClient(SClient* pClient); void sendMsg(SClient* pClient, SRpcMsg* pMsg); + +// class DndTest \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ebf9d0e083..cf46d044ca 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -756,7 +756,11 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); + if (pDnode->status == DND_STATUS_READY) { + STR_TO_VARSTR(pWrite, ""); + } else { + STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); + } cols++; numOfRows++; From 29aabe5da11aea7619f3761a558d5ae38c58a1e9 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 12 Dec 2021 10:02:34 -0500 Subject: [PATCH 16/22] TD-11819 mock catalog function for parser ut --- source/libs/parser/inc/insertParser.h | 8 + source/libs/parser/src/insertParser.c | 12 +- source/libs/parser/test/insertTest.cpp | 88 ++++++++ source/libs/parser/test/mockCatalog.cpp | 52 +++++ source/libs/parser/test/mockCatalog.h | 27 +++ .../libs/parser/test/mockCatalogService.cpp | 188 ++++++++++++++++++ source/libs/parser/test/mockCatalogService.h | 63 ++++++ source/libs/parser/test/parserMain.cpp | 41 ++++ source/libs/parser/test/tokenizerTest.cpp | 5 - 9 files changed, 470 insertions(+), 14 deletions(-) create mode 100644 source/libs/parser/test/insertTest.cpp create mode 100644 source/libs/parser/test/mockCatalog.cpp create mode 100644 source/libs/parser/test/mockCatalog.h create mode 100644 source/libs/parser/test/mockCatalogService.cpp create mode 100644 source/libs/parser/test/mockCatalogService.h create mode 100644 source/libs/parser/test/parserMain.cpp diff --git a/source/libs/parser/inc/insertParser.h b/source/libs/parser/inc/insertParser.h index a2a910fcca..b0191b155d 100644 --- a/source/libs/parser/inc/insertParser.h +++ b/source/libs/parser/inc/insertParser.h @@ -16,8 +16,16 @@ #ifndef TDENGINE_INSERTPARSER_H #define TDENGINE_INSERTPARSER_H +#ifdef __cplusplus +extern "C" { +#endif + #include "parser.h" int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_INSERTPARSER_H diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 0d9d6d7580..33b02f4261 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -145,16 +145,10 @@ static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, b } static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) { - SInsertStmtInfo *info = calloc(1, sizeof(SQueryStmtInfo)); - if (NULL == info) { + *pInsertInfo = calloc(1, sizeof(SQueryStmtInfo)); + if (NULL == *pInsertInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - // info->pTableBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - // if (NULL == info->pTableBlockHashList) { - // tfree(info); - // return TSDB_CODE_TSC_OUT_OF_MEMORY; - // } - *pInsertInfo = info; return TSDB_CODE_SUCCESS; } @@ -808,7 +802,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // no data in the sql string anymore. if (sToken.n == 0) { if (0 == pCxt->totalNum) { - return TSDB_CODE_TSC_INVALID_OPERATION; + return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");; } break; } diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp new file mode 100644 index 0000000000..08ca515b17 --- /dev/null +++ b/source/libs/parser/test/insertTest.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "insertParser.h" +#include "mockCatalog.h" + +using namespace std; +using namespace testing; + +namespace { + string toString(int32_t code) { + return tstrerror(code); + } +} + +// syntax: +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +class InsertTest : public Test { +protected: + void bind(const char* sql) { + reset(); + cxt.pSql = sql; + cxt.sqlLen = strlen(sql); + } + + int32_t run() { + code = parseInsertSql(&cxt, &res); + if (code != TSDB_CODE_SUCCESS) { + cout << "code:" << toString(code) << ", msg:" << errMagBuf << endl; + } + return code; + } + + SInsertStmtInfo* reslut() { + return res; + } + +private: + static const int max_err_len = 1024; + + void reset() { + memset(&cxt, 0, sizeof(cxt)); + memset(errMagBuf, 0, max_err_len); + cxt.pMsg = errMagBuf; + cxt.msgLen = max_err_len; + code = TSDB_CODE_SUCCESS; + res = nullptr; + } + + char errMagBuf[max_err_len]; + SParseContext cxt; + int32_t code; + SInsertStmtInfo* res; +}; + +// INSERT INTO tb_name VALUES (field1_value, ...) +TEST_F(InsertTest, simpleTest) { + bind("insert into .. values (...)"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + SInsertStmtInfo* res = reslut(); + // todo check +} + +TEST_F(InsertTest, toleranceTest) { + bind("insert into"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); + bind("insert into t"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); +} diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp new file mode 100644 index 0000000000..fc988512c9 --- /dev/null +++ b/source/libs/parser/test/mockCatalog.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mockCatalog.h" + +#include + +void generateMetaData(MockCatalogService* mcs) { + { + ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, MockCatalogService::numOfDataTypes) + .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP); + for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) { + if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) { + continue; + } + builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type); + } + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, MockCatalogService::numOfDataTypes, 2) + .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(2).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP); + for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) { + if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) { + continue; + } + builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type); + } + builder.done(); + } + mcs->showTables(); +} + +struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { + return mockCatalogService->getCatalogHandle(pMgmtEps); +} + +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { + return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); +} diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h new file mode 100644 index 0000000000..32001d19fd --- /dev/null +++ b/source/libs/parser/test/mockCatalog.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef MOCK_CATALOG_H +#define MOCK_CATALOG_H + +#include "mockCatalogService.h" + +void generateMetaData(MockCatalogService* mcs); + +// mock +struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); + +#endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp new file mode 100644 index 0000000000..de4fa8ee7e --- /dev/null +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mockCatalogService.h" + +#include +#include +#include + +#include "ttypes.h" + +std::unique_ptr mockCatalogService; + +class TableBuilder : public ITableBuilder { +public: + virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { + assert(colIndex_ < meta_->tableInfo.numOfTags + meta_->tableInfo.numOfColumns); + SSchema* col = meta_->schema + colIndex_; + col->type = type; + col->colId = colIndex_++; + col->bytes = bytes; + strcpy(col->name, name.c_str()); + return *this; + } + + virtual TableBuilder& setVgid(int16_t vgid) { + meta_->vgId = vgid; + return *this; + } + + virtual TableBuilder& setPrecision(uint8_t precision) { + meta_->tableInfo.precision = precision; + return *this; + } + + virtual void done() { + meta_->tableInfo.rowSize = rowsize_; + } + +private: + friend class MockCatalogServiceImpl; + + static std::unique_ptr createTableBuilder(int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { + STableMeta* meta = (STableMeta*)std::calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); + if (nullptr == meta) { + throw std::bad_alloc(); + } + meta->tableType = tableType; + meta->tableInfo.numOfTags = numOfTags; + meta->tableInfo.numOfColumns = numOfColumns; + return std::unique_ptr(new TableBuilder(meta)); + } + + TableBuilder(STableMeta* meta) : colIndex_(0), rowsize_(0), meta_(meta) { + } + + STableMeta* table() { + return meta_; + } + + int32_t colIndex_; + int32_t rowsize_; + STableMeta* meta_; +}; + +class MockCatalogServiceImpl { +public: + static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); + + MockCatalogServiceImpl() { + } + + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { + return (struct SCatalog*)0x01; + } + + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { + return 0; + } + + TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { + builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); + meta_[db][tbname].reset(builder_->table()); + meta_[db][tbname]->uid = id_++; + return *(builder_.get()); + } + + void showTables() const { + // number of forward fills + #define NOF(n) ((n) / 2) + // number of backward fills + #define NOB(n) ((n) % 2 ? (n) / 2 + 1 : (n) / 2) + // center aligned + #define CA(n, s) std::setw(NOF((n) - (s).length())) << "" << (s) << std::setw(NOB((n) - (s).length())) << "" << "|" + // string field length + #define SFL 20 + // string field header + #define SH(h) CA(SFL, std::string(h)) + // string field + #define SF(n) CA(SFL, n) + // integer field length + #define IFL 10 + // integer field header + #define IH(i) CA(IFL, std::string(i)) + // integer field + #define IF(i) CA(IFL, std::to_string(i)) + // split line + #define SL(sn, in) std::setfill('=') << std::setw((sn) * (SFL + 1) + (in) * (IFL + 1)) << "" << std::setfill(' ') + + for (const auto& db : meta_) { + std::cout << SH("Database") << SH("Table") << SH("Type") << SH("Precision") << IH(std::string("Vgid")) << std::endl; + std::cout << SL(4, 1) << std::endl; + for (const auto& table : db.second) { + std::cout << SF(db.first) << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; + // int16_t numOfFields = table.second->tableInfo.numOfTags + table.second->tableInfo.numOfColumns; + // for (int16_t i = 0; i < numOfFields; ++i) { + // const SSchema* schema = table.second->schema + i; + // std::cout << schema->name << " " << schema->type << " " << schema->bytes << std::endl; + // } + } + } + } + +private: + std::string ttToString(int8_t tableType) const { + switch (tableType) { + case TSDB_SUPER_TABLE: + return "super table"; + case TSDB_CHILD_TABLE: + return "child table"; + case TSDB_NORMAL_TABLE: + return "normal table"; + default: + return "unknown"; + } + } + + std::string pToString(uint8_t precision) const { + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + return "millisecond"; + case TSDB_TIME_PRECISION_MICRO: + return "microsecond"; + case TSDB_TIME_PRECISION_NANO: + return "nanosecond"; + default: + return "unknown"; + } + } + + uint64_t id_; + std::unique_ptr builder_; + std::map > > meta_; +}; + +MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { +} + +MockCatalogService::~MockCatalogService() { +} + +struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) { + return impl_->getCatalogHandle(pMgmtEps); +} + +int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { + return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); +} + +ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { + return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); +} + +void MockCatalogService::showTables() const { + impl_->showTables(); +} \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h new file mode 100644 index 0000000000..ba974ab200 --- /dev/null +++ b/source/libs/parser/test/mockCatalogService.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef MOCK_CATALOG_SERVICE_H +#define MOCK_CATALOG_SERVICE_H + +#include +#include + +#include "catalog.h" + +class ITableBuilder { +public: + ITableBuilder& addTag(const std::string& name, int8_t type) { + return addColumn(name, type, tDataTypes[type].bytes); + } + + ITableBuilder& addTag(const std::string& name, int8_t type, int32_t bytes) { + return addColumn(name, type, bytes); + } + + ITableBuilder& addColumn(const std::string& name, int8_t type) { + return addColumn(name, type, tDataTypes[type].bytes); + } + + virtual ITableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) = 0; + virtual ITableBuilder& setVgid(int16_t vgid) = 0; + virtual ITableBuilder& setPrecision(uint8_t precision) = 0; + virtual void done() = 0; +}; + +class MockCatalogServiceImpl; + +class MockCatalogService { +public: + static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); + + MockCatalogService(); + ~MockCatalogService(); + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); + ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); + void showTables() const; + +private: + std::unique_ptr impl_; +}; + +extern std::unique_ptr mockCatalogService; + +#endif // MOCK_CATALOG_SERVICE_H diff --git a/source/libs/parser/test/parserMain.cpp b/source/libs/parser/test/parserMain.cpp new file mode 100644 index 0000000000..5ec304a006 --- /dev/null +++ b/source/libs/parser/test/parserMain.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include + +#include "mockCatalog.h" + +class ParserEnv : public testing::Environment { +public: + virtual void SetUp() { + mockCatalogService.reset(new MockCatalogService()); + generateMetaData(mockCatalogService.get()); + } + + virtual void TearDown() { + mockCatalogService.reset(); + } + + ParserEnv() {} + virtual ~ParserEnv() {} +}; + +int main(int argc, char* argv[]) { + testing::AddGlobalTestEnvironment(new ParserEnv()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 07ba46427f..032a93cd32 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -79,11 +79,6 @@ static void _init_tvariant_nchar(SVariant* t) { t->nLen = twcslen(t->wpz); } -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - TEST(testCase, validateToken_test) { char t01[] = "abc"; EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS); From fac8ac88b9962a263849548d78aaa9f0b8899b9c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 12 Dec 2021 23:22:26 +0800 Subject: [PATCH 17/22] update fst search frame --- source/libs/index/inc/index_fst_automation.h | 10 +++++++++ source/libs/index/src/index_fst_automation.c | 23 ++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 6deeb1878a..c2ab61bf5f 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -38,8 +38,18 @@ typedef struct Complement { // automation typedef struct AutomationCtx { AutomationType type; + void *data; } AutomationCtx; + + +typedef enum StartWithStateKind { Done, Running } StartWithStateKind; + +typedef struct StartWithStateValue { + StartWithStateKind kind; + void *value; +} StartWithStateValue; + typedef struct AutomationFunc { void* (*start)(AutomationCtx *ctx) ; bool (*isMatch)(AutomationCtx *ctx, void *); diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 392d8e6e8d..6a08b41b12 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -17,8 +17,9 @@ // prefix query, impl later -static void* prefixStart(AutomationCtx *ctx) { - return NULL; +static void* prefixStart(AutomationCtx *ctx) { + StartWithStateValue *data = (StartWithStateValue *)(ctx->data); + return data; }; static bool prefixIsMatch(AutomationCtx *ctx, void *data) { return true; @@ -82,16 +83,24 @@ AutomationCtx* automCtxCreate(void *data, AutomationType type) { AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); if (ctx == NULL) { return NULL; } - ctx->type = type; - if (ctx->type == AUTOMATION_PREFIX) { - - } else if (ctx->type == AUTMMATION_MATCH) { - + if (type == AUTOMATION_PREFIX) { + StartWithStateValue *swsv = (StartWithStateValue *)calloc(1, sizeof(StartWithStateValue)); + swsv->kind = Done; + swsv->value = NULL; + ctx->data = (void *)swsv; + } else if (type == AUTMMATION_MATCH) { + } else { // add more search type } + + ctx->type = type; return ctx; } void automCtxDestroy(AutomationCtx *ctx) { + if (ctx->type == AUTOMATION_PREFIX) { + free(ctx->data); + } else if (ctx->type == AUTMMATION_MATCH) { + } free(ctx); } From b2083ea62febf483b425895b5c7849aad40270c2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 13 Dec 2021 13:58:54 +0800 Subject: [PATCH 18/22] fix memory leak --- include/libs/wal/wal.h | 1 + include/util/tarray.h | 7 +++ source/libs/wal/src/walMeta.c | 16 ++++-- source/libs/wal/src/walMgmt.c | 15 ++++-- source/libs/wal/src/walWrite.c | 14 ++--- source/libs/wal/test/walMetaTest.cpp | 77 +++++++++++++++++++--------- source/util/src/tarray.c | 23 ++++++--- 7 files changed, 104 insertions(+), 49 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index fe3ee17d00..48c775292a 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -52,6 +52,7 @@ typedef struct { int32_t fsyncPeriod; // millisecond int32_t retentionPeriod; // secs int32_t rollPeriod; // secs + int32_t retentionSize; // secs int64_t segSize; EWalType walLevel; // wal level } SWalCfg; diff --git a/include/util/tarray.h b/include/util/tarray.h index ad02b20868..e0f14dcd25 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -43,6 +43,13 @@ typedef struct SArray { */ void* taosArrayInit(size_t size, size_t elemSize); +/** + * + * @param tsize + * @return + */ +int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); + /** * * @param pArray diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 89588b1ddf..705cf051be 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -46,6 +46,7 @@ int walRollFileInfo(SWal* pWal) { pInfo->closeTs = ts; } + //TODO: change to emplace back WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo)); if(pNewInfo == NULL) { return -1; @@ -56,12 +57,13 @@ int walRollFileInfo(SWal* pWal) { pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; taosArrayPush(pWal->fileInfoSet, pNewInfo); + free(pNewInfo); return 0; } char* walMetaSerialize(SWal* pWal) { char buf[30]; - if(pWal == NULL || pWal->fileInfoSet == NULL) return 0; + ASSERT(pWal->fileInfoSet); int sz = pWal->fileInfoSet->size; cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); @@ -103,7 +105,9 @@ char* walMetaSerialize(SWal* pWal) { sprintf(buf, "%" PRId64, pInfo->fileSize); cJSON_AddStringToObject(pField, "fileSize", buf); } - return cJSON_Print(pRoot); + char* serialized = cJSON_Print(pRoot); + cJSON_Delete(pRoot); + return serialized; } int walMetaDeserialize(SWal* pWal, const char* bytes) { @@ -123,7 +127,8 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pFiles = cJSON_GetObjectItem(pRoot, "files"); int sz = cJSON_GetArraySize(pFiles); //deserialize - SArray* pArray = taosArrayInit(sz, sizeof(WalFileInfo)); + SArray* pArray = pWal->fileInfoSet; + taosArrayEnsureCap(pArray, sz); WalFileInfo *pData = pArray->pData; for(int i = 0; i < sz; i++) { cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); @@ -141,6 +146,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { } taosArraySetSize(pArray, sz); pWal->fileInfoSet = pArray; + cJSON_Delete(pRoot); return 0; } @@ -171,6 +177,8 @@ static int walFindCurMetaVer(SWal* pWal) { break; } } + closedir(dir); + regfree(&walMetaRegexPattern); return metaVer; } @@ -195,6 +203,7 @@ int walWriteMeta(SWal* pWal) { walBuildMetaName(pWal, metaVer, fnameStr); remove(fnameStr); } + free(serialized); return 0; } @@ -215,6 +224,7 @@ int walReadMeta(SWal* pWal) { if(buf == NULL) { return -1; } + memset(buf, 0, size+5); int tfd = tfOpenRead(fnameStr); if(tfRead(tfd, buf, size) != size) { free(buf); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 9090348c04..a47f27f142 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -80,6 +80,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } + memset(pWal, 0, sizeof(SWal)); pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; pWal->writeCur = -1; @@ -89,6 +90,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fsyncPeriod = pCfg->fsyncPeriod; pWal->rollPeriod = pCfg->rollPeriod; pWal->segSize = pCfg->segSize; + pWal->retentionSize = pCfg->retentionSize; + pWal->retentionPeriod = pCfg->retentionPeriod; pWal->level = pCfg->walLevel; //init version info @@ -99,6 +102,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->snapshottingVer = -1; + pWal->totSize = 0; + //init status pWal->lastRollSeq = -1; @@ -122,6 +127,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walFreeObj(pWal); return NULL; } + walReadMeta(pWal); wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); @@ -153,9 +159,12 @@ void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); tfClose(pWal->writeLogTfd); + pWal->writeLogTfd = -1; tfClose(pWal->writeIdxTfd); - /*taosArrayDestroy(pWal->fileInfoSet);*/ - /*pWal->fileInfoSet = NULL;*/ + pWal->writeIdxTfd = -1; + walWriteMeta(pWal); + taosArrayDestroy(pWal->fileInfoSet); + pWal->fileInfoSet = NULL; pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } @@ -165,7 +174,7 @@ static int32_t walInitObj(SWal *pWal) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - pWal->fileInfoSet = taosArrayInit(0, sizeof(WalFileInfo)); + pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); if(pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 87d52ad236..972ac5c682 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -228,6 +228,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { pthread_mutex_unlock(&pWal->mutex); return 0; } + int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { pWal->snapshottingVer = ver; //check file rolling @@ -276,10 +277,12 @@ int32_t walEndTakeSnapshot(SWal *pWal) { //make new array, remove files taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + pWal->writeCur = -1; pWal->firstVersion = -1; } else { pWal->firstVersion = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;; pWal->totSize = newTotSize; pWal->snapshottingVer = -1; @@ -340,19 +343,10 @@ int walRoll(SWal *pWal) { } static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - int code = 0; - //get index file - if(!tfValid(pWal->writeIdxTfd)) { - code = TAOS_SYSTEM_ERROR(errno); - - wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); - return code; - } - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); WalIdxEntry entry = { .ver = ver, .offset = offset }; int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); if(size != sizeof(WalIdxEntry)) { + //TODO truncate return -1; } return 0; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index f244917f77..c81343d03b 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -18,12 +18,15 @@ class WalCleanEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; pCfg->walLevel = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -49,11 +52,13 @@ class WalCleanDeleteEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; pCfg->walLevel = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -77,13 +82,22 @@ class WalKeepEnv : public ::testing::Test { walCleanUp(); } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; pCfg->walLevel = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -124,6 +138,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); char*ss = walMetaSerialize(pWal); printf("%s\n", ss); + free(ss); code = walWriteMeta(pWal); ASSERT(code == 0); } @@ -140,29 +155,38 @@ TEST_F(WalCleanEnv, removeOldMeta) { ASSERT(code == 0); } -//TEST_F(WalKeepEnv, readOldMeta) { - //int code = walRollFileInfo(pWal); - //ASSERT(code == 0); - //code = walWriteMeta(pWal); - //ASSERT(code == 0); - //code = walRollFileInfo(pWal); - //ASSERT(code == 0); - //code = walWriteMeta(pWal); - //ASSERT(code == 0); - //char*oldss = walMetaSerialize(pWal); +TEST_F(WalKeepEnv, readOldMeta) { + walResetEnv(); + const char* ranStr = "tvapq02tcp"; + int len = strlen(ranStr); + int code; - //TearDown(); - //SetUp(); - //code = walReadMeta(pWal); - //ASSERT(code == 0); - //char* newss = walMetaSerialize(pWal); + for(int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i+1, (void*)ranStr, len); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->lastVersion, i); + code = walWrite(pWal, i+2, i, (void*)ranStr, len); + ASSERT_EQ(code, -1); + ASSERT_EQ(pWal->lastVersion, i); + } + char* oldss = walMetaSerialize(pWal); - //int len = strlen(oldss); - //ASSERT_EQ(len, strlen(newss)); - //for(int i = 0; i < len; i++) { - //EXPECT_EQ(oldss[i], newss[i]); - //} -//} + TearDown(); + SetUp(); + + ASSERT_EQ(pWal->firstVersion, 0); + ASSERT_EQ(pWal->lastVersion, 9); + + char* newss = walMetaSerialize(pWal); + + len = strlen(oldss); + ASSERT_EQ(len, strlen(newss)); + for(int i = 0; i < len; i++) { + EXPECT_EQ(oldss[i], newss[i]); + } + free(oldss); + free(newss); +} TEST_F(WalCleanEnv, write) { const char* ranStr = "tvapq02tcp"; @@ -228,6 +252,9 @@ TEST_F(WalCleanDeleteEnv, roll) { ASSERT_EQ(pWal->commitVersion, i); } - code = walWriteMeta(pWal); + //code = walWriteMeta(pWal); + code = walBeginTakeSnapshot(pWal, i - 1); + ASSERT_EQ(code, 0); + code = walEndTakeSnapshot(pWal); ASSERT_EQ(code, 0); } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 2191419670..6216188496 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -58,24 +58,31 @@ static int32_t taosArrayResize(SArray* pArray) { return 0; } -void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { - if (pArray == NULL || pData == NULL) { - return NULL; - } - - if (pArray->size + nEles > pArray->capacity) { +int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { + if (newCap > pArray->capacity) { size_t tsize = (pArray->capacity << 1u); - while (pArray->size + nEles > tsize) { + while (newCap > tsize) { tsize = (tsize << 1u); } pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); if (pArray->pData == NULL) { - return NULL; + return -1; } pArray->capacity = tsize; } + return 0; +} + +void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { + if (pArray == NULL || pData == NULL) { + return NULL; + } + + if(taosArrayEnsureCap(pArray, pArray->size + nEles) != 0){ + return NULL; + } void* dst = TARRAY_GET_ELEM(pArray, pArray->size); memcpy(dst, pData, pArray->elemSize * nEles); From b67074c094ab0908ea3ae680a7a2a8760ff4cf91 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 13 Dec 2021 14:48:10 +0800 Subject: [PATCH 19/22] TD-10431 create dnode test --- source/dnode/mgmt/impl/src/dndDnode.c | 9 +- source/dnode/mgmt/impl/src/dnode.c | 2 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 162 ++++++++++++-------- source/dnode/mgmt/impl/test/sut/deploy.cpp | 6 +- source/dnode/mgmt/impl/test/sut/deploy.h | 2 - source/dnode/mnode/impl/src/mndShow.c | 6 +- 6 files changed, 116 insertions(+), 71 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 2cdebab6bf..508a40922f 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -281,9 +281,8 @@ PRASE_DNODE_OVER: if (pMgmt->dnodeEps == NULL) { pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp)); pMgmt->dnodeEps->num = 1; - pMgmt->dnodeEps->eps[0].isMnode = 1; - pMgmt->dnodeEps->eps[0].port = pDnode->opt.serverPort; - tstrncpy(pMgmt->dnodeEps->eps[0].fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN); + pMgmt->dnodeEps->eps[0].isMnode = 1; + taosGetFqdnPortFromEp(pDnode->opt.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); } dndResetDnodes(pDnode, pMgmt->dnodeEps); @@ -371,6 +370,8 @@ void dndSendStatusMsg(SDnode *pDnode) { SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_STATUS}; pMgmt->statusSent = 1; + + dTrace("pDnode:%p, send status msg to mnode", pDnode); dndSendMsgToMnode(pDnode, &rpcMsg); } @@ -476,11 +477,11 @@ static void *dnodeThreadRoutine(void *param) { while (true) { pthread_testcancel(); + taosMsleep(ms); if (dndGetStat(pDnode) == DND_STAT_RUNNING && !pMgmt->statusSent) { dndSendStatusMsg(pDnode); } - taosMsleep(ms); } } diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index a5b118e67b..7f51979246 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -196,7 +196,7 @@ SDnode *dndInit(SDnodeOpt *pOption) { dndSetStat(pDnode, DND_STAT_RUNNING); dndSendStatusMsg(pDnode); dndReportStartup(pDnode, "TDengine", "initialized successfully"); - dInfo("TDengine is initialized successfully"); + dInfo("TDengine is initialized successfully, pDnode:%p", pDnode); return pDnode; } diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index dd8ba40b4f..02285de2a4 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -24,15 +24,15 @@ class DndTestDnode : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dndTestDnode1"); + initLog("/tmp/dndTestDnode"); const char* fqdn = "localhost"; const char* firstEp = "localhost:9521"; pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp); - // pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); - // pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); - // pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); + pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); + pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); pClient = createClient("root", "taosdata", fqdn, 9521); taosMsleep(300); } @@ -58,7 +58,6 @@ class DndTestDnode : public ::testing::Test { void TearDown() override {} void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { - //--- meta --- SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); pShow->type = showType; strcpy(pShow->db, ""); @@ -108,7 +107,7 @@ class DndTestDnode : public ::testing::Test { EXPECT_STREQ(pSchema->name, name); } - void SendThenCheckShowRetrieveMsg(int32_t rows) { + void SendThenCheckShowRetrieveMsg(int32_t rows, int32_t completed) { SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); pRetrieve->showId = htonl(showId); pRetrieve->free = 0; @@ -134,7 +133,7 @@ class DndTestDnode : public ::testing::Test { EXPECT_EQ(pRetrieveRsp->numOfRows, rows); EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); - EXPECT_EQ(pRetrieveRsp->completed, 1); + EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); EXPECT_EQ(pRetrieveRsp->reserved, 0); @@ -193,7 +192,7 @@ TEST_F(DndTestDnode, ShowDnode) { CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline reason"); - SendThenCheckShowRetrieveMsg(1); + SendThenCheckShowRetrieveMsg(1, 1); CheckInt16(1); CheckBinary("localhost:9521", TSDB_EP_LEN); CheckInt16(0); @@ -219,7 +218,7 @@ TEST_F(DndTestDnode, CreateDnode_01) { taosMsleep(1300); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(2); + SendThenCheckShowRetrieveMsg(2, 1); CheckInt16(1); CheckInt16(2); CheckBinary("localhost:9521", TSDB_EP_LEN); @@ -305,66 +304,109 @@ TEST_F(DndTestDnode, AlterUser_01) { } } -TEST_F(DndTestDnode, DropUser_01) { - ASSERT_NE(pClient, nullptr); +#endif - //--- drop user --- - SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); - strcpy(pReq->user, "u1"); +TEST_F(DndTestDnode, DropDnode_01) { + SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); + pReq->dnodeId = htonl(2); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + rpcMsg.contLen = sizeof(SDropDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; sendMsg(pClient, &rpcMsg); SRpcMsg* pMsg = pClient->pRsp; ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 2); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } + taosMsleep(1300); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + SendThenCheckShowRetrieveMsg(1, 0); + CheckInt16(1); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(1); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckBinary("", 24); } -#endif \ No newline at end of file +TEST_F(DndTestDnode, CreateDnode_02) { + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9523"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9524"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9525"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + taosMsleep(1300); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + SendThenCheckShowRetrieveMsg(4, 0); + CheckInt16(1); + CheckInt16(3); + CheckInt16(4); + CheckInt16(5); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9523", TSDB_EP_LEN); + CheckBinary("localhost:9524", TSDB_EP_LEN); + CheckBinary("localhost:9525", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(0); + CheckInt16(0); + CheckInt16(0); + CheckInt16(1); + CheckInt16(1); + CheckInt16(1); + CheckInt16(1); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("", 24); + CheckBinary("", 24); + CheckBinary("", 24); + CheckBinary("", 24); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 0aa01ed0d0..13baab54fa 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -16,9 +16,9 @@ #include "deploy.h" void initLog(const char* path) { - dDebugFlag = 207; + dDebugFlag = 143; vDebugFlag = 0; - mDebugFlag = 207; + mDebugFlag = 143; cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 0; @@ -35,6 +35,8 @@ void initLog(const char* path) { tsdbDebugFlag = 0; cqDebugFlag = 0; + taosMkDir(path); + char temp[PATH_MAX]; snprintf(temp, PATH_MAX, "%s/taosdlog", path); if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index d3b911640d..d77bebc6f8 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -45,5 +45,3 @@ void dropServer(SServer* pServer); SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); void dropClient(SClient* pClient); void sendMsg(SClient* pClient, SRpcMsg* pMsg); - -// class DndTest \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index a610eaeeb9..c2b2acfa0c 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "mndShow.h" +#define SHOW_STEP_SIZE 100 + static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); @@ -211,7 +213,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { } /* return no more than 100 tables in one round trip */ - if (rowsToRead > 100) rowsToRead = 100; + if (rowsToRead > SHOW_STEP_SIZE) rowsToRead = SHOW_STEP_SIZE; /* * the actual number of table may be larger than the value of pShow->numOfRows, if a query is @@ -220,7 +222,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (rowsToRead < 0) rowsToRead = 0; size = pShow->rowSize * rowsToRead; - size += 100; + size += SHOW_STEP_SIZE; SRetrieveTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { mndReleaseShowObj(pShow, false); From 1c5fda4bc9c557155ee3393dd1f22ec49b562334 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 13 Dec 2021 15:35:33 +0800 Subject: [PATCH 20/22] TD-10431 drop dnode test --- source/dnode/mgmt/impl/src/dndDnode.c | 3 +- source/dnode/mgmt/impl/src/dndMnode.c | 35 +++++--- source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 89 ++++----------------- source/dnode/mnode/impl/src/mnode.c | 8 +- 5 files changed, 50 insertions(+), 87 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 508a40922f..f880085351 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -448,7 +448,7 @@ static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { a static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { assert(1); } static void dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) { - dDebug("config msg is received"); + dError("config msg is received, but not supported yet"); SCfgDnodeMsg *pCfg = pMsg->pCont; int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; @@ -585,4 +585,5 @@ void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { default: dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); } + rpcFreeCont(pMsg->pCont); } diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 7eb6ee980a..3cf08e619e 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -559,9 +559,12 @@ static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { break; } - SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; taosFreeQitem(pMsg); } @@ -645,9 +648,12 @@ void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; - rpcSendResponse(&rsp); + if (pRpcMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; + rpcSendResponse(&rsp); + } rpcFreeCont(pRpcMsg->pCont); + pRpcMsg->pCont = NULL; taosFreeQitem(pMsg); } } @@ -656,9 +662,12 @@ void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } dndReleaseMnode(pDnode, pMnode); @@ -668,9 +677,12 @@ void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } dndReleaseMnode(pDnode, pMnode); @@ -680,9 +692,12 @@ void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } dndReleaseMnode(pDnode, pMnode); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 245a1e41f6..2ec02ac203 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -141,8 +141,8 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { dTrace("RPC %p, rsp:%s is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->code & 0XFFFF); } else { dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]); + rpcFreeCont(pMsg->pCont); } - rpcFreeCont(pMsg->pCont); } static int32_t dndInitClient(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 02285de2a4..6484e83846 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -202,6 +202,22 @@ TEST_F(DndTestDnode, ShowDnode) { CheckBinary("", 24); } +TEST_F(DndTestDnode, ConfigDnode_01) { + SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg)); + pReq->dnodeId = htonl(1); + strcpy(pReq->config, "ddebugflag 131"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCfgDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CONFIG_DNODE; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); +} + TEST_F(DndTestDnode, CreateDnode_01) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); strcpy(pReq->ep, "localhost:9522"); @@ -235,77 +251,6 @@ TEST_F(DndTestDnode, CreateDnode_01) { CheckBinary("", 24); } -#if 0 -TEST_F(DndTestDnode, AlterUser_01) { - ASSERT_NE(pClient, nullptr); - - //--- drop user --- - SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p2"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 3); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "u1"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } -} - -#endif - TEST_F(DndTestDnode, DropDnode_01) { SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); pReq->dnodeId = htonl(2); @@ -409,4 +354,4 @@ TEST_F(DndTestDnode, CreateDnode_02) { CheckBinary("", 24); CheckBinary("", 24); CheckBinary("", 24); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index c83fd9124c..e543478608 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -336,8 +336,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { } SRpcConnInfo connInfo = {0}; - if (rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { - mndCleanupMsg(pMsg); + if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { + taosFreeQitem(pMsg); terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; mError("RPC:%p, app:%p failed to create msg since %s", pRpcMsg->handle, pRpcMsg->ahandle, terrstr()); return NULL; @@ -354,6 +354,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { void mndCleanupMsg(SMnodeMsg *pMsg) { mTrace("msg:%p, app:%p is destroyed, RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); + rpcFreeCont(pMsg->rpcMsg.pCont); + pMsg->rpcMsg.pCont = NULL; taosFreeQitem(pMsg); } @@ -367,7 +369,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { int32_t code = 0; int32_t msgType = pMsg->rpcMsg.msgType; void *ahandle = pMsg->rpcMsg.ahandle; - bool isReq = (msgType % 2 == 1); + bool isReq = (msgType & 1U); mTrace("msg:%p, app:%p type:%s will be processed", pMsg, ahandle, taosMsg[msgType]); From 4b74a29e816adce9e3c0605ea9af467d789a0555 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 13 Dec 2021 15:46:13 +0800 Subject: [PATCH 21/22] minor chages --- source/dnode/mgmt/impl/src/dndDnode.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index f880085351..130cbbef07 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -454,7 +454,6 @@ static void dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) { int32_t code = TSDB_CODE_OPS_NOT_SUPPORT; SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0, .code = code}; rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); } static void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) { @@ -467,7 +466,6 @@ static void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)}; rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); } static void *dnodeThreadRoutine(void *param) { @@ -567,8 +565,10 @@ void dndProcessDnodeReq(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { dError("RPC %p, dnode req:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED}; rpcSendResponse(&rspMsg); - rpcFreeCont(pMsg->pCont); } + + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -585,5 +585,7 @@ void dndProcessDnodeRsp(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { default: dError("RPC %p, dnode rsp:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); } + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; } From e073ab4bd47c30019e6249b5bebb656d211f7469 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 13 Dec 2021 17:06:04 +0800 Subject: [PATCH 22/22] TD-10431 restart dnode test --- source/dnode/mgmt/impl/inc/dndInt.h | 1 + source/dnode/mgmt/impl/src/dnode.c | 15 +++- source/dnode/mgmt/impl/test/acct/acct.cpp | 2 +- .../dnode/mgmt/impl/test/cluster/cluster.cpp | 2 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 83 ++++++++++++++++--- .../dnode/mgmt/impl/test/profile/profile.cpp | 2 +- source/dnode/mgmt/impl/test/show/show.cpp | 2 +- source/dnode/mgmt/impl/test/sut/deploy.cpp | 19 ++++- source/dnode/mgmt/impl/test/sut/deploy.h | 3 +- source/dnode/mgmt/impl/test/user/user.cpp | 2 +- source/libs/wal/src/walMgmt.c | 3 +- 11 files changed, 109 insertions(+), 25 deletions(-) diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index f65810feab..d8acda530d 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -112,6 +112,7 @@ typedef struct SDnode { EStat stat; SDnodeOpt opt; SDnodeDir dir; + FileFd lockFd; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; SVnodesMgmt vmgmt; diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 7f51979246..0dbee0d337 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -55,7 +55,7 @@ void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup) { pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static int32_t dndCheckRunning(char *dataDir) { +static FileFd dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); @@ -74,11 +74,12 @@ static int32_t dndCheckRunning(char *dataDir) { return -1; } - return 0; + return fd; } static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { - if (dndCheckRunning(pOption->dataDir) != 0) { + pDnode->lockFd = dndCheckRunning(pOption->dataDir); + if (pDnode->lockFd < 0) { return -1; } @@ -133,6 +134,12 @@ static void dndCleanupEnv(SDnode *pDnode) { tfree(pDnode->dir.dnode); } + if (pDnode->lockFd >= 0) { + taosUnLockFile(pDnode->lockFd); + taosCloseFile(pDnode->lockFd); + pDnode->lockFd = 0; + } + taosStopCacheRefreshWorker(); } @@ -202,6 +209,8 @@ SDnode *dndInit(SDnodeOpt *pOption) { } void dndCleanup(SDnode *pDnode) { + if (pDnode == NULL) return; + if (dndGetStat(pDnode) == DND_STAT_STOPPED) { dError("dnode is shutting down"); return; diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index cda44a3be8..9050a938fa 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -33,7 +33,7 @@ class DndTestAcct : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index 99614ec921..d47e63a85f 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -33,7 +33,7 @@ class DndTestCluster : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 6484e83846..c156852905 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -38,12 +38,18 @@ class DndTestDnode : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer1); - dropServer(pServer2); - dropServer(pServer3); - dropServer(pServer4); - dropServer(pServer5); + stopServer(pServer1); + stopServer(pServer2); + stopServer(pServer3); + stopServer(pServer4); + stopServer(pServer5); dropClient(pClient); + pServer1 = NULL; + pServer2 = NULL; + pServer3 = NULL; + pServer4 = NULL; + pServer5 = NULL; + pClient = NULL; } static SServer* pServer1; @@ -107,7 +113,7 @@ class DndTestDnode : public ::testing::Test { EXPECT_STREQ(pSchema->name, name); } - void SendThenCheckShowRetrieveMsg(int32_t rows, int32_t completed) { + void SendThenCheckShowRetrieveMsg(int32_t rows) { SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); pRetrieve->showId = htonl(showId); pRetrieve->free = 0; @@ -133,7 +139,7 @@ class DndTestDnode : public ::testing::Test { EXPECT_EQ(pRetrieveRsp->numOfRows, rows); EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); - EXPECT_EQ(pRetrieveRsp->completed, completed); + // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); EXPECT_EQ(pRetrieveRsp->reserved, 0); @@ -192,7 +198,7 @@ TEST_F(DndTestDnode, ShowDnode) { CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline reason"); - SendThenCheckShowRetrieveMsg(1, 1); + SendThenCheckShowRetrieveMsg(1); CheckInt16(1); CheckBinary("localhost:9521", TSDB_EP_LEN); CheckInt16(0); @@ -234,7 +240,7 @@ TEST_F(DndTestDnode, CreateDnode_01) { taosMsleep(1300); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(2, 1); + SendThenCheckShowRetrieveMsg(2); CheckInt16(1); CheckInt16(2); CheckBinary("localhost:9521", TSDB_EP_LEN); @@ -267,7 +273,7 @@ TEST_F(DndTestDnode, DropDnode_01) { taosMsleep(1300); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(1, 0); + SendThenCheckShowRetrieveMsg(1); CheckInt16(1); CheckBinary("localhost:9521", TSDB_EP_LEN); CheckInt16(0); @@ -325,7 +331,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { taosMsleep(1300); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(4, 0); + SendThenCheckShowRetrieveMsg(4); CheckInt16(1); CheckInt16(3); CheckInt16(4); @@ -355,3 +361,58 @@ TEST_F(DndTestDnode, CreateDnode_02) { CheckBinary("", 24); CheckBinary("", 24); } + +TEST_F(DndTestDnode, RestartDnode_01) { + uInfo("===> stop all server"); + stopServer(pServer1); + stopServer(pServer2); + stopServer(pServer3); + stopServer(pServer4); + stopServer(pServer5); + pServer1 = NULL; + pServer2 = NULL; + pServer3 = NULL; + pServer4 = NULL; + pServer5 = NULL; + + taosMsleep(3000); // wait tcp port cleanedup + uInfo("===> start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9521"; + pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); + + uInfo("===> all server is running"); + + // taosMsleep(1300); + // SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + // SendThenCheckShowRetrieveMsg(4); + // CheckInt16(1); + // CheckInt16(3); + // CheckInt16(4); + // CheckInt16(5); + // CheckBinary("localhost:9521", TSDB_EP_LEN); + // CheckBinary("localhost:9523", TSDB_EP_LEN); + // CheckBinary("localhost:9524", TSDB_EP_LEN); + // CheckBinary("localhost:9525", TSDB_EP_LEN); + // CheckInt16(0); + // CheckInt16(0); + // CheckInt16(0); + // CheckInt16(0); + // CheckInt16(1); + // CheckInt16(1); + // CheckInt16(1); + // CheckInt16(1); + // CheckBinary("ready", 10); + // CheckBinary("ready", 10); + // CheckBinary("ready", 10); + // CheckBinary("ready", 10); + // CheckTimestamp(); + // CheckTimestamp(); + // CheckTimestamp(); + // CheckTimestamp(); + // CheckBinary("", 24); + // CheckBinary("", 24); + // CheckBinary("", 24); + // CheckBinary("", 24); +} diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index cb93f1572f..2cbb9d5c2c 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -33,7 +33,7 @@ class DndTestProfile : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index 2208de1b59..c535386ecd 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -33,7 +33,7 @@ class DndTestShow : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 13baab54fa..61fc27a595 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -26,7 +26,7 @@ void initLog(const char* path) { httpDebugFlag = 0; mqttDebugFlag = 0; monDebugFlag = 0; - uDebugFlag = 0; + uDebugFlag = 143; rpcDebugFlag = 0; odbcDebugFlag = 0; qDebugFlag = 0; @@ -34,7 +34,9 @@ void initLog(const char* path) { sDebugFlag = 0; tsdbDebugFlag = 0; cqDebugFlag = 0; + tscEmbeddedInUtil = 1; + taosRemoveDir(path); taosMkDir(path); char temp[PATH_MAX]; @@ -70,8 +72,7 @@ void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t snprintf(pOption->firstEp, TSDB_EP_LEN, "%s", firstEp); } -SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - taosRemoveDir(path); +SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { taosMkDir(path); SDnodeOpt option = {0}; @@ -90,11 +91,21 @@ SServer* createServer(const char* path, const char* fqdn, uint16_t port, const c return pServer; } -void dropServer(SServer* pServer) { +SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + taosRemoveDir(path); + return startServer(path, fqdn, port, firstEp); +} + +void stopServer(SServer* pServer) { if (pServer == NULL) return; if (pServer->threadId != NULL) { taosDestoryThread(pServer->threadId); } + + if (pServer->pDnode != NULL) { + dndCleanup(pServer->pDnode); + pServer->pDnode = NULL; + } } void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index d77bebc6f8..4c082df5f3 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -41,7 +41,8 @@ typedef struct { void initLog(const char* path); SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); -void dropServer(SServer* pServer); +SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); +void stopServer(SServer* pServer); SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); void dropClient(SClient* pClient); void sendMsg(SClient* pClient, SRpcMsg* pMsg); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 9e75e620e7..c3c1d1d406 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -34,7 +34,7 @@ class DndTestUser : public ::testing::Test { } static void TearDownTestSuite() { - dropServer(pServer); + stopServer(pServer); dropClient(pClient); } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 9090348c04..d18652233a 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -249,8 +249,9 @@ static int32_t walCreateThread() { static void walStopThread() { atomic_store_8(&tsWal.stop, 1); - if (taosCheckPthreadValid(tsWal.thread)) { + if (tsWal.thread != NULL && taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); + tsWal.thread = NULL; } wDebug("wal thread is stopped");