diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 49e13b7831..d827c6640a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -168,13 +168,6 @@ pipeline { stages { stage('run test') { parallel { - stage('windows test') { - agent {label " windows11 "} - steps { - pre_test_win() - pre_test_build_win() - } - } stage('linux test') { agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } diff --git a/cmake/cmake.define b/cmake/cmake.define index d1d9266bca..aeab39cab4 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -62,7 +62,7 @@ ELSE () MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fpermissive -fPIC -gdwarf-2 -g3") ENDIF () MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") diff --git a/example/src/tstream.c b/example/src/tstream.c index 5bd833213d..65fd005954 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -81,10 +81,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = - taos_query(pConn, - "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k " - "from tu1 interval(10m)"); + pRes = taos_query( + pConn, + "create stream stream1 trigger window_close into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k " + "from tu1 interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 84ea208508..dd55722513 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -192,6 +192,21 @@ typedef struct SExprInfo { struct tExprNode* pExpr; } SExprInfo; +typedef struct { + const char* key; + int32_t keyLen; + uint8_t type; + int16_t length; + union{ + const char* value; + int64_t i; + uint64_t u; + double d; + float f; + }; + int32_t valueLen; +} SSmlKv; + #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 11d0f34c45..e94be797b9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -372,10 +372,9 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche // ----------------- Data column structure // SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { - int8_t type; // column type - uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint8_t bitmapMode : 1; // default is 0(2 bits), otherwise 1(1 bit) - uint8_t reserve : 6; + int8_t type; // column type + uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows + uint8_t reserve : 7; int16_t colId; // column ID int32_t bytes; // column data bytes defined int32_t offset; // data offset in a SDataRow (including the header size) @@ -387,8 +386,6 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; - - #define isAllRowsNull(pCol) ((pCol)->len == 0) #define isAllRowsNone(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } @@ -482,7 +479,7 @@ void tdResetDataCols(SDataCols *pCols); int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3889784f2f..26c8072df6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -326,11 +326,11 @@ typedef struct { int8_t alterType; int32_t numOfFields; SArray* pFields; -} SMAltertbReq; +} SMAlterStbReq; -int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq); -int32_t tDeserializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq); -void tFreeSMAltertbReq(SMAltertbReq* pReq); +int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq); +int32_t tDeserializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq); +void tFreeSMAltertbReq(SMAlterStbReq* pReq); typedef struct SEpSet { int8_t inUse; @@ -672,7 +672,6 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; - int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); @@ -685,7 +684,6 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); - typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -1554,7 +1552,9 @@ typedef struct SVDropStbReq { int32_t tEncodeSVDropStbReq(SCoder* pCoder, const SVDropStbReq* pReq); int32_t tDecodeSVDropStbReq(SCoder* pCoder, SVDropStbReq* pReq); +#define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { + int32_t flags; tb_uid_t uid; int64_t ctime; const char* name; diff --git a/include/common/trow.h b/include/common/trow.h index f87ea1f009..0d34c6e49f 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -42,7 +42,7 @@ extern "C" { * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available */ -#define TD_VTYPE_NORM 0x00U // normal val: not none, not null(no need assign value) +#define TD_VTYPE_NORM 0x00U // normal val: not none, not null #define TD_VTYPE_NULL 0x01U // null val #define TD_VTYPE_NONE 0x02U // none or unknown/undefined #define TD_VTYPE_MAX 0x03U // @@ -140,8 +140,6 @@ typedef struct { }; /// row total length uint32_t len; - /// row version - // uint64_t ver; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -241,13 +239,13 @@ static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colI static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, - int8_t bitmapMode); + int8_t bitmapMode, bool isMerge); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); /** * @brief @@ -718,6 +716,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } + return TSDB_CODE_SUCCESS; } diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 405b20c521..377b443843 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -186,14 +186,14 @@ typedef struct { #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) #define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) -#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) -#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) -#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) -#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) -#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) -#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +#define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c94d75338a..fa4cb1d2bd 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -56,7 +56,8 @@ typedef enum { QUERY_LESS_EQUAL, QUERY_GREATER_THAN, QUERY_GREATER_EQUAL, - QUERY_RANGE + QUERY_RANGE, + QUERY_MAX } EIndexQueryType; /* @@ -178,8 +179,8 @@ void indexOptsDestroy(SIndexOpts* opts); * @param: */ -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType, - const char* colName, int32_t nColName, const char* colVal, int32_t nColVal); +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal); void indexTermDestroy(SIndexTerm* p); /* diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 84fa166881..7c9602734b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -91,17 +91,23 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); void qDestroyStmtDataBlock(void* pBlock); + int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen); +void* smlInitHandle(SQuery *pQuery); +void smlDestroyHandle(void *pHandle); +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen); +int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 441e87eff7..8ff62f825c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -327,29 +327,32 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) #define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) #define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) -#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) -#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) -#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) -#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) -#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) -#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) -#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) -#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) -#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) -#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) -#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) -#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) -#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) -#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) -#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) -#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) -#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) -#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) -#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) -#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x0617) -#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0618) -#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0619) -#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x0620) +#define TSDB_CODE_TDB_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0604) +#define TSDB_CODE_TDB_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0605) +#define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0606) +#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0607) +#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0608) +#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0609) +#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x060A) +#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x060B) +#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x060C) +#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060D) +#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060E) +#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060F) +#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x0600) +#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x0601) +#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x0602) +#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0613) +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0614) +#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0615) +#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0616) +#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0617) +#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0618) +#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0619) +#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A) +#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B) +#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C) +#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x062D) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) @@ -642,6 +645,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #define TSDB_CODE_FUNC_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2604) +#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) +#define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) +#define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index 316640797b..4669a29883 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -29,6 +29,8 @@ extern "C" { #define TSKEY_MAX (INT64_MAX - 1) #define TSKEY_INITIAL_VAL TSKEY_MIN +#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle + // Bytes for each type. extern const int32_t TYPE_BYTES[15]; @@ -84,9 +86,13 @@ extern const int32_t TYPE_BYTES[15]; #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' + #define TSDB_TIME_PRECISION_MILLI 0 #define TSDB_TIME_PRECISION_MICRO 1 #define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TIME_PRECISION_HOURS 3 +#define TSDB_TIME_PRECISION_MINUTES 4 +#define TSDB_TIME_PRECISION_SECONDS 5 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 818436b411..9d75586917 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -787,8 +787,3 @@ int taos_stmt_close(TAOS_STMT *stmt) { return stmtClose(stmt); } -TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - // TODO - return NULL; -} - diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c new file mode 100644 index 0000000000..5f9138bb43 --- /dev/null +++ b/source/client/src/clientSml.c @@ -0,0 +1,1757 @@ +#include +#include +#include +#include + +#include "query.h" +#include "taos.h" +#include "taoserror.h" +#include "tdef.h" +#include "tlog.h" +#include "tmsg.h" +#include "tstrbuild.h" +#include "ttime.h" +#include "ttypes.h" +#include "tcommon.h" +#include "catalog.h" +#include "clientInt.h" +//================================================================================================= + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' +#define tsMaxSQLStringLen (1024*1024) + +//================================================================================================= +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef enum { + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SHashObj *tags; + SHashObj *fields; +} SCreateSTableActionInfo; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SSmlKv * field; +} SAlterSTableActionInfo; + +typedef struct { + ESchemaAction action; + union { + SCreateSTableActionInfo createSTable; + SAlterSTableActionInfo alterSTable; + }; +} SSchemaAction; + +typedef struct { + const char* measure; + const char* tags; + const char* cols; + const char* timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + uint8_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; + + SArray *tags; + + // colsFormat store cols formated, for quick parse, if info->formatData is true + SArray *colsFormat; // elements are SArray + + // cols & colsColumn store cols un formated + SArray *cols; // elements are SHashObj for find by key quickly + SHashObj *columnsHash; // elements are , just for judge if key exists quickly. +} SSmlTableInfo; + +typedef struct { + SHashObj *tagHash; + SHashObj *fieldHash; + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + uint64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool dataFormat; // true means that the name, number and order of keys in each line are the same + + SHashObj *childTables; + SHashObj *superTables; + SHashObj *pVgHash; + void *exec; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + int32_t affectedRows; + SSmlMsgBuf msgBuf; +} SSmlHandle; +//================================================================================================= + +static uint64_t linesSmlHandleId = 0; +static const char* TS = "_ts"; +static const char* TAG = "_tagNone"; + +//================================================================================================= + +static uint64_t smlGenId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + +static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) { + if(msg1) strncat(pBuf->buf, msg1, pBuf->len); + int32_t left = pBuf->len - strlen(pBuf->buf); + if(left > 2 && msg2) { + strncat(pBuf->buf, ":", left - 1); + strncat(pBuf->buf, msg2, left - 2); + } + return TSDB_CODE_SML_INVALID_DATA; +} + +static int smlCompareKv(const void* p1, const void* p2) { + SSmlKv* kv1 = (SSmlKv*)p1; + SSmlKv* kv2 = (SSmlKv*)p2; + int32_t kvLen1 = kv1->keyLen; + int32_t kvLen2 = kv2->keyLen; + int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} + +static void smlBuildChildTableName(SSmlTableInfo *tags) { + int32_t size = taosArrayGetSize(tags->tags); + ASSERT(size > 0); + taosArraySort(tags->tags, smlCompareKv); + + SStringBuilder sb = {0}; + taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); + for (int j = 0; j < size; ++j) { + SSmlKv *tagKv = taosArrayGetP(tags->tags, j); + taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); + taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + tMD5Final(&context); + uint64_t digest1 = *(uint64_t*)(context.digest); + uint64_t digest2 = *(uint64_t*)(context.digest + 8); + snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); + taosStringBuilderDestroy(&sb); + tags->uid = digest1; +} + +static int32_t smlGenerateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], + SSchemaAction* action, bool* actionNeeded, SSmlHandle* info) { +// char fieldName[TSDB_COL_NAME_LEN] = {0}; +// strcpy(fieldName, pointColField->name); +// +// size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); +// if (pDbIndex) { +// SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); +// assert(strcasecmp(dbAttr->name, pointColField->name) == 0); +// if (pointColField->type != dbAttr->type) { +// uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, +// pointColField->type, dbAttr->type); +// return TSDB_CODE_TSC_INVALID_VALUE; +// } +// +// if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { +// if (isTag) { +// action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; +// } else { +// action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; +// } +// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); +// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); +// action->alterSTable.field = pointColField; +// *actionNeeded = true; +// } +// } else { +// if (isTag) { +// action->action = SCHEMA_ACTION_ADD_TAG; +// } else { +// action->action = SCHEMA_ACTION_ADD_COLUMN; +// } +// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); +// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); +// action->alterSTable.field = pointColField; +// *actionNeeded = true; +// } +// if (*actionNeeded) { +// uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, +// action->action); +// } + return 0; +} + +static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, int32_t* outBytes) { + uint8_t type = field->type; + char tname[TSDB_TABLE_NAME_LEN] = {0}; + memcpy(tname, field->key, field->keyLen); + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t bytes = field->valueLen; // todo + int out = snprintf(buf, bufSize,"%s %s(%d)", + tname,tDataTypes[field->type].name, bytes); + *outBytes = out; + } else { + int out = snprintf(buf, bufSize, "%s %s", tname, tDataTypes[type].name); + *outBytes = out; + } + + return 0; +} + +static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { + int32_t code = 0; + int32_t outBytes = 0; + char *result = (char *)taosMemoryCalloc(1, tsMaxSQLStringLen+1); + int32_t capacity = tsMaxSQLStringLen + 1; + + uDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); + switch (action->action) { + case SCHEMA_ACTION_ADD_COLUMN: { + int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + const char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); + } + taos_free_result(res); + +// if (code == TSDB_CODE_MND_FIELD_ALREADY_EXIST || code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { + if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_ADD_TAG: { + int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); + smlBuildColumnDescription(action->alterSTable.field, + result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + const char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + +// if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { + if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { + int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); + smlBuildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + +// if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + if (code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CHANGE_TAG_SIZE: { + int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); + smlBuildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + +// if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + if (code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CREATE_STABLE: { + int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); + char* pos = result + n; int freeBytes = capacity - n; + + size_t size = taosHashGetSize(action->createSTable.fields); + SArray *cols = taosArrayInit(size, POINTER_BYTES); + SSmlKv **kv = taosHashIterate(action->createSTable.fields, NULL); + while(kv){ + if(strncmp((*kv)->key, TS, strlen(TS)) == 0 && (*kv)->type == TSDB_DATA_TYPE_TIMESTAMP){ + taosArrayInsert(cols, 0, kv); + }else{ + taosArrayPush(cols, kv); + } + kv = taosHashIterate(action->createSTable.fields, kv); + } + + for(int i = 0; i < taosArrayGetSize(cols); i++){ + SSmlKv *kvNew = taosArrayGetP(cols, i); + smlBuildColumnDescription(kvNew, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + taosArrayDestroy(cols); + + --pos; ++freeBytes; + + outBytes = snprintf(pos, freeBytes, ") tags ("); + pos += outBytes; freeBytes -= outBytes; + + kv = taosHashIterate(action->createSTable.tags, NULL); + while(kv){ + smlBuildColumnDescription(*kv, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + kv = taosHashIterate(action->createSTable.tags, kv); + } + pos--; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ")"); + TAOS_RES* res = taos_query(info->taos, result); + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_STB_ALREADY_EXIST) { + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + + default: + break; + } + + taosMemoryFreeClear(result); + if (code != 0) { + uError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); + } + return code; +} + +static int32_t smlModifyDBSchemas(SSmlHandle* info) { + int32_t code = 0; + + SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); + while (tableMetaSml) { + SSmlSTableMeta* cTablePoints = *tableMetaSml; + + STableMeta *pTableMeta = NULL; + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + + size_t superTableLen = 0; + void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); // todo escape + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, superTable, superTableLen); + + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_MND_INVALID_STB) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); + schemaAction.createSTable.tags = cTablePoints->tagHash; + schemaAction.createSTable.fields = cTablePoints->fieldHash; + code = smlApplySchemaAction(info, &schemaAction); + if (code != 0) { + uError("SML:0x%"PRIx64" smlApplySchemaAction failed. can not create %s", info->id, schemaAction.createSTable.sTableName); + return code; + } + + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); + if (code != 0) { + uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, schemaAction.createSTable.sTableName); + return code; + } + }else if (code == TSDB_CODE_SUCCESS) { + } else { + uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); + return code; + } + cTablePoints->tableMeta = pTableMeta; + + tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); + } + return 0; +} + +//========================================================================= + +/* Field Escape charaters + 1: measurement Comma,Space + 2: tag_key, tag_value, field_key Comma,Equal Sign,Space + 3: field_value Double quote,Backslash +*/ +//static void escapeSpecialCharacter(uint8_t field, const char **pos) { +// const char *cur = *pos; +// if (*cur != '\\') { +// return; +// } +// switch (field) { +// case 1: +// switch (*(cur + 1)) { +// case ',': +// case ' ': +// cur++; +// break; +// default: +// break; +// } +// break; +// case 2: +// switch (*(cur + 1)) { +// case ',': +// case ' ': +// case '=': +// cur++; +// break; +// default: +// break; +// } +// break; +// case 3: +// switch (*(cur + 1)) { +// case '"': +// case '\\': +// cur++; +// break; +// default: +// break; +// } +// break; +// default: +// break; +// } +// *pos = cur; +//} + +static bool smlParseTinyInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 2) { + return false; + } + const char *signalPos = pVal + len - 2; + if (!strncasecmp(signalPos, "i8", 2)) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid tiny int", endptr); + }else if(!IS_VALID_TINYINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseTinyUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 2) { + return false; + } + if (pVal[0] == '-') { + return false; + } + const char *signalPos = pVal + len - 2; + if (!strncasecmp(signalPos, "u8", 2)) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr); + }else if(!IS_VALID_UTINYINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseSmallInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 3) { + return false; + } + const char *signalPos = pVal + len - 3; + if (!strncasecmp(signalPos, "i16", 3)) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid small int", endptr); + }else if(!IS_VALID_SMALLINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseSmallUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + const char *signalPos = pVal + len - 3; + if (strncasecmp(signalPos, "u16", 3) == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid unsigned small int", endptr); + }else if(!IS_VALID_USMALLINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 3) { + return false; + } + const char *signalPos = pVal + len - 3; + if (strncasecmp(signalPos, "i32", 3) == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid int", endptr); + }else if(!IS_VALID_INT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + const char *signalPos = pVal + len - 3; + if (strncasecmp(signalPos, "u32", 3) == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid unsigned int", endptr); + }else if(!IS_VALID_UINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len > 3 && strncasecmp(pVal + len - 3, "i64", 3) == 0) { + char *endptr = NULL; + errno = 0; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != pVal + len - 3){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid big int", endptr); + }else if(errno == ERANGE || !IS_VALID_BIGINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + }else if (len > 1 && pVal[len - 1] == 'i') { + char *endptr = NULL; + errno = 0; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != pVal + len - 1){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid big int", endptr); + }else if(errno == ERANGE || !IS_VALID_BIGINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + const char *signalPos = pVal + len - 3; + if (strncasecmp(signalPos, "u64", 3) == 0) { + char *endptr = NULL; + errno = 0; + uint64_t result = strtoull(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid unsigned big int", endptr); + }else if(errno == ERANGE || !IS_VALID_UBIGINT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); + }else{ + kvVal->u = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + char *endptr = NULL; + errno = 0; + float result = strtof(pVal, &endptr); + if(endptr == pVal + len && errno != ERANGE && IS_VALID_FLOAT(result)){ // 78 + kvVal->f = result; + *isValid = true; + return true; + } + + if (len > 3 && strncasecmp(pVal + len - 3, "f32", 3) == 0) { + if(endptr != pVal + len - 3){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid float", endptr); + }else if(errno == ERANGE || !IS_VALID_FLOAT(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", endptr); + }else{ + kvVal->f = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseDouble(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len <= 3) { + return false; + } + const char *signalPos = pVal + len - 3; + if (strncasecmp(signalPos, "f64", 3) == 0) { + char *endptr = NULL; + errno = 0; + double result = strtod(pVal, &endptr); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid double", endptr); + }else if(errno == ERANGE || !IS_VALID_DOUBLE(result)){ + *isValid = false; + smlBuildInvalidDataMsg(msg, "double out of range[-1.7976931348623158e+308,1.7976931348623158e+308]", endptr); + }else{ + kvVal->d = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool smlParseBool(SSmlKv *kvVal) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if ((len == 1) && pVal[len - 1] == 't') { + kvVal->i = true; + return true; + } + + if ((len == 1) && pVal[len - 1] == 'f') { + kvVal->i = false; + return true; + } + + if((len == 4) && !strncasecmp(pVal, "true", len)) { + kvVal->i = true; + return true; + } + if((len == 5) && !strncasecmp(pVal, "false", len)) { + kvVal->i = false; + return true; + } + return false; +} + +static bool smlIsBinary(const char *pVal, uint16_t len) { + //binary: "abc" + if (len < 2) { + return false; + } + if (pVal[0] == '"' && pVal[len - 1] == '"') { + return true; + } + return false; +} + +static bool smlIsNchar(const char *pVal, uint16_t len) { + //nchar: L"abc" + if (len < 3) { + return false; + } + if ((pVal[0] == 'l' || pVal[0] == 'L')&& pVal[1] == '"' && pVal[len - 1] == '"') { + return true; + } + return false; +} + +static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { + // put high probability matching type first + bool isValid = false; + + //binary + if (smlIsBinary(pVal->value, pVal->valueLen)) { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->valueLen -= 2; + pVal->length = pVal->valueLen; + pVal->value++; + return true; + } + //nchar + if (smlIsNchar(pVal->value, pVal->valueLen)) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->valueLen -= 3; + pVal->length = pVal->valueLen; + pVal->value += 2; + return true; + } + //float + if (smlParseFloat(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + //double + if (smlParseDouble(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + + return true; + } + //bool + if (smlParseBool(pVal)) { + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + + if (smlParseTinyInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (smlParseTinyUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UTINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (smlParseSmallInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (smlParseSmallUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_USMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (smlParseInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (smlParseUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (smlParseBigInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (smlParseBigUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UBIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + + smlBuildInvalidDataMsg(msg, "invalid data", pVal->value); + return false; +} + +static bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlHandle* info) { + char *val = NULL; + val = taosHashGet(pHash, key, strlen(key)); + if (val) { + uError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, key); + return true; + } + + uint8_t dummy_val = 0; + taosHashPut(pHash, key, strlen(key), &dummy_val, sizeof(uint8_t)); + + return false; +} + +static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){ + if(!sql) return TSDB_CODE_SML_INVALID_DATA; + while (*sql != '\0') { // jump the space at the begining + if(*sql != SPACE) { + elements->measure = sql; + break; + } + sql++; + } + if (!elements->measure || *sql == COMMA) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse measure and tag + while (*sql != '\0') { + if (elements->measureLen == 0 && *sql == COMMA && *(sql - 1) != SLASH) { // find the first comma + elements->measureLen = sql - elements->measure; + sql++; + elements->tags = sql; + continue; + } + + if (*sql == SPACE && *(sql - 1) != SLASH) { // find the first space + if (elements->measureLen == 0) { + elements->measureLen = sql - elements->measure; + elements->tags = sql; + } + elements->tagsLen = sql - elements->tags; + elements->measureTagsLen = sql - elements->measure; + break; + } + + sql++; + } + if(elements->tagsLen == 0){ // measure, cols1=a measure cols1=a + elements->measureTagsLen = elements->measureLen; + } + if(elements->measureLen == 0) { + smlBuildInvalidDataMsg(msg, "invalid measure", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse cols + while (*sql != '\0') { + if(*sql != SPACE) { + elements->cols = sql; + break; + } + sql++; + } + if(!elements->cols) { + smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols); + return TSDB_CODE_SML_INVALID_DATA; + } + + bool isInQuote = false; + while (*sql != '\0') { + if(*sql == QUOTE && *(sql - 1) != SLASH){ + isInQuote = !isInQuote; + } + if(!isInQuote && *sql == SPACE && *(sql - 1) != SLASH) { + break; + } + sql++; + } + if(isInQuote){ + smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); + return TSDB_CODE_SML_INVALID_DATA; + } + elements->colsLen = sql - elements->cols; + + // parse ts,ts can be empty + while (*sql != '\0') { + if(*sql != SPACE && elements->timestamp == NULL) { + elements->timestamp = sql; + } + if(*sql == SPACE && elements->timestamp != NULL){ + break; + } + sql++; + } + if(elements->timestamp){ + elements->timestampLen = sql - elements->timestamp; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool isTag, SSmlMsgBuf *msg){ + if(isTag && len == 0){ + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + kv->key = TAG; + kv->keyLen = strlen(TAG); + kv->value = TAG; + kv->valueLen = strlen(TAG); + kv->type = TSDB_DATA_TYPE_NCHAR; + if(cols) taosArrayPush(cols, &kv); + return TSDB_CODE_SUCCESS; + } + + for(int i = 0; i < len; i++){ + // parse key + const char *key = data + i; + int32_t keyLen = 0; + while(i < len){ + if(data[i] == EQUAL && i > 0 && data[i-1] != SLASH){ + keyLen = data + i - key; + break; + } + i++; + } + if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse value + i++; + const char *value = data + i; + bool isInQuote = false; + while(i < len){ + if(data[i] == QUOTE && data[i-1] != SLASH){ + isInQuote = !isInQuote; + } + if(!isInQuote && data[i] == COMMA && i > 0 && data[i-1] != SLASH){ + break; + } + i++; + } + if(isInQuote){ + smlBuildInvalidDataMsg(msg, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } + int32_t valueLen = data + i - value; + if(valueLen == 0){ + smlBuildInvalidDataMsg(msg, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + + // add kv to SSmlKv + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + kv->key = key; + kv->keyLen = keyLen; + kv->value = value; + kv->valueLen = valueLen; + if(isTag){ + kv->type = TSDB_DATA_TYPE_NCHAR; + }else{ + if(!smlParseValue(kv, msg)){ + return TSDB_CODE_SML_INVALID_DATA; + } + } + + if(cols) taosArrayPush(cols, &kv); + } + + return TSDB_CODE_SUCCESS; +} + +static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { + char *endPtr = NULL; + double ts = (double)strtoll(value, &endPtr, 10); + if(value + len != endPtr){ + return -1; + } + switch (type) { + case TSDB_TIME_PRECISION_HOURS: + ts *= (3600 * 1e9); + break; + case TSDB_TIME_PRECISION_MINUTES: + ts *= (60 * 1e9); + break; + case TSDB_TIME_PRECISION_SECONDS: + ts *= (1e9); + break; + case TSDB_TIME_PRECISION_MILLI: + ts *= (1e6); + break; + case TSDB_TIME_PRECISION_MICRO: + ts *= (1e3); + break; + case TSDB_TIME_PRECISION_NANO: + break; + default: + ASSERT(0); + } + if(ts > (double)INT64_MAX || ts < 0){ + return -1; + } + + return (int64_t)ts; +} + +static int64_t smlGetTimeNow(int8_t precision) { + switch (precision) { + case TSDB_TIME_PRECISION_HOURS: + return taosGetTimestampMs()/1000/3600; + case TSDB_TIME_PRECISION_MINUTES: + return taosGetTimestampMs()/1000/60; + case TSDB_TIME_PRECISION_SECONDS: + return taosGetTimestampMs()/1000; + case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_NANO: + return taosGetTimestamp(precision); + default: + ASSERT(0); + } +} + +static int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI_DIGITS; + } else { + return -1; + } +} + +static int8_t smlGetTsTypeByPrecision(int8_t precision) { + switch (precision) { + case TSDB_SML_TIMESTAMP_HOURS: + return TSDB_TIME_PRECISION_HOURS; + case TSDB_SML_TIMESTAMP_MILLI_SECONDS: + return TSDB_TIME_PRECISION_MILLI; + case TSDB_SML_TIMESTAMP_NANO_SECONDS: + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + return TSDB_TIME_PRECISION_NANO; + case TSDB_SML_TIMESTAMP_MICRO_SECONDS: + return TSDB_TIME_PRECISION_MICRO; + case TSDB_SML_TIMESTAMP_SECONDS: + return TSDB_TIME_PRECISION_SECONDS; + case TSDB_SML_TIMESTAMP_MINUTES: + return TSDB_TIME_PRECISION_MINUTES; + default: + return -1; + } +} + +static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){ + int8_t tsType = smlGetTsTypeByPrecision(info->precision); + if (tsType == -1) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); + return -1; + } + if(!data){ + return smlGetTimeNow(tsType); + } + + int64_t ts = smlGetTimeValue(data, len, tsType); + if(ts == -1){ + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t len){ + if(!data){ + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + int8_t tsType = smlGetTsTypeByLen(len); + if (tsType == -1) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, tsType); + if(ts == -1){ + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArray *cols){ + int64_t ts = 0; + if(info->protocol == TSDB_SML_LINE_PROTOCOL){ + ts = smlParseInfluxTime(info, data, len); + }else{ + ts = smlParseOpenTsdbTime(info, data, len); + } + if(ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP; + + // add ts to + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + if(!kv){ + return TSDB_CODE_OUT_OF_MEMORY; + } + + kv->key = TS; + kv->keyLen = strlen(kv->key); + kv->i = ts; + kv->type = TSDB_DATA_TYPE_TIMESTAMP; + kv->length = (int16_t)tDataTypes[kv->type].bytes; + if(cols) taosArrayPush(cols, &kv); + return TSDB_CODE_SUCCESS; +} + +//static int32_t parseSmlCols(const char* data, SArray *cols){ +// while(*data != '\0'){ +// if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA; +// const char *key = data; +// int32_t keyLen = 0; +// while(*data != '\0'){ +// if(*data == EQUAL && *(data-1) != SLASH){ +// keyLen = data - key; +// data ++; +// break; +// } +// data++; +// } +// if(keyLen == 0){ +// return TSDB_CODE_SML_INVALID_DATA; +// } +// +// if(*data == COMMA) return TSDB_CODE_SML_INVALID_DATA; +// const char *value = data; +// int32_t valueLen = 0; +// while(*data != '\0'){ +// if(*data == COMMA && *(data-1) != SLASH){ +// valueLen = data - value; +// data ++; +// break; +// } +// data++; +// } +// if(valueLen == 0){ +// return TSDB_CODE_SML_INVALID_DATA; +// } +// +// TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); +// kv->key = key; +// kv->keyLen = keyLen; +// kv->value = value; +// kv->valueLen = valueLen; +// kv->type = TSDB_DATA_TYPE_NCHAR; +// if(cols) taosArrayPush(cols, &kv); +// } +// return TSDB_CODE_SUCCESS; +//} + +static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){ + if(tags){ + for (int i = 0; i < taosArrayGetSize(tags); ++i) { + SSmlKv *kv = taosArrayGetP(tags, i); + ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR); + + SSmlKv **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); + if(value){ + ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR); + if(kv->valueLen > (*value)->valueLen){ // tags type is nchar + *value = kv; + } + }else{ + taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } + } + + if(cols){ + for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp + SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); + if(value){ + if(kv->type != (*value)->type){ + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return false; + }else{ + if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger + if(kv->valueLen > (*value)->valueLen){ + *value = kv; + } + } + } + }else{ + taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } + } + return true; +} + +static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ + if(tags){ + for (int i = 0; i < taosArrayGetSize(tags); ++i) { + SSmlKv *kv = taosArrayGetP(tags, i); + taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } + + if(cols){ + for (int i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = taosArrayGetP(cols, i); + taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } +} + +static SSmlTableInfo* smlBuildTableInfo(bool format){ + SSmlTableInfo *tag = taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if(!tag){ + return NULL; + } + + if(format){ + tag->colsFormat = taosArrayInit(16, POINTER_BYTES); + if (tag->colsFormat == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + }else{ + tag->cols = taosArrayInit(16, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + + tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (tag->columnsHash == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + } + + tag->tags = taosArrayInit(16, POINTER_BYTES); + if (tag->tags == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + return tag; + +cleanup: + taosMemoryFreeClear(tag); + return NULL; +} + +static void smlDestroyBuildTableInfo(SSmlTableInfo *tag, bool format){ + if(format){ + taosArrayDestroy(tag->colsFormat); + }else{ + tag->cols = taosArrayInit(16, POINTER_BYTES); + for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){ + SHashObj *kvHash = taosArrayGetP(tag->cols, i); + void** p1 = taosHashIterate(kvHash, NULL); + while (p1) { + SSmlKv* kv = *p1; + taosMemoryFreeClear(kv); + p1 = taosHashIterate(kvHash, p1); + } + taosHashCleanup(kvHash); + } + taosHashCleanup(tag->columnsHash); + } + taosArrayDestroy(tag->tags); + taosMemoryFreeClear(tag); +} + +static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ + if(dataFormat){ + taosArrayPush(oneTable->colsFormat, &cols); + }else{ + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if(!kvHash){ + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + for(size_t i = 0; i < taosArrayGetSize(cols); i++){ + SSmlKv *kv = taosArrayGetP(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); // todo key need escape, like \=, because find by schema name later + + if(taosHashGet(oneTable->columnsHash, kv->key, kv->keyLen) != NULL){ + continue; + } + taosHashPut(oneTable->columnsHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + taosArrayPush(oneTable->cols, &kvHash); + } + return TSDB_CODE_SUCCESS; +} + +static SSmlSTableMeta* smlBuildSTableMeta(){ + SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if(!meta){ + return NULL; + } + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + + meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->fieldHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + return meta; + +cleanup: + taosMemoryFreeClear(meta); + return NULL; +} + +static void smlDestroySTableMeta(SSmlSTableMeta *meta){ + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->fieldHash); + taosMemoryFree(meta->tableMeta); +} + +static int32_t smlParseLine(SSmlHandle* info, const char* sql) { + SSmlLineInfo elements = {0}; + int ret = smlParseString(sql, &elements, &info->msgBuf); + if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseString failed", info->id); + return ret; + } + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + if (cols == NULL) { + uError("SML:0x%"PRIx64" smlParseLine failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); + if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseTS failed", info->id); + return ret; + } + ret = smlParseCols(elements.cols, elements.colsLen, cols, false, &info->msgBuf); + if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseCols parse cloums fields failed", info->id); + return ret; + } + if(taosArrayGetSize(cols) > TSDB_MAX_COLUMNS){ + smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + SSmlTableInfo **oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); + if(oneTable){ + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + ASSERT(tableMeta); + ret = smlUpdateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta cols + if(!ret){ + uError("SML:0x%"PRIx64" smlUpdateMeta cols failed", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + ret = smlDealCols(*oneTable, info->dataFormat, cols); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + }else{ + SSmlTableInfo *tag = smlBuildTableInfo(info->dataFormat); + if(!tag){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + ret = smlDealCols(tag, info->dataFormat, cols); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + ret = smlParseCols(elements.tags, elements.tagsLen, tag->tags, true, &info->msgBuf); + if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseCols parse tag fields failed", info->id); + return ret; + } + + if(taosArrayGetSize(tag->tags) > TSDB_MAX_TAGS){ + smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + tag->sTableName = elements.measure; + tag->sTableNameLen = elements.measureLen; + smlBuildChildTableName(tag); + uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tag->childTableName); + + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + if(tableMeta){ // update meta + ret = smlUpdateMeta(*tableMeta, tag->tags, cols, &info->msgBuf); + if(!ret){ + uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + }else{ + SSmlSTableMeta *meta = smlBuildSTableMeta(); + smlInsertMeta(meta, tag->tags, cols); + taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); + } + + taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tag, POINTER_BYTES); + } + return TSDB_CODE_SUCCESS; +} + +static void smlDestroyInfo(SSmlHandle* info){ + if(!info) return; + qDestroyQuery(info->pQuery); + smlDestroyHandle(info->exec); + + // destroy info->childTables + void** p1 = taosHashIterate(info->childTables, NULL); + while (p1) { + SSmlTableInfo* oneTable = *p1; + smlDestroyBuildTableInfo(oneTable, info->dataFormat); + p1 = taosHashIterate(info->childTables, p1); + } + taosHashCleanup(info->childTables); + + // destroy info->superTables + p1 = taosHashIterate(info->superTables, NULL); + while (p1) { + SSmlSTableMeta* oneTable = *p1; + smlDestroySTableMeta(oneTable); + p1 = taosHashIterate(info->superTables, p1); + } + taosHashCleanup(info->superTables); + + // destroy info->pVgHash + taosHashCleanup(info->pVgHash); + + taosMemoryFreeClear(info); +} + +static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ + int32_t code = TSDB_CODE_SUCCESS; + SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); + if (NULL == info) { + return NULL; + } + info->id = smlGenId(); + + info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + if (NULL == info->pQuery) { + uError("SML:0x%"PRIx64" create info->pQuery error", info->id); + goto cleanup; + } + info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + info->pQuery->haveResultSet = false; + info->pQuery->msgType = TDMT_VND_SUBMIT; + info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if(NULL == info->pQuery->pRoot){ + uError("SML:0x%"PRIx64" create info->pQuery->pRoot error", info->id); + goto cleanup; + } + ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; + + info->taos = taos; + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if(code != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); + goto cleanup; + } + + info->precision = precision; + info->protocol = protocol; + info->dataFormat = dataFormat; + info->pRequest = request; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + + info->exec = smlInitHandle(info->pQuery); + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + + if(NULL == info->exec || NULL == info->childTables + || NULL == info->superTables || NULL == info->pVgHash){ + uError("SML:0x%"PRIx64" create info failed", info->id); + goto cleanup; + } + + return info; +cleanup: + smlDestroyInfo(info); + return NULL; +} +static int32_t smlInsertData(SSmlHandle* info) { + int32_t code = TSDB_CODE_SUCCESS; + + SSmlTableInfo** oneTable = taosHashIterate(info->childTables, NULL); + while (oneTable) { + SSmlTableInfo* tableData = *oneTable; + + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); + if (code != 0) { + uError("SML:0x%"PRIx64" catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); + return code; + } + taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); + + SSmlSTableMeta** pMeta = taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); + ASSERT (NULL != pMeta && NULL != *pMeta); + + // use tablemeta of stable to save vgid and uid of child table + (*pMeta)->tableMeta->vgId = vg.vgId; + (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + + code = smlBindData(info->exec, tableData->tags, tableData->colsFormat, tableData->columnsHash, + tableData->cols, info->dataFormat, (*pMeta)->tableMeta, tableData->childTableName, info->msgBuf.buf, info->msgBuf.len); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + oneTable = taosHashIterate(info->childTables, oneTable); + } + + smlBuildOutput(info->exec, info->pVgHash); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); + + info->affectedRows = taos_affected_rows(info->pRequest); + return info->pRequest->code; +} + +static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { + int32_t code = TSDB_CODE_SUCCESS; + + if (numLines <= 0 || numLines > 65536) { + uError("SML:0x%"PRIx64" smlInsertLines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); + code = TSDB_CODE_TSC_APP_ERROR; + goto cleanup; + } + + for (int32_t i = 0; i < numLines; ++i) { + code = smlParseLine(info, lines[i]); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" smlParseLine failed. line %d : %s", info->id, i, lines[i]); + goto cleanup; + } + } + uDebug("SML:0x%"PRIx64" smlInsertLines parse success. tables %d", info->id, taosHashGetSize(info->childTables)); + uDebug("SML:0x%"PRIx64" smlInsertLines parse success. super tables %d", info->id, taosHashGetSize(info->superTables)); + + code = smlModifyDBSchemas(info); + if (code != 0) { + uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code)); + goto cleanup; + } + + code = smlInsertData(info); + if (code != 0) { + uError("SML:0x%"PRIx64" smlInsertData error : %s", info->id, tstrerror(code)); + goto cleanup; + } + + uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); + +cleanup: + return code; +} + +/** + * taos_schemaless_insert() parse and insert data points into database according to + * different protocol. + * + * @param $lines input array may contain multiple lines, each line indicates a data point. + * If protocol=2 is used input array should contain single JSON + * string(e.g. char *lines[] = {"$JSON_string"}). If need to insert + * multiple data points in JSON format, should include them in $JSON_string + * as a JSON array. + * @param $numLines indicates how many data points in $lines. + * If protocol = 2 is used this param will be ignored as $lines should + * contain single JSON string. + * @param $protocol indicates which protocol to use for parsing: + * 0 - influxDB line protocol + * 1 - OpenTSDB telnet line protocol + * 2 - OpenTSDB JSON format protocol + * @return return zero for successful insertion. Otherwise return none-zero error code of + * failure reason. + * + */ + +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { + SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + if(!request){ + return NULL; + } + + SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, false); + if(!info){ + return (TAOS_RES*)request; + } + + switch (protocol) { + case TSDB_SML_LINE_PROTOCOL:{ + smlInsertLines(info, lines, numLines); + break; + } + case TSDB_SML_TELNET_PROTOCOL: + //code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); + break; + case TSDB_SML_JSON_PROTOCOL: + //code = taos_insert_json_payload(taos, *lines, protocol, tsType, &affected_rows); + break; + default: + break; + } + smlDestroyInfo(info); + +end: + return (TAOS_RES*)request; +} + diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index f7ef1f7e81..3c94de1cef 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -458,7 +458,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); return TSDB_CODE_SUCCESS; } diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 9df6b260f6..3fc91d07b6 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -17,14 +17,27 @@ TARGET_LINK_LIBRARIES( PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom ) +ADD_EXECUTABLE(smlTest smlTest.cpp) +TARGET_LINK_LIBRARIES( + smlTest + PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom +) + TARGET_INCLUDE_DIRECTORIES( clientTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) TARGET_INCLUDE_DIRECTORIES( tmqTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) + +TARGET_INCLUDE_DIRECTORIES( + smlTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp new file mode 100644 index 0000000000..5f0f188b0b --- /dev/null +++ b/source/client/test/smlTest.cpp @@ -0,0 +1,498 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "../src/clientSml.c" +#include "taos.h" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, smlParseString_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + SSmlLineInfo elements = {0}; + + // case 1 + char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; + int ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st,t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 2 false + sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); + + // case 3 false + sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + + // case 4 tag is null + sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 2 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 5 tag is null + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 3 + elements.colsLen + 2); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 6 + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + + // case 7 + sql = " st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen(",")); + + // case 8 false + sql = ", st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); +} + +TEST(testCase, smlParseCols_Error_Test) { + const char *data[] = { + "c=\"89sd", // binary, nchar + "c=j\"89sd\"", + "c=\"89sd\"k", + "c=u", // bool + "c=truet", + "c=f64", // double + "c=8f64f", + "c=8ef64", + "c=1.7976931348623158e+390f64", + "c=f32", // float + "c=8f32f", + "c=8wef32", + "c=-3.402823466e+39f32", + "c=", // float + "c=8f", + "c=8we", + "c=3.402823466e+39", + "c=i8", // tiny int + "c=-8i8f", + "c=8wei8", + "c=-999i8", + "c=u8", // u tiny int + "c=8fu8", + "c=8weu8", + "c=999u8", + "c=-8u8", + "c=i16", // small int + "c=8fi16u", + "c=8wei16", + "c=-67787i16", + "c=u16", // u small int + "c=8u16f", + "c=8weu16", + "c=-9u16", + "c=67787u16", + "c=i32", // int + "c=8i32f", + "c=8wei32", + "c=2147483649i32", + "c=u32", // u int + "c=8u32f", + "c=8weu32", + "c=-4u32", + "c=42949672958u32", + "c=i64", // big int + "c=8i64i", + "c=8wei64", + "c=-9223372036854775809i64", + "c=i", // big int + "c=8fi", + "c=8wei", + "c=9223372036854775808i", + "c=u64", // u big int + "c=8u64f", + "c=8weu64", + "c=-3.402823466e+39u64", + "c=-339u64", + "c=18446744073709551616u64", + }; + + for(int i = 0; i < sizeof(data)/sizeof(data[0]); i++){ + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + int32_t len = strlen(data[i]); + int32_t ret = smlParseCols(data[i], len, NULL, false, &msgBuf); + ASSERT_NE(ret, TSDB_CODE_SUCCESS); + } +} + +TEST(testCase, smlParseCols_tag_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + ASSERT_NE(cols, NULL); + + const char *data = + "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + int32_t len = strlen(data); + int32_t ret = smlParseCols(data, len, cols, true, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + int32_t size = taosArrayGetSize(cols); + ASSERT_EQ(size, 19); + + // nchar + SSmlKv *kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, 18); + ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 3); + ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, 7); + ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); + taosMemoryFree(kv); + + taosArrayClear(cols); + data = "t=3e"; + len = 0; + memset(msgBuf.buf, 0, msgBuf.len); + ret = smlParseCols(data, len, cols, true, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + size = taosArrayGetSize(cols); + ASSERT_EQ(size, 1); + + // nchar + kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, TAG, strlen(TAG)), 0); + ASSERT_EQ(kv->keyLen, strlen(TAG)); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, strlen(TAG)); + ASSERT_EQ(strncasecmp(kv->value, TAG, strlen(TAG)), 0); + taosMemoryFree(kv); +} + +TEST(testCase, smlParseCols_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + ASSERT_NE(cols, NULL); + + const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + int32_t len = strlen(data); + int32_t ret = smlParseCols(data, len, cols, false, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + int32_t size = taosArrayGetSize(cols); + ASSERT_EQ(size, 19); + + // binary + SSmlKv *kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); + ASSERT_EQ(kv->length, 16); + ASSERT_EQ(strncasecmp(kv->value, "passit", 6), 0); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 1); + ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 7); + ASSERT_EQ(strncasecmp(kv->value, "iisd", 4), 0); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 2); + ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, false); + taosMemoryFree(kv); + + // double + kv = taosArrayGetP(cols, 3); + ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(kv->length, 8); + //ASSERT_EQ(kv->d, 4.31); + printf("4.31 = kv->f:%f\n", kv->d); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 4); + ASSERT_EQ(strncasecmp(kv->key, "cf32_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + //ASSERT_EQ(kv->f, 8.32); + printf("8.32 = kv->f:%f\n", kv->f); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 5); + ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + //ASSERT_EQ(kv->f, 8.23); + printf("8.23 = kv->f:%f\n", kv->f); + taosMemoryFree(kv); + + // tiny int + kv = taosArrayGetP(cols, 6); + ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, -34); + taosMemoryFree(kv); + + // unsigned tiny int + kv = taosArrayGetP(cols, 7); + ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->u, 89); + taosMemoryFree(kv); + + // small int + kv = taosArrayGetP(cols, 8); + ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 233); + taosMemoryFree(kv); + + // unsigned smallint + kv = taosArrayGetP(cols, 9); + ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 898); + taosMemoryFree(kv); + + // int + kv = taosArrayGetP(cols, 10); + ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 98289); + taosMemoryFree(kv); + + // unsigned int + kv = taosArrayGetP(cols, 11); + ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 12323); + taosMemoryFree(kv); + + + // bigint + kv = taosArrayGetP(cols, 12); + ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, -89238); + taosMemoryFree(kv); + + // bigint + kv = taosArrayGetP(cols, 13); + ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); + ASSERT_EQ(kv->keyLen, 2); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, 989); + taosMemoryFree(kv); + + // unsigned bigint + kv = taosArrayGetP(cols, 14); + ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->u, 8989323); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 15); + ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); + ASSERT_EQ(kv->keyLen, 9); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, true); + taosMemoryFree(kv); + + + // bool + kv = taosArrayGetP(cols, 16); + ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); + ASSERT_EQ(kv->keyLen, 6); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, true); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 17); + ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); + ASSERT_EQ(kv->keyLen, 6); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, false); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 18); + ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); + taosMemoryFree(kv); + + taosArrayDestroy(cols); +} + +TEST(testCase, smlParseLine_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, NULL); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, NULL); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, NULL); + + const char *sql[3] = { + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_2,fleet=North,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000" + }; + smlInsertLines(info, sql, 3); +// for (int i = 0; i < 3; i++) { +// smlParseLine(info, sql[i]); +// } +} + +// TEST(testCase, smlParseTS_Test) { +// char msg[256] = {0}; +// SSmlMsgBuf msgBuf; +// msgBuf.buf = msg; +// msgBuf.len = 256; +// SSmlLineInfo elements = {0}; +// +// SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); +// if(!info){ +// return (TAOS_RES*)request; +// } +// ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); +// if(ret != TSDB_CODE_SUCCESS){ +// uError("SML:0x%"PRIx64" smlParseTS failed", info->id); +// return ret; +// } +// } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a4b36dcb67..2c17f2c2fc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1478,11 +1478,11 @@ void blockDebugShowData(const SArray* dataBlocks) { * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in * SDataBlock->info.uid * @param suid // TODO: check with Liao whether suid response is reasonable - * + * * TODO: colId should be set */ -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, - tb_uid_t suid) { +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, + tb_uid_t uid, tb_uid_t suid) { int32_t sz = taosArrayGetSize(pDataBlocks); int32_t bufSize = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; ++i) { @@ -1494,16 +1494,16 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks ASSERT(bufSize < 3 * 1024 * 1024); *pReq = taosMemoryCalloc(1, bufSize); - if(!(*pReq)) { + if (!(*pReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } void* pDataBuf = *pReq; - int32_t msgLen = sizeof(SSubmitReq); + int32_t msgLen = sizeof(SSubmitReq); int32_t numOfBlks = 0; SRowBuilder rb = {0}; - tdSRowInit(&rb, 0); // TODO: use the latest version + tdSRowInit(&rb, 0); // TODO: use the latest version for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); @@ -1511,8 +1511,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t rows = pDataBlock->info.rows; int32_t rowSize = pDataBlock->info.rowSize; int64_t groupId = pDataBlock->info.groupId; - - if(rb.nCols != colNum) { + + if (rb.nCols != colNum) { tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); } @@ -1525,10 +1525,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks msgLen += sizeof(SSubmitBlk); int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row - tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf + for (int32_t j = 0; j < rows; ++j) { // iterate by row + tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf printf("|"); - bool isStartKey = false; + bool isStartKey = false; for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); @@ -1536,7 +1536,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks case TSDB_DATA_TYPE_TIMESTAMP: if (!isStartKey) { isStartKey = true; - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, + 0, 0); } else { tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k); break; @@ -1629,14 +1630,14 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) { blkHead->uid = htobe64(pDataBlock->info.uid); int32_t rows = pDataBlock->info.rows; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ /*blkHead->dataLen = htonl(rows * maxLen);*/ blkHead->dataLen = 0; void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); STSRow* rowData = blockData; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { + for (int32_t j = 0; j < rows; j++) { SRowBuilder rb = {0}; tdSRowInit(&rb, pTSchema->version); tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6278b52a04..daa25719ad 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -594,7 +594,7 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) return 0; } -int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { +int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -615,7 +615,7 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { return tlen; } -int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { +int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -645,7 +645,7 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) return 0; } -void tFreeSMAltertbReq(SMAltertbReq *pReq) { +void tFreeSMAltertbReq(SMAlterStbReq *pReq) { taosArrayDestroy(pReq->pFields); pReq->pFields = NULL; } @@ -1255,7 +1255,6 @@ int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) return 0; } - int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2091,7 +2090,7 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } -int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tSerializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2110,7 +2109,7 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp return tlen; } -int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tDeserializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2136,7 +2135,7 @@ int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchR return 0; } -void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); @@ -2146,7 +2145,6 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ taosArrayDestroy(pRsp->pArray); } - int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3746,6 +3744,7 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; if (tEncodeI64(pCoder, pReq->uid) < 0) return -1; if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1; @@ -3771,6 +3770,7 @@ int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1; if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 7157c1e0f0..50e51a40dd 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -24,7 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{ }, { // 1 bit - TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I, + TD_VTYPE_NORM_BYTE_I, // normal + TD_VTYPE_NULL_BYTE_I, TD_VTYPE_NULL_BYTE_I, // padding } @@ -33,6 +34,24 @@ const uint8_t tdVTypeByte[2][3] = {{ // declaration static uint8_t tdGetBitmapByte(uint8_t byte); +// static void dataColSetNEleNull(SDataCol *pCol, int nEle); + +/** + * @brief src2 data has more priority than src1 + * + * @param target + * @param src1 + * @param iter1 + * @param limit1 + * @param src2 + * @param iter2 + * @param limit2 + * @param tRows + * @param update + */ +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows, bool update); + // implementation /** * @brief Compress bitmap bytes comprised of 2-bits to counterpart of 1-bit. @@ -229,23 +248,23 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) { void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { int32_t i = 0, j = 0; int32_t nBytes = TD_BITMAP_BYTES(nBits); - int32_t nStrictBytes = nBits / 4; - int32_t nPartialBits = nBits - nStrictBytes * 4; + int32_t nRoundBytes = nBits / 4; + int32_t nRemainderBits = nBits - nRoundBytes * 4; - switch (nPartialBits) { + switch (nRemainderBits) { case 0: // NOTHING TODO break; case 1: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xC0; } break; case 2: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xF0; } break; case 3: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xFC; } break; default: @@ -266,10 +285,6 @@ void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { } } -// static void dataColSetNEleNull(SDataCol *pCol, int nEle); -static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull); - static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; @@ -329,7 +344,7 @@ bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode if (*((uint8_t *)pBitmap) != vTypeByte) { return false; } - pBitmap = POINTER_SHIFT(pBitmap, 1); + pBitmap = POINTER_SHIFT(pBitmap, i); } int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); @@ -410,11 +425,12 @@ STSRow *tdRowDup(STSRow *row) { * @param val * @param numOfRows * @param maxPoints - * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) + * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) + * @param isMerge merge to current row * @return int */ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, - int8_t bitmapMode) { + int8_t bitmapMode, bool isMerge) { TASSERT(pCol != NULL); // Assume that the columns not specified during insert/upsert mean None. @@ -430,33 +446,58 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int dataColSetNEleNone(pCol, numOfRows, bitmapMode); } } - if (!tdValTypeIsNorm(valType)) { + const void *value = val; + if (!tdValTypeIsNorm(valType) || !val) { // TODO: // 1. back compatibility and easy to debug with codes of 2.0 to save NULL values. // 2. later on, considering further optimization, don't save Null/None for VarType. - val = getNullValue(pCol->type); + value = getNullValue(pCol->type); } - if (IS_VAR_DATA_TYPE(pCol->type)) { - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); - // Update the length - pCol->len += varDataTLen(val); - } else { - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes); - pCol->len += pCol->bytes; + if (!isMerge) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + // set offset + pCol->dataOff[numOfRows] = pCol->len; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); + // Update the length + pCol->len += varDataTLen(value); + } else { + ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); + pCol->len += pCol->bytes; + } + } else if (!tdValTypeIsNone(valType)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + // keep the last offset + // discard the last var data + int32_t lastVarLen = varDataTLen(POINTER_SHIFT(pCol->pData, pCol->dataOff[numOfRows])); + pCol->len -= lastVarLen; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); + // Update the length + pCol->len += varDataTLen(value); + } else { + ASSERT(pCol->len - TYPE_BYTES[pCol->type] == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), value, pCol->bytes); + } } + #ifdef TD_SUPPORT_BITMAP - tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + if (!isMerge || !tdValTypeIsNone(valType)) { + tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + } #endif return 0; } // internal -static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { +#if 0 ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow)); +#endif + + // Multi-Version rows with the same key and different versions supported + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) <= TD_ROW_KEY(pRow)); int rcol = 1; int dcol = 1; @@ -464,12 +505,14 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; continue; } @@ -480,22 +523,26 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; ++rcol; } else if (pRowCol->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; } } +#if 0 ++pCols->numOfRows; +#endif return TSDB_CODE_SUCCESS; } // internal -static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow)); int rcol = 0; @@ -506,12 +553,14 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= tRowCols || rcol >= tSchemaCols) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; continue; } @@ -527,17 +576,21 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; ++rcol; } else if (pIdx->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; } } +#if 0 ++pCols->numOfRows; +#endif return TSDB_CODE_SUCCESS; } @@ -548,20 +601,30 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols * @param pRow * @param pSchema * @param pCols - * @param forceSetNull */ -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { if (TD_IS_TP_ROW(pRow)) { - return tdAppendTpRowToDataCol(pRow, pSchema, pCols); + return tdAppendTpRowToDataCol(pRow, pSchema, pCols, isMerge); } else if (TD_IS_KV_ROW(pRow)) { - return tdAppendKvRowToDataCol(pRow, pSchema, pCols); + return tdAppendKvRowToDataCol(pRow, pSchema, pCols, isMerge); } else { ASSERT(0); } return TSDB_CODE_SUCCESS; } -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, +/** + * @brief source data has more priority than target + * + * @param target + * @param source + * @param rowsToMerge + * @param pOffset + * @param update + * @param maxVer + * @return int + */ +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool update, TDRowVerT maxVer) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); @@ -576,17 +639,38 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); // TODO: filter the maxVer - for (int i = 0; i < rowsToMerge; i++) { + TSKEY lastKey = TSKEY_INITIAL_VAL; + for (int i = 0; i < rowsToMerge; ++i) { + bool merge = false; for (int j = 0; j < source->numOfCols; j++) { if (source->cols[j].len > 0 || target->cols[j].len > 0) { SCellVal sVal = {0}; if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) { TASSERT(0); } + + if (j == 0) { + if (lastKey == *(TSKEY *)sVal.val) { + if (!update) { + break; + } + merge = true; + } else if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; + } + + lastKey = *(TSKEY *)sVal.val; + } + if (i == 0) { + (target->cols + j)->bitmap = (source->cols + j)->bitmap; + } + tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); + target->bitmapMode, merge); } } + } + if (lastKey != TSKEY_INITIAL_VAL) { ++target->numOfRows; } (*pOffset) += rowsToMerge; @@ -596,7 +680,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * int iter1 = 0; tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows, - pTarget->numOfRows + rowsToMerge, forceSetNull); + pTarget->numOfRows + rowsToMerge, update); } tdFreeDataCols(pTarget); @@ -607,67 +691,95 @@ _err: return -1; } -// src2 data has more priority than src1 +static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, bool isMerge) { + for (int i = 0; i < src->numOfCols; ++i) { + ASSERT(target->cols[i].type == src->cols[i].type); + if (src->cols[i].len > 0 || target->cols[i].len > 0) { + SCellVal sVal = {0}; + if (tdGetColDataOfRow(&sVal, src->cols + i, iter, src->bitmapMode) < 0) { + TASSERT(0); + } + if (isMerge) { + if (!tdValTypeIsNone(sVal.valType)) { + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode, isMerge); + } else { + // Keep the origin value for None + } + } else { + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode, isMerge); + } + } + } +} +/** + * @brief src2 data has more priority than src1 + * + * @param target + * @param src1 + * @param iter1 + * @param limit1 + * @param src2 + * @param iter2 + * @param limit2 + * @param tRows + * @param update + */ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull) { + int limit2, int tRows, bool update) { tdResetDataCols(target); + target->bitmapMode = src1->bitmapMode; ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); + int32_t nRows = 0; - while (target->numOfRows < tRows) { + // TODO: filter the maxVer + // TODO: handle the delete function + TSKEY lastKey = TSKEY_INITIAL_VAL; + while (nRows < tRows) { if (*iter1 >= limit1 && *iter2 >= limit2) break; TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); - TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); + // TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); // TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); - ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - // TODO: filter the maxVer - if (key1 < key2) { - for (int i = 0; i < src1->numOfCols; ++i) { - ASSERT(target->cols[i].type == src1->cols[i].type); - if (src1->cols[i].len > 0 || target->cols[i].len > 0) { - SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } - } + // ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - ++target->numOfRows; + if (key1 <= key2) { + // select key1 if not delete + if (update && (lastKey == key1)) { + tdAppendValToDataCols(target, src1, *iter1, true); + } else if (lastKey != key1) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; + } + tdAppendValToDataCols(target, src1, *iter1, false); + } + ++nRows; ++(*iter1); - } else if (key1 >= key2) { - // TODO: filter the maxVer - if ((key1 > key2) || ((key1 == key2) && !TKEY_IS_DELETED(key2))) { - for (int i = 0; i < src2->numOfCols; ++i) { - SCellVal sVal = {0}; - ASSERT(target->cols[i].type == src2->cols[i].type); - if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) { - TASSERT(0); - } - if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) { - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } else if (target->cols[i].len > 0) { - dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode); - } + lastKey = key1; + } else { + // use key2 if not deleted + // TODO: handle the delete function + if (update && (lastKey == key2)) { + tdAppendValToDataCols(target, src2, *iter2, true); + } else if (lastKey != key2) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; } - ++target->numOfRows; + tdAppendValToDataCols(target, src2, *iter2, false); } + ++nRows; ++(*iter2); - if (key1 == key2) ++(*iter1); + lastKey = key2; } - ASSERT(target->numOfRows <= target->maxPoints); + ASSERT(target->numOfRows <= target->maxPoints - 1); + } + if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; } } @@ -777,7 +889,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->sversion = pDataCols->sversion; if (keepData) pRet->numOfRows = pDataCols->numOfRows; - for (int i = 0; i < pDataCols->numOfCols; i++) { + for (int i = 0; i < pDataCols->numOfCols; ++i) { pRet->cols[i].type = pDataCols->cols[i].type; pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; pRet->cols[i].colId = pDataCols->cols[i].colId; @@ -797,8 +909,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); } if (!TD_COL_ROWS_NORM(pRet->cols + i)) { - int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows); - memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); + memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, TD_BITMAP_BYTES(pDataCols->numOfRows)); } } } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index e3d67cd488..3fd2ef8e73 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -557,7 +557,7 @@ static const void *nullValues[] = { }; const void *getNullValue(int32_t type) { - assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); // TODO: extend the types return nullValues[type - 1]; } diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 805b4c7264..11b91f0568 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -101,6 +101,7 @@ void qmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, qmProcessFetchMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e3816396c9..9329e7b15d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -293,14 +293,10 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); // sync integration dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index ae7663a775..890879912f 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -31,6 +31,7 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser); int32_t mndCheckNodeAuth(SUserObj *pOperUser); int32_t mndCheckFuncAuth(SUserObj *pOperUser); +int32_t mndCheckTransAuth(SUserObj *pOperUser); int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 89638524ae..bb0739805c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -72,7 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, - + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, @@ -115,7 +115,10 @@ typedef enum { TRN_TYPE_STB_SCOPE_END, } ETrnType; -typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy; +typedef enum { + TRN_POLICY_ROLLBACK = 0, + TRN_POLICY_RETRY = 1, +} ETrnPolicy; typedef enum { DND_REASON_ONLINE = 0, @@ -131,6 +134,15 @@ typedef enum { DND_REASON_OTHERS } EDndReason; +typedef enum { + CONSUMER_UPDATE__TOUCH = 1, + CONSUMER_UPDATE__ADD, + CONSUMER_UPDATE__REMOVE, + CONSUMER_UPDATE__LOST, + CONSUMER_UPDATE__RECOVER, + CONSUMER_UPDATE__MODIFY, +} ECsmUpdateType; + typedef struct { int32_t id; ETrnStage stage; @@ -386,7 +398,6 @@ typedef struct { int32_t codeSize; char* pComment; char* pCode; - char pData[]; } SFuncObj; typedef struct { @@ -425,18 +436,8 @@ typedef struct { int64_t offset; } SMqOffsetObj; -static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pOffset->key); - tlen += taosEncodeFixedI64(buf, pOffset->offset); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) { - buf = taosDecodeStringTo(buf, pOffset->key); - buf = taosDecodeFixedI64(buf, &pOffset->offset); - return buf; -} +int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset); +void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset); typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; @@ -459,26 +460,15 @@ typedef struct { SSchemaWrapper schema; } SMqTopicObj; -enum { - CONSUMER_UPDATE__TOUCH = 1, - CONSUMER_UPDATE__ADD, - CONSUMER_UPDATE__REMOVE, - CONSUMER_UPDATE__LOST, - CONSUMER_UPDATE__RECOVER, - CONSUMER_UPDATE__MODIFY, -}; - typedef struct { - int64_t consumerId; - char cgroup[TSDB_CGROUP_LEN]; - char appId[TSDB_CGROUP_LEN]; - int8_t updateType; // used only for update - int32_t epoch; - int32_t status; - // hbStatus is not applicable to serialization - int32_t hbStatus; - // lock is used for topics update - SRWLatch lock; + int64_t consumerId; + char cgroup[TSDB_CGROUP_LEN]; + char appId[TSDB_CGROUP_LEN]; + int8_t updateType; // used only for update + int32_t epoch; + int32_t status; + int32_t hbStatus; // hbStatus is not applicable to serialization + SRWLatch lock; // lock is used for topics update SArray* currentTopics; // SArray SArray* rebNewTopics; // SArray SArray* rebRemovedTopics; // SArray @@ -492,7 +482,6 @@ typedef struct { int64_t upTime; int64_t subscribeTime; int64_t rebalanceTime; - } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); @@ -581,19 +570,18 @@ typedef struct { } SMqRebOutputObj; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char sourceDb[TSDB_DB_FNAME_LEN]; - char targetDb[TSDB_DB_FNAME_LEN]; - char targetSTbName[TSDB_TABLE_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; - int64_t dbUid; - int32_t version; - int32_t vgNum; - SRWLatch lock; - int8_t status; - // int32_t sqlLen; + char name[TSDB_TOPIC_FNAME_LEN]; + char sourceDb[TSDB_DB_FNAME_LEN]; + char targetDb[TSDB_DB_FNAME_LEN]; + char targetSTbName[TSDB_TABLE_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + int64_t dbUid; + int32_t version; + int32_t vgNum; + SRWLatch lock; + int8_t status; int8_t createdBy; // STREAM_CREATED_BY__USER or SMA int32_t fixedSinkVgId; // 0 for shuffle int64_t smaId; // 0 for unused @@ -601,7 +589,6 @@ typedef struct { int32_t triggerParam; int64_t waterMark; char* sql; - char* logicalPlan; char* physicalPlan; SArray* tasks; // SArray> SSchemaWrapper outputSchema; diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index a415d39434..28d3215b98 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -31,6 +31,11 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t n int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); +int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate); +SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName); +int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb); +int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 2140d0fa67..016ec3e6e9 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -29,7 +29,8 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); -int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 8aecc22454..1e95859157 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -27,10 +27,14 @@ void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +SArray *mndBuildDnodesArray(SMnode *pMnode); +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2); + void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 696d850266..8e5ec40c47 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -74,10 +74,7 @@ static int32_t mndProcessAuthReq(SNodeMsg *pReq) { } int32_t mndCheckCreateUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } @@ -118,28 +115,25 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, } int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckNodeAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckFuncAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } + if (pOperUser->superUser) return 0; + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} +int32_t mndCheckTransAuth(SUserObj *pOperUser) { + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 257db0dcd4..96845fcd42 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -17,8 +17,8 @@ #include "mndCluster.h" #include "mndShow.h" -#define TSDB_CLUSTER_VER_NUMBE 1 -#define TSDB_CLUSTER_RESERVE_SIZE 64 +#define CLUSTER_VER_NUMBE 1 +#define CLUSTER_RESERVE_SIZE 64 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); @@ -30,14 +30,16 @@ static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter); int32_t mndInitCluster(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_CLUSTER, - .keyType = SDB_KEY_INT64, - .deployFp = (SdbDeployFp)mndCreateDefaultCluster, - .encodeFp = (SdbEncodeFp)mndClusterActionEncode, - .decodeFp = (SdbDecodeFp)mndClusterActionDecode, - .insertFp = (SdbInsertFp)mndClusterActionInsert, - .updateFp = (SdbUpdateFp)mndClusterActionUpdate, - .deleteFp = (SdbDeleteFp)mndClusterActionDelete}; + SSdbTable table = { + .sdbType = SDB_CLUSTER, + .keyType = SDB_KEY_INT64, + .deployFp = (SdbDeployFp)mndCreateDefaultCluster, + .encodeFp = (SdbEncodeFp)mndClusterActionEncode, + .decodeFp = (SdbDecodeFp)mndClusterActionDecode, + .insertFp = (SdbInsertFp)mndClusterActionInsert, + .updateFp = (SdbUpdateFp)mndClusterActionUpdate, + .deleteFp = (SdbDeleteFp)mndClusterActionDelete, + }; mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndRetrieveClusters); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndCancelGetNextCluster); @@ -79,19 +81,19 @@ int64_t mndGetClusterId(SMnode *pMnode) { static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, TSDB_CLUSTER_VER_NUMBE, sizeof(SClusterObj) + TSDB_CLUSTER_RESERVE_SIZE); - if (pRaw == NULL) goto CLUSTER_ENCODE_OVER; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, CLUSTER_VER_NUMBE, sizeof(SClusterObj) + CLUSTER_RESERVE_SIZE); + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT64(pRaw, dataPos, pCluster->id, CLUSTER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime, CLUSTER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, CLUSTER_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, CLUSTER_ENCODE_OVER) - SDB_SET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE, CLUSTER_ENCODE_OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->id, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) + SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; -CLUSTER_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("cluster:%" PRId64 ", failed to encode to raw:%p since %s", pCluster->id, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -106,29 +108,29 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CLUSTER_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_CLUSTER_VER_NUMBE) { + if (sver != CLUSTER_VER_NUMBE) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto CLUSTER_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SClusterObj)); - if (pRow == NULL) goto CLUSTER_DECODE_OVER; + if (pRow == NULL) goto _OVER; SClusterObj *pCluster = sdbGetRowObj(pRow); - if (pCluster == NULL) goto CLUSTER_DECODE_OVER; + if (pCluster == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT64(pRaw, dataPos, &pCluster->id, CLUSTER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pCluster->createdTime, CLUSTER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, CLUSTER_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, CLUSTER_DECODE_OVER) - SDB_GET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE, CLUSTER_DECODE_OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->id, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) + SDB_GET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; -CLUSTER_DECODE_OVER: +_OVER: if (terrno != 0) { mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -161,7 +163,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { int32_t code = taosGetSystemUUID(clusterObj.name, TSDB_CLUSTER_ID_LEN); if (code != 0) { - strcpy(clusterObj.name, "tdengine2.0"); + strcpy(clusterObj.name, "tdengine3.0"); mError("failed to get name from system, set to default val %s", clusterObj.name); } @@ -190,8 +192,8 @@ static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pCluster->id, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->id, false); char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes); @@ -200,7 +202,7 @@ static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock colDataAppend(pColInfo, numOfRows, buf, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pCluster->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false); sdbRelease(pSdb, pCluster); numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index af7efb5543..70bdda5855 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,6 +261,104 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_VNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_VNODE; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { @@ -278,6 +376,8 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { } static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1; if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; @@ -299,10 +399,12 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; + if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; - if (pCfg->strict > pCfg->replications) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; + + terrno = 0; return TSDB_CODE_SUCCESS; } @@ -381,24 +483,8 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } @@ -412,24 +498,8 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, false) != 0) { return -1; } } @@ -482,7 +552,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate } if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -570,37 +639,37 @@ _OVER: static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) { + if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) { pDb->cfg.buffer = pAlter->buffer; terrno = 0; } - if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) { + if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) { pDb->cfg.pages = pAlter->pages; terrno = 0; } - if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) { + if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) { pDb->cfg.pageSize = pAlter->pageSize; terrno = 0; } - if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { + if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { pDb->cfg.daysPerFile = pAlter->daysPerFile; terrno = 0; } - if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { + if (pAlter->daysToKeep0 > 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; terrno = 0; } - if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { + if (pAlter->daysToKeep1 > 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; terrno = 0; } - if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { + if (pAlter->daysToKeep2 > 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; terrno = 0; } @@ -625,8 +694,9 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } - if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { + if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) { pDb->cfg.replications = pAlter->replications; + pDb->vgVersion++; terrno = 0; } @@ -651,35 +721,57 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { + if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { + return -1; + } } + } else { + SVgObj newVgroup = {0}; + memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + if (newVgroup.replica < pDb->cfg.replications) { + mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, + pVgroup->vnodeGid[0].dnodeId); + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) { + mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + } else { + mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); + + SVnodeGid del1 = {0}; + SVnodeGid del2 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1, &del2) != 0) { + mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + } + + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } return 0; } static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SArray *pArray = mndBuildDnodesArray(pMnode); while (1) { SVgObj *pVgroup = NULL; @@ -687,9 +779,10 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); + taosArrayDestroy(pArray); return -1; } } @@ -697,6 +790,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * sdbRelease(pSdb, pVgroup); } + taosArrayDestroy(pArray); return 0; } @@ -726,6 +820,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { SDbObj *pDb = NULL; SUserObj *pUser = NULL; SAlterDbReq alterReq = {0}; + SDbObj dbObj = {0}; if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -749,15 +844,17 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { goto _OVER; } - SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - dbObj.cfg.numOfRetensions = 0; - dbObj.cfg.pRetensions = NULL; + if (dbObj.cfg.pRetensions != NULL) { + dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions); + if (dbObj.cfg.pRetensions == NULL) goto _OVER; + } code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); - if (code != 0) { - goto _OVER; - } + if (code != 0) goto _OVER; + + code = mndCheckDbCfg(pMnode, &dbObj.cfg); + if (code != 0) goto _OVER; dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); @@ -771,6 +868,7 @@ _OVER: mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); + taosArrayDestroy(dbObj.cfg.pRetensions); return code; } @@ -899,24 +997,8 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } @@ -988,6 +1070,17 @@ static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; + SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser); + if (pUser != NULL) { + pUser->authVersion++; + SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + goto _OVER; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + } + int32_t rspLen = 0; void *pRsp = NULL; if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 5bce6ac218..b8c120905c 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndDef.h" #include "mndConsumer.h" SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) { SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -515,3 +517,16 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { #endif return 0; } + +int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pOffset->key); + tlen += taosEncodeFixedI64(buf, pOffset->offset); + return tlen; +} + +void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { + buf = taosDecodeStringTo(buf, pOffset->key); + buf = taosDecodeFixedI64(buf, &pOffset->offset); + return buf; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8619df978b..02a8eaa832 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -21,7 +21,7 @@ #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndShow.h" -#include "mndStb.c" +#include "mndStb.h" #include "mndStream.h" #include "mndTrans.h" #include "mndUser.h" @@ -419,7 +419,6 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre streamObj.fixedSinkVgId = smaObj.dstVgId; streamObj.smaId = smaObj.uid; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8ae0d5d19c..b6912d0777 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndStb.h" #include "mndAuth.h" #include "mndDb.h" @@ -27,8 +28,8 @@ #include "mndVgroup.h" #include "tname.h" -#define TSDB_STB_VER_NUMBER 1 -#define TSDB_STB_RESERVE_SIZE 64 +#define STB_VER_NUMBER 1 +#define STB_RESERVE_SIZE 64 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); @@ -45,13 +46,15 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBl static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); int32_t mndInitStb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_STB, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndStbActionEncode, - .decodeFp = (SdbDecodeFp)mndStbActionDecode, - .insertFp = (SdbInsertFp)mndStbActionInsert, - .updateFp = (SdbUpdateFp)mndStbActionUpdate, - .deleteFp = (SdbDeleteFp)mndStbActionDelete}; + SSdbTable table = { + .sdbType = SDB_STB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStbActionEncode, + .decodeFp = (SdbDecodeFp)mndStbActionDecode, + .insertFp = (SdbInsertFp)mndStbActionInsert, + .updateFp = (SdbUpdateFp)mndStbActionUpdate, + .deleteFp = (SdbDeleteFp)mndStbActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessMCreateStbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessMAlterStbReq); @@ -73,8 +76,8 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen + - pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); + pStb->ast1Len + pStb->ast2Len + STB_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; @@ -98,6 +101,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSchema->flags, _OVER) SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -106,6 +110,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSchema->flags, _OVER) SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -120,7 +125,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { if (pStb->ast2Len > 0) { SDB_SET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) + SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; @@ -142,7 +147,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_STB_VER_NUMBER) { + if (sver != STB_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -182,6 +187,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSchema->flags, _OVER) SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -190,6 +196,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSchema->flags, _OVER) SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -210,7 +217,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { if (pStb->pAst2 == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) + SDB_GET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) terrno = 0; @@ -342,7 +349,7 @@ void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) { sdbRelease(pSdb, pStb); } -static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { +SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { SName name = {0}; tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -463,7 +470,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, return pHead; } -static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { +int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { if (pCreate->igExists < 0 || pCreate->igExists > 1) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -487,7 +494,7 @@ static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { SField *pField1 = taosArrayGet(pCreate->pColumns, i); - if (pField->type < 0) { + if (pField1->type < 0) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -573,6 +580,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_CREATE_STB; + action.acceptableCode = TSDB_CODE_TDB_STB_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); @@ -612,6 +620,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; + action.acceptableCode = TSDB_CODE_TDB_STB_NOT_EXIST; if (mndTransAppendUndoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); @@ -634,91 +643,96 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { return NULL; } -static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { - SStbObj stbObj = {0}; - memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); - memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); - stbObj.createdTime = taosGetTimestampMs(); - stbObj.updateTime = stbObj.createdTime; - stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); - stbObj.dbUid = pDb->uid; - stbObj.version = 1; - stbObj.nextColId = 1; - stbObj.xFilesFactor = pCreate->xFilesFactor; - stbObj.delay = pCreate->delay; - stbObj.ttl = pCreate->ttl; - stbObj.numOfColumns = pCreate->numOfColumns; - stbObj.numOfTags = pCreate->numOfTags; - stbObj.commentLen = pCreate->commentLen; - if (stbObj.commentLen > 0) { - stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); - if (stbObj.comment == NULL) { +int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) { + memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN); + memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); + pDst->createdTime = taosGetTimestampMs(); + pDst->updateTime = pDst->createdTime; + pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + pDst->dbUid = pDb->uid; + pDst->version = 1; + pDst->nextColId = 1; + pDst->xFilesFactor = pCreate->xFilesFactor; + pDst->delay = pCreate->delay; + pDst->ttl = pCreate->ttl; + pDst->numOfColumns = pCreate->numOfColumns; + pDst->numOfTags = pCreate->numOfTags; + pDst->commentLen = pCreate->commentLen; + if (pDst->commentLen > 0) { + pDst->comment = taosMemoryCalloc(pDst->commentLen, 1); + if (pDst->comment == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); + memcpy(pDst->comment, pCreate->comment, pDst->commentLen); } - stbObj.ast1Len = pCreate->ast1Len; - if (stbObj.ast1Len > 0) { - stbObj.pAst1 = taosMemoryCalloc(stbObj.ast1Len, 1); - if (stbObj.pAst1 == NULL) { + pDst->ast1Len = pCreate->ast1Len; + if (pDst->ast1Len > 0) { + pDst->pAst1 = taosMemoryCalloc(pDst->ast1Len, 1); + if (pDst->pAst1 == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.pAst1, pCreate->pAst1, stbObj.ast1Len); + memcpy(pDst->pAst1, pCreate->pAst1, pDst->ast1Len); } - stbObj.ast2Len = pCreate->ast2Len; - if (stbObj.ast2Len > 0) { - stbObj.pAst2 = taosMemoryCalloc(stbObj.ast2Len, 1); - if (stbObj.pAst2 == NULL) { + pDst->ast2Len = pCreate->ast2Len; + if (pDst->ast2Len > 0) { + pDst->pAst2 = taosMemoryCalloc(pDst->ast2Len, 1); + if (pDst->pAst2 == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.pAst2, pCreate->pAst2, stbObj.ast2Len); + memcpy(pDst->pAst2, pCreate->pAst2, pDst->ast2Len); } - stbObj.pColumns = taosMemoryCalloc(1, stbObj.numOfColumns * sizeof(SSchema)); - stbObj.pTags = taosMemoryCalloc(1, stbObj.numOfTags * sizeof(SSchema)); - if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { + pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema)); + pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema)); + if (pDst->pColumns == NULL || pDst->pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < stbObj.numOfColumns; ++i) { + for (int32_t i = 0; i < pDst->numOfColumns; ++i) { SField *pField = taosArrayGet(pCreate->pColumns, i); - SSchema *pSchema = &stbObj.pColumns[i]; + SSchema *pSchema = &pDst->pColumns[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; pSchema->flags = pField->flags; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); - pSchema->colId = stbObj.nextColId; - stbObj.nextColId++; + pSchema->colId = pDst->nextColId; + pDst->nextColId++; } - for (int32_t i = 0; i < stbObj.numOfTags; ++i) { + for (int32_t i = 0; i < pDst->numOfTags; ++i) { SField *pField = taosArrayGet(pCreate->pTags, i); - SSchema *pSchema = &stbObj.pTags[i]; + SSchema *pSchema = &pDst->pTags[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); - pSchema->colId = stbObj.nextColId; - stbObj.nextColId++; + pSchema->colId = pDst->nextColId; + pDst->nextColId++; } + return 0; +} + +static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { + SStbObj stbObj = {0}; int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); - mndTransSetDbInfo(pTrans, pDb); - if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) { + goto _OVER; + } + + if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -728,6 +742,16 @@ _OVER: return code; } +int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + mndTransSetDbInfo(pTrans, pDb); + if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; + return 0; +} + static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; @@ -777,7 +801,10 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { } int32_t numOfStbs = -1; - mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs); + if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { + goto _OVER; + } + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; goto _OVER; @@ -804,7 +831,7 @@ static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndCheckAlterStbReq(SMAltertbReq *pAlter) { +static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -1155,7 +1182,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { +static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); @@ -1219,12 +1246,12 @@ _OVER: } static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SDbObj *pDb = NULL; - SStbObj *pStb = NULL; - SUserObj *pUser = NULL; - SMAltertbReq alterReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SUserObj *pUser = NULL; + SMAlterStbReq alterReq = {0}; if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5c6e2ce771..2c767619a1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -290,6 +290,86 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast return 0; } +static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { + SStbObj *pStb = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + + SMCreateStbReq createReq = {0}; + tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); + createReq.numOfColumns = pStream->outputSchema.nCols; + createReq.numOfTags = 1; // group id + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + // build fields + taosArraySetSize(createReq.pColumns, createReq.numOfColumns); + for (int32_t i = 0; i < createReq.numOfColumns; i++) { + SField *pField = taosArrayGet(createReq.pColumns, i); + tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN); + pField->flags = pStream->outputSchema.pSchema[i].flags; + pField->type = pStream->outputSchema.pSchema[i].type; + pField->bytes = pStream->outputSchema.pSchema[i].bytes; + } + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, 1); + // build tags + SField *pField = taosArrayGet(createReq.pTags, 0); + strcpy(pField->name, "group_id"); + pField->type = TSDB_DATA_TYPE_UBIGINT; + pField->flags = 0; + pField->bytes = 8; + + if (mndCheckCreateStbReq(&createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.name); + if (pStb != NULL) { + terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; + goto _OVER; + } + + pDb = mndAcquireDbByStb(pMnode, createReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + int32_t numOfStbs = -1; + if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { + goto _OVER; + } + + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { + terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; + goto _OVER; + } + + SStbObj stbObj = {0}; + + if (mndBuildStbFromReq(pMnode, &stbObj, &createReq, pDb) != 0) { + goto _OVER; + } + + if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; + + return 0; +_OVER: + mndReleaseStb(pMnode, pStb); + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + return -1; +} + static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; @@ -307,11 +387,10 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.fixedSinkVgId = 0; streamObj.smaId = 0; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; streamObj.trigger = pCreate->triggerType; streamObj.waterMark = pCreate->watermark; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -324,6 +403,12 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe return -1; } + if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 43e2735984..e57e9a0461 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1336,8 +1336,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { goto _OVER; } - if (!pUser->superUser) { - terrno = TSDB_CODE_MND_NO_RIGHTS; + if (mndCheckTransAuth(pUser) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 8b5fa06faf..81ded4ef8b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -106,6 +106,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) + SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) @@ -161,6 +162,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -588,7 +590,7 @@ _OVER: return code; } -static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = pUser->superUser; pRsp->version = pUser->authVersion; @@ -601,7 +603,7 @@ static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUser terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -659,7 +661,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: - + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -711,7 +713,8 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen) { SUserAuthBatchRsp batchRsp = {0}; batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); if (batchRsp.pArray == NULL) { @@ -731,7 +734,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ mndReleaseUser(pMnode, pUser); continue; } - + SGetUserAuthRsp rsp = {0}; code = mndSetUserAuthRsp(pMnode, pUser, &rsp); if (code) { @@ -740,7 +743,6 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ goto _OVER; } - taosArrayPush(batchRsp.pArray, &rsp); mndReleaseUser(pMnode, pUser); } @@ -748,7 +750,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ if (taosArrayGetSize(batchRsp.pArray) <= 0) { *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return 0; } @@ -772,10 +774,7 @@ _OVER: *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return code; } - - - diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index d1e4be1161..48a475ec63 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -370,7 +370,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 return true; } -static SArray *mndBuildDnodesArray(SMnode *pMnode) { +SArray *mndBuildDnodesArray(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); @@ -421,7 +421,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } - mDebug("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); pDnode->numOfVnodes++; } @@ -440,12 +440,10 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { } pArray = mndBuildDnodesArray(pMnode); - if (pArray == NULL) { - goto _OVER; - } + if (pArray == NULL) goto _OVER; - mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), - pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); + mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), + pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); int32_t allocedVgroups = 0; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); @@ -483,7 +481,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { *ppVgroups = pVgroups; code = 0; - mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); + mInfo("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); _OVER: if (code != 0) taosMemoryFree(pVgroups); @@ -491,6 +489,88 @@ _OVER: return code; } +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + int32_t maxPos = 1; + for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + bool used = false; + for (int32_t vn = 0; vn < maxPos; ++vn) { + if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { + used = true; + break; + } + } + if (used) continue; + + if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + + SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; + pVgid->dnodeId = pDnode->id; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pDnode->numOfVnodes++; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); + maxPos++; + if (maxPos == 3) return 0; + } + + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; +} + +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { + int32_t removedNum = 0; + + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + for (int32_t vn = 0; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId == pDnode->id) { + if (removedNum == 0) *del1 = *pVgid; + if (removedNum == 1) *del2 = *pVgid; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + memset(pVgid, 0, sizeof(SVnodeGid)); + removedNum++; + pDnode->numOfVnodes--; + + if (removedNum == 2) goto _OVER; + } + } + } + +_OVER: + if (removedNum != 2) return -1; + + for (int32_t vn = 1; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId != 0) { + memcpy(&pVgroup->vnodeGid[0], pVgid, sizeof(SVnodeGid)); + memset(pVgid, 0, sizeof(SVnodeGid)); + } + } + + mInfo("db:%s, vgId:%d, dnode:%d is keeped", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); + return 0; +} + SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 3c5bf5c083..545f9f22bb 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -73,14 +73,19 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { SAlterDbReq alterdbReq = {0}; strcpy(alterdbReq.db, "1.d1"); + alterdbReq.buffer = 12; - alterdbReq.daysToKeep0 = 300; - alterdbReq.daysToKeep1 = 400; - alterdbReq.daysToKeep2 = 500; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; - alterdbReq.strict = 2; + alterdbReq.strict = 1; alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/test/dnode/mdnode.cpp b/source/dnode/mnode/impl/test/dnode/mdnode.cpp index a4cbc201a9..15ade8166f 100644 --- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp @@ -266,4 +266,96 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { taosMsleep(1300); test.SendShowReq(TSDB_MGMT_TABLE_DNODE, "dnodes", ""); EXPECT_EQ(test.GetShowRows(), 4); + + // alter replica +#if 0 + { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; + createReq.daysPerFile = 1000; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.strict = 1; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + createReq.numOfStables = 0; + createReq.numOfRetensions = 0; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + EXPECT_EQ(test.GetShowRows(), 3); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 3; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + +#endif } diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 220a73def6..16974ad541 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -137,7 +137,7 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { } void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -158,7 +158,7 @@ void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagnam } void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -180,7 +180,7 @@ void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagna void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 2; req.pFields = taosArrayInit(2, sizeof(SField)); @@ -208,7 +208,7 @@ void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char* void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char* tagname, int32_t bytes, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -229,7 +229,7 @@ void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char } void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* colname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -250,7 +250,7 @@ void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* col } void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* colname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -272,7 +272,7 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 7a226a4c6b..b21141001a 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -84,6 +84,8 @@ int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { // return vnodeGetTableMeta(pQnode, pMsg); case TDMT_VND_CONSUME: // return tqProcessConsumeReq(pQnode->pTq, pMsg); + case TDMT_VND_QUERY_HEARTBEAT: + return qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg); default: qError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2716692bd7..58003a97d7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -273,7 +273,8 @@ typedef enum { typedef struct { uint8_t last : 1; - uint8_t blkVer : 7; + uint8_t hasDupKey : 1; // 0: no dup TS key, 1: has dup TS key(since supporting Multi-Version) + uint8_t blkVer : 6; uint8_t numOfSubBlocks; col_id_t numOfCols; // not including timestamp column uint32_t len; // data block length @@ -324,9 +325,8 @@ typedef struct { typedef struct { int16_t colId; uint16_t type : 6; - uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) - uint32_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint32_t len : 31; // data length + bitmap length + uint16_t blen : 10; // 0 no bitmap if all rows are NORM, > 0 bitmap length + uint32_t len; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 397e074061..a68e176fa9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -196,12 +196,12 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int8_t type; int64_t ctime; tb_uid_t suid; - int c, ret; + int c = 0, ret; // search & delete the name idx tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); - if (ret < 0 || c) { + if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { tdbDbcClose(pNameIdxc); terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5ec3ab0b47..f531d3f5fb 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -173,7 +173,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t curRow = 0; tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - *pUid = pHandle->pBlock->uid; // set the uid of table for submit block + *pUid = pHandle->msgIter.uid; // set the uid of table for submit block while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 97b796c6a5..f0b1baf1da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -943,16 +943,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF &(pAggrBlkCol->numOfNull)); if (pAggrBlkCol->numOfNull == 0) { - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsOfBlockSma; } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) { // check if all rows normal - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsNotAllNull; @@ -985,7 +985,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; int32_t tBitmapsLen = 0; - if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((ncol != 0) && (pBlockCol->blen > 0)) { tBitmaps = isSuper ? sBitmaps : nBitmaps; } #endif @@ -1330,13 +1330,15 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt TSKEY maxKey, int maxRows, int8_t update) { TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; + TSKEY lastKey = TSKEY_INITIAL_VAL; STSchema *pSchema = NULL; ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); tdResetDataCols(pTarget); pTarget->bitmapMode = pDataCols->bitmapMode; - + // TODO: filter Multi-Version + // TODO: support delete function while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); STSRow *row = tsdbNextIterRow(pCommitIter->pIter); @@ -1349,6 +1351,9 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt if (key1 == INT64_MAX && key2 == INT64_MAX) break; if (key1 < key2) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } for (int i = 0; i < pDataCols->numOfCols; ++i) { // TODO: dataColAppendVal may fail SCellVal sVal = {0}; @@ -1356,10 +1361,10 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt TASSERT(0); } tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, - pTarget->bitmapMode); + pTarget->bitmapMode, false); } - ++pTarget->numOfRows; + lastKey = key1; ++(*iter); } else if (key1 > key2) { if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { @@ -1367,7 +1372,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendSTSRowToDataCol(row, pSchema, pTarget); + if (key2 == lastKey) { + if (TD_SUPPORT_UPDATE(update)) { + tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); + } + } else { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); + lastKey = key2; + } tSkipListIterNext(pCommitIter->pIter); } else { @@ -1397,6 +1412,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ++(*iter); tSkipListIterNext(pCommitIter->pIter); #endif + + if(lastKey != key1) { + lastKey = key1; + ++pTarget->numOfRows; + } + // copy disk data for (int i = 0; i < pDataCols->numOfCols; ++i) { SCellVal sVal = {0}; @@ -1405,7 +1426,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt } // TODO: tdAppendValToDataCol may fail tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, - pTarget->bitmapMode); + pTarget->bitmapMode, false); } if (TD_SUPPORT_UPDATE(update)) { @@ -1416,26 +1437,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt } // TODO: merge with Multi-Version - STSRow *curRow = row; - - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); - STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter); - - if (key2 < TD_ROW_KEY(nextRow)) { - tdAppendSTSRowToDataCol(row, pSchema, pTarget); - } else { - tdAppendSTSRowToDataCol(row, pSchema, pTarget); - } - // TODO: merge with Multi-Version - } else { - ++pTarget->numOfRows; - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); } + ++(*iter); + tSkipListIterNext(pCommitIter->pIter); } - if (pTarget->numOfRows >= maxRows) break; + if (pTarget->numOfRows >= (maxRows - 1)) break; + } + + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index b52a34cfb2..48d32c0dd3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -20,7 +20,7 @@ static void tsdbFreeTbData(STbData *pTbData); static char *tsdbGetTsTupleKey(const void *data); static int tsdbTbDataComp(const void *arg1, const void *arg2); static char *tsdbTbDataGetUid(const void *arg); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge); int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) { STsdbMemTable *pMemTable; @@ -82,14 +82,19 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; - STSchema *pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; + STSchema *pSchema = NULL; + TSKEY rowKey = 0; + TSKEY fKey = 0; + // only fetch lastKey from mem data as file data not used in this function actually + TSKEY lastKey = TSKEY_INITIAL_VAL; bool isRowDel = false; int filterIter = 0; STSRow *row = NULL; SMergeInfo mInfo; + // TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by + // query handle) + if (pMergeInfo == NULL) pMergeInfo = &mInfo; memset(pMergeInfo, 0, sizeof(*pMergeInfo)); @@ -111,7 +116,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } else { fKey = tdGetKey(filterKeys[filterIter]); } - + // 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols + // 2. rowKey - would dup since Multi-Version supported while (true) { if (fKey == INT64_MAX && rowKey == INT64_MAX) break; @@ -125,12 +131,14 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } else { fKey = tdGetKey(filterKeys[filterIter]); } +#if 0 } else if (fKey > rowKey) { if (isRowDel) { pMergeInfo->rowsDeleteFailed++; } else { if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsInserted++; pMergeInfo->nOperations++; pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); @@ -185,6 +193,94 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey fKey = tdGetKey(filterKeys[filterIter]); } } +#endif +#if 1 + } else if (fKey > rowKey) { + if (isRowDel) { + // TODO: support delete function + pMergeInfo->rowsDeleteFailed++; + } else { + if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + + if (lastKey != rowKey) { + pMergeInfo->rowsInserted++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + if (pCols) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pCols->numOfRows; + } + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); + } + lastKey = rowKey; + } else { + if (keepDup) { + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true); + } else { + // discard + } + } + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || TD_ROW_KEY(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = TD_ROW_KEY(row); + isRowDel = TD_ROW_IS_DELETED(row); + } + } else { // fkey == rowKey + if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?) + ASSERT(!keepDup); + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsDeleteSucceed++; + pMergeInfo->nOperations++; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); + } else { + if (keepDup) { + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + if (lastKey != rowKey) { + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + lastKey = rowKey; + ++pCols->numOfRows; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); + } else { + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true); + } + } else { + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); + } + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || TD_ROW_KEY(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = TD_ROW_KEY(row); + isRowDel = TD_ROW_IS_DELETED(row); + } + + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } +#endif + } + if (lastKey != TSKEY_INITIAL_VAL) { + ++pCols->numOfRows; } return 0; @@ -254,9 +350,12 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { pTbData->keyMin = TSKEY_MAX; pTbData->keyMax = TSKEY_MIN; pTbData->nrows = 0; - +#if 0 pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); +#endif + pTbData->pData = + tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, tsdbGetTsTupleKey); if (pTbData->pData == NULL) { taosMemoryFree(pTbData); return NULL; @@ -291,7 +390,7 @@ static char *tsdbTbDataGetUid(const void *arg) { STbData *pTbData = (STbData *)arg; return (char *)(&(pTbData->uid)); } -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row) { +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge) { if (pCols) { if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) { *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, TD_ROW_SVER(row)); @@ -301,7 +400,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } } - tdAppendSTSRowToDataCol(row, *ppSchema, pCols); + tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge); } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index ae0d4630d1..e7b733369a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,10 +15,11 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListCfg SMemSkipListCfg; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemSkipListCurosr SMemSkipListCurosr; struct SMemTable { STsdb *pTsdb; @@ -32,15 +33,17 @@ struct SMemTable { SMemData **pBuckets; }; -struct SMemSkipListCfg { - int8_t maxLevel; - int32_t nKey; - int32_t nData; +struct SMemSkipListNode { + int8_t level; + SMemSkipListNode *forwards[]; }; struct SMemSkipList { - int8_t level; - uint32_t seed; + uint32_t seed; + int8_t maxLevel; + int8_t level; + int32_t size; + SMemSkipListNode pHead[]; }; struct SMemData { @@ -55,6 +58,20 @@ struct SMemData { SMemSkipList sl; }; +struct SMemSkipListCurosr { + SMemSkipList *pSl; + SMemSkipListNode *pNodeC; +}; + +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) +#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +#define SL_HEAD_NODE(sl) ((sl)->pHead) +#define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel)) + // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { SMemTable *pMemTb = NULL; @@ -76,6 +93,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); if (pMemTb->pBuckets == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pMemTb); return -1; } @@ -83,8 +101,121 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { return 0; } -int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { - // TODO +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { + if (pMemTb) { + // loop to destroy the contents (todo) + taosMemoryFree(pMemTb->pBuckets); + taosMemoryFree(pMemTb); + } + return 0; +} + +int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { + SMemData *pMemData; + STsdb *pTsdb = pMemTb->pTsdb; + SVnode *pVnode = pTsdb->pVnode; + SVBufPool *pPool = pVnode->inUse; + int32_t hash; + int32_t tlen; + uint8_t buf[16]; + int32_t rlen; + const uint8_t *p; + SMemSkipListNode *pSlNode; + const STSRow *pTSRow; + SMemSkipListCurosr slc = {0}; + + // search hash + hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket; + for (pMemData = pMemTb->pBuckets[hash]; pMemData; pMemData = pMemData->pHashNext) { + if (pMemData->suid == pSubmitBlk->suid && pMemData->uid == pSubmitBlk->uid) break; + } + + // create pMemData if need + if (pMemData == NULL) { + int8_t maxLevel = pVnode->config.tsdbCfg.slLevel; + int32_t tsize = sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2; + SMemSkipListNode *pHead, *pTail; + + pMemData = vnodeBufPoolMalloc(pPool, tsize); + if (pMemData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemData->pHashNext = NULL; + pMemData->suid = pSubmitBlk->suid; + pMemData->uid = pSubmitBlk->uid; + pMemData->minKey = TSKEY_MAX; + pMemData->maxKey = TSKEY_MIN; + pMemData->minVer = -1; + pMemData->maxVer = -1; + pMemData->nRows = 0; + pMemData->sl.seed = taosRand(); + pMemData->sl.maxLevel = maxLevel; + pMemData->sl.level = 0; + pMemData->sl.size = 0; + pHead = SL_HEAD_NODE(&pMemData->sl); + pTail = SL_TAIL_NODE(&pMemData->sl); + pHead->level = maxLevel; + pTail->level = maxLevel; + for (int iLevel = 0; iLevel < maxLevel; iLevel++) { + SL_NODE_FORWARD(pHead, iLevel) = pTail; + SL_NODE_FORWARD(pTail, iLevel) = pHead; + } + + // add to MemTable + hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket; + pMemData->pHashNext = pMemTb->pBuckets[hash]; + pMemTb->pBuckets[hash] = pMemData; + pMemTb->nHash++; + } + +// loop to insert data to skiplist +#if 0 + tsdbMemSkipListCursorOpen(&slc, &pMemData->sl); + p = pSubmitBlk->pData; + for (;;) { + if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break; + + const uint8_t *pt = p; + p = tGetBinary(p, &pTSRow, &rlen); + + // check the row (todo) + + // move the cursor to position to write (todo) + int32_t c; + tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c); + ASSERT(c); + + // encode row + int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); + int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (p - pt); + pSlNode = vnodeBufPoolMalloc(pPool, tsize); + pSlNode->level = level; + + uint8_t *pData = SL_NODE_DATA(pSlNode); + *(int64_t *)pData = version; + pData += sizeof(version); + memcpy(pData, pt, p - pt); + + // insert row + tsdbMemSkipListCursorPut(&slc, pSlNode); + + // update status + if (pTSRow->ts < pMemData->minKey) pMemData->minKey = pTSRow->ts; + if (pTSRow->ts > pMemData->maxKey) pMemData->maxKey = pTSRow->ts; + } + tsdbMemSkipListCursorClose(&slc); +#endif + + if (pMemData->minVer == -1) pMemData->minVer = version; + if (pMemData->maxVer == -1 || pMemData->maxVer < version) pMemData->maxVer = version; + + if (pMemTb->minKey < pMemData->minKey) pMemTb->minKey = pMemData->minKey; + if (pMemTb->maxKey < pMemData->maxKey) pMemTb->maxKey = pMemData->maxKey; + if (pMemTb->minVer == -1) pMemTb->minVer = version; + if (pMemTb->maxVer == -1 || pMemTb->maxVer < version) pMemTb->maxVer = version; + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0da18ebdeb..0b9f21dbcf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -163,7 +163,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); // static void* destroyTableCheckInfo(SArray* pTableCheckInfo); static bool tsdbGetExternalRow(tsdbReaderT pHandle); -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions); +static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -351,36 +351,43 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } -#if 0 -int nQUERY = 0; +#if 1 +int nQUERY = 0; #endif -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { +static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { int level = 0; -#if 1 +#if 0 int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + i; if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { break; } + ++level; } #endif -#if 0 - ++nQUERY; - if(nQUERY%3 == 0) { - level = 2; - } else if(nQUERY%2 == 0) { - level = 1; - } else { - level = 0; +#if 1 + switch ((nQUERY++) % 3) { + case 0: + level = 0; + break; + case 1: + level = 1; + break; + default: + level = 2; + break; } #endif if (level == TSDB_RETENTION_L0) { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA1(pVnode); } else { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA2(pVnode); } } @@ -393,7 +400,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* goto _end; } - STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); + STsdb* pTsdb = getTsdbByRetentions(pVnode, pReadHandle, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); pReadHandle->order = pCond->order; pReadHandle->pTsdb = pTsdb; @@ -803,12 +810,16 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { +static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) { STSRow *rmem = NULL, *rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); + // TODO: filter max version + // if (TD_ROW_VER(rmem) > maxVer) { + // rmem = NULL; + // } } } @@ -816,6 +827,10 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); + // TODO: filter max version + // if (TD_ROW_VER(rimem) > maxVer) { + // rimem = NULL; + // } } } @@ -837,6 +852,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, TSKEY r2 = TD_ROW_KEY(rimem); if (r1 == r2) { +#if 0 if (update == TD_ROW_DISCARD_UPDATE) { pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; tSkipListIterNext(pCheckInfo->iter); @@ -846,6 +862,13 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, } else { pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; } +#endif + if (TD_SUPPORT_UPDATE(update)) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + } else { + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + tSkipListIterNext(pCheckInfo->iter); + } return r1; } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) { pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; @@ -856,12 +879,18 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, } } -static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow) { + +static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow, TDRowVerT maxVer) { STSRow *rmem = NULL, *rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); +#if 0 // TODO: skiplist refactor + if (TD_ROW_VER(rmem) > maxVer) { + rmem = NULL; + } +#endif } } @@ -869,6 +898,11 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); +#if 0 // TODO: skiplist refactor + if (TD_ROW_VER(rimem) > maxVer) { + rimem = NULL; + } +#endif } } @@ -890,6 +924,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int TSKEY r2 = TD_ROW_KEY(rimem); if (r1 == r2) { +#if 0 if (update == TD_ROW_DISCARD_UPDATE) { tSkipListIterNext(pCheckInfo->iter); pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; @@ -903,6 +938,16 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int *extraRow = rimem; return rmem; } +#endif + if (TD_SUPPORT_UPDATE(update)) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + *extraRow = rimem; + return rmem; + } else { + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + return rimem; + } } else { if (ASCENDING_TRAVERSE(order)) { if (r1 < r2) { @@ -973,7 +1018,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL); + STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL, TD_VER_MAX); if (row == NULL) { return false; } @@ -1250,7 +1295,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); + key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update, TD_VER_MAX); if (key != TSKEY_INITIAL_VAL) { tsdbDebug("%p key in mem:%" PRId64 ", %s", pTsdbReadHandle, key, pTsdbReadHandle->idStr); @@ -1497,10 +1542,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t TASSERT(0); } - if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, rowIndex); - } else { + if (sVal.valType == TD_VTYPE_NORM) { colDataAppend(pColInfo, rowIndex, sVal.val, false); + } else { + colDataAppendNULL(pColInfo, rowIndex); } } } else { // handle the var-string @@ -1513,10 +1558,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t TASSERT(0); } - if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, rowIndex); - } else { + if (sVal.valType == TD_VTYPE_NORM) { colDataAppend(pColInfo, rowIndex, sVal.val, false); + } else { + colDataAppendNULL(pColInfo, rowIndex); } } } @@ -1541,11 +1586,26 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t return numOfRows + num; } -// TODO fix bug for reverse copy data problem -// Note: row1 always has high priority -static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1, - STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, - bool forceSetNull) { +/** + * @brief // TODO fix bug for reverse copy data problem + * Note: row1 always has high priority + * + * @param pTsdbReadHandle + * @param capacity + * @param curRow + * @param row1 + * @param row2 + * @param numOfCols + * @param uid + * @param pSchema1 + * @param pSchema2 + * @param update + * @param lastRowKey + * @return int32_t The quantity of rows appended + */ +static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t* curRow, STSRow* row1, + STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, + bool update, TSKEY* lastRowKey) { #if 1 STSchema* pSchema; STSRow* row; @@ -1557,12 +1617,17 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit bool isChosenRowDataRow; int32_t chosen_itr; SCellVal sVal = {0}; + TSKEY rowKey = TSKEY_INITIAL_VAL; + int32_t nResult = 0; + int32_t mergeOption = 0; // 0 discard 1 overwrite 2 merge // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); + // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); + // TODO: use the real schemaVersion + pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); } #ifdef TD_DEBUG_PRINT_ROW @@ -1579,7 +1644,9 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit if (row2) { isRow2DataRow = TD_IS_TP_ROW(row2); if (pSchema2 == NULL) { - pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); + // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); + // TODO: use the real schemaVersion + pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); } if (isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1610,19 +1677,19 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit colIdOfRow2 = tdKvRowColIdAt(row2, k); } - if (colIdOfRow1 == colIdOfRow2) { + if (colIdOfRow1 < colIdOfRow2) { // the most probability if (colIdOfRow1 < pColInfo->info.colId) { - j++; - k++; + ++j; continue; } row = row1; pSchema = pSchema1; isChosenRowDataRow = isRow1DataRow; chosen_itr = j; - } else if (colIdOfRow1 < colIdOfRow2) { + } else if (colIdOfRow1 == colIdOfRow2) { if (colIdOfRow1 < pColInfo->info.colId) { - j++; + ++j; + ++k; continue; } row = row1; @@ -1631,7 +1698,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit chosen_itr = j; } else { if (colIdOfRow2 < pColInfo->info.colId) { - k++; + ++k; continue; } row = row2; @@ -1639,16 +1706,48 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit chosen_itr = k; isChosenRowDataRow = isRow2DataRow; } + if (isChosenRowDataRow) { colId = pSchema->columns[chosen_itr].colId; offset = pSchema->columns[chosen_itr].offset; // TODO: use STSRowIter tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr - 1, &sVal); + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + rowKey = *(TSKEY*)sVal.val; + if (rowKey != *lastRowKey) { + mergeOption = 1; + if (*lastRowKey != TSKEY_INITIAL_VAL) { + ++(*curRow); + } + ++nResult; + } else if (update){ + mergeOption = 2; + } else { + mergeOption = 0; + break; + } + + *lastRowKey = rowKey; + } } else { // TODO: use STSRowIter if (chosen_itr == 0) { colId = PRIMARYKEY_TIMESTAMP_COL_ID; tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); + rowKey = *(TSKEY*)sVal.val; + if (rowKey != *lastRowKey) { + mergeOption = 1; + if (*lastRowKey != TSKEY_INITIAL_VAL) { + ++(*curRow); + } + ++nResult; + } else if(update) { + mergeOption = 2; + } else { + mergeOption = 0; + break; + } + *lastRowKey = rowKey; } else { SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1); colId = pColIdx->colId; @@ -1657,35 +1756,46 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit } } + ASSERT(rowKey != TSKEY_INITIAL_VAL); + if (colId == pColInfo->info.colId) { if (tdValTypeIsNorm(sVal.valType)) { - colDataAppend(pColInfo, numOfRows, sVal.val, false); - } else if (forceSetNull) { - colDataAppend(pColInfo, numOfRows, NULL, true); + colDataAppend(pColInfo, *curRow, sVal.val, false); + } else if (tdValTypeIsNull(sVal.valType)) { + colDataAppend(pColInfo, *curRow, NULL, true); + } else if (tdValTypeIsNone(sVal.valType)) { + // TODO: Set null if nothing append for this row + if (mergeOption == 1) { + colDataAppend(pColInfo, *curRow, NULL, true); + } + } else { + ASSERT(0); } - i++; + ++i; if (row == row1) { - j++; + ++j; } else { - k++; + ++k; } } else { - if (forceSetNull) { - colDataAppend(pColInfo, numOfRows, NULL, true); + if (mergeOption == 1) { + colDataAppend(pColInfo, *curRow, NULL, true); } - i++; + ++i; } } - if (forceSetNull) { + if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - colDataAppend(pColInfo, numOfRows, NULL, true); - i++; + colDataAppend(pColInfo, *curRow, NULL, true); + ++i; } } + + return nResult; #endif } @@ -1859,6 +1969,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; + int32_t curRow = 0; int16_t rv1 = -1; int16_t rv2 = -1; @@ -1874,9 +1985,11 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + do { STSRow* row2 = NULL; - STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2); + STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2, TD_VER_MAX); if (row1 == NULL) { break; } @@ -1905,9 +2018,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf rv2 = TD_ROW_SVER(row2); } - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true); - numOfRows += 1; + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); + // numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; } @@ -1918,6 +2031,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it +#if 0 if (pCfg->update) { if (pCfg->update == TD_ROW_PARTIAL_UPDATE) { doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos); @@ -1933,7 +2047,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull); + pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull, &lastRowKey); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1943,6 +2057,35 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->lastKey = key + step; cur->mixBlock = true; + moveToNextRowInMem(pCheckInfo); + pos += step; + } else { + moveToNextRowInMem(pCheckInfo); + } +#endif + if (TD_SUPPORT_UPDATE(pCfg->update)) { + doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); + + if (rv1 != TD_ROW_SVER(row1)) { + // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + rv1 = TD_ROW_SVER(row1); + } + if (row2 && rv2 != TD_ROW_SVER(row2)) { + // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + rv2 = TD_ROW_SVER(row2); + } + + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); + // ++numOfRows; + if (cur->win.skey == TSKEY_INITIAL_VAL) { + cur->win.skey = key; + } + + cur->win.ekey = key; + cur->lastKey = key + step; + cur->mixBlock = true; + moveToNextRowInMem(pCheckInfo); pos += step; } else { @@ -1958,11 +2101,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf assert(end != -1); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it +#if 0 if (pCfg->update == TD_ROW_DISCARD_UPDATE) { moveToNextRowInMem(pCheckInfo); } else { end -= step; } +#endif + if (!TD_SUPPORT_UPDATE(pCfg->update)) { + moveToNextRowInMem(pCheckInfo); + } else { + end -= step; + } } int32_t qstart = 0, qend = 0; @@ -2572,6 +2722,7 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) { static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; + int curRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; @@ -2579,9 +2730,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int int64_t st = taosGetTimestampUs(); int16_t rv = -1; STSchema* pSchema = NULL; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + do { - STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL); + STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL, TD_VER_MAX); if (row == NULL) { break; } @@ -2604,16 +2757,18 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); rv = TD_ROW_SVER(row); } - mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, - NULL, true); + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, + NULL, pCfg->update, &lastRowKey); - if (++numOfRows >= maxRowsToRead) { + if (numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); break; } } while (moveToNextRowInMem(pCheckInfo)); + taosMemoryFreeClear(pSchema); // free the STSChema + assert(numOfRows <= maxRowsToRead); // if the buffer is not full in case of descending order query, move the data in the front of the buffer @@ -2731,6 +2886,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { STSRow* pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + int32_t curRow = 0; if (++pTsdbReadHandle->activeIndex < numOfTables) { STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); @@ -2738,8 +2895,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // if (ret != TSDB_CODE_SUCCESS) { // return false; // } - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId, - NULL, NULL, true); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, pRow, NULL, numOfCols, pCheckInfo->tableId, + NULL, NULL, true, &lastRowKey); taosMemoryFreeClear(pRow); // update the last key value diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index d7c9a70c18..bebdfb3b63 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -246,6 +246,12 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { return 0; } +static FORCE_INLINE void tsdbSwapDataCols(SDataCols *pDest, SDataCols *pSrc) { + SDataCol *pCols = pDest->cols; + memcpy(pDest, pSrc, sizeof(SDataCols)); + pSrc->cols = pCols; +} + int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { ASSERT(pBlock->numOfSubBlocks > 0); STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo); @@ -266,17 +272,29 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, - update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0) + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; } + // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line + if (pBlock->numOfSubBlocks == 1) { + tdResetDataCols(pReadh->pDCols[1]); + pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode; + if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL, + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { + return -1; + } + tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); + ASSERT(pReadh->pDCols[0]->bitmapMode != 0); + } - ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); return 0; } +// TODO: filter by Multi-Version int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds, bool mergeBitmap) { ASSERT(pBlock->numOfSubBlocks > 0); @@ -297,9 +315,21 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, - update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0) + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; } + // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line + if (pBlock->numOfSubBlocks == 1) { + tdResetDataCols(pReadh->pDCols[1]); + pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode; + if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL, + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { + return -1; + } + tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); + ASSERT(pReadh->pDCols[0]->bitmapMode != 0); + } + if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { @@ -312,7 +342,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } } - ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); @@ -586,7 +616,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tcolId = pBlockCol->colId; toffset = tsdbGetBlockColOffset(pBlockCol); tlen = pBlockCol->len; - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; } else { ASSERT(pDataCol->colId == tcolId); TD_SET_COL_ROWS_NORM(pDataCol); @@ -594,17 +624,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((dcol != 0) && (pBlockCol->blen > 0)) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } if (tcolId == pDataCol->colId) { @@ -623,15 +644,15 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat } if (dcol != 0) { - ccol++; + ++ccol; } - dcol++; + ++dcol; } else if (tcolId < pDataCol->colId) { - ccol++; + ++ccol; } else { // Set current column as NULL and forward dataColReset(pDataCol); - dcol++; + ++dcol; } } @@ -754,8 +775,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { // load the key row blockCol.colId = colId; - TD_SET_COL_ROWS_NORM(&blockCol); // default is NORM for the primary key column - blockCol.blen = 0; + blockCol.blen = 0; // default is NORM for the primary key column blockCol.len = pBlock->keyLen; blockCol.type = pDataCol->type; blockCol.offset = TSDB_KEY_COL_OFFSET; @@ -785,7 +805,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * ASSERT(pBlockCol->colId == pDataCol->colId); } // set the bitmap - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1; } @@ -803,17 +823,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if (!TD_COL_ROWS_NORM(pBlockCol)) { + if (pBlockCol->blen) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 480b131914..105bc330d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1635,17 +1635,20 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { SSmaCfg vCreateSmaReq = {0}; if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); + tsdbWarn("vgId:%d tsma create msg received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); return -1; } - tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName, - vCreateSmaReq.tSma.indexUid); + + tsdbDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", REPO_ID(pTsdb), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); // record current timezone of server side vCreateSmaReq.tSma.timezoneInt = tsTimezone; if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) { // TODO: handle error + tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); tdDestroyTSma(&vCreateSmaReq.tSma); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8460400b59..403c02b440 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -360,7 +360,11 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, // do create table if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) { - cRsp.code = terrno; + if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + cRsp.code = TSDB_CODE_SUCCESS; + } else { + cRsp.code = terrno; + } } else { cRsp.code = TSDB_CODE_SUCCESS; tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); @@ -529,6 +533,13 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } } + msgIter.uid = createTbReq.uid; + if (createTbReq.type == TSDB_CHILD_TABLE) { + msgIter.suid = createTbReq.ctb.suid; + } else { + msgIter.suid = 0; + } + tCoderClear(&coder); } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index b1e8be6528..ade2487239 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -17,4 +17,4 @@ add_subdirectory(tfs) add_subdirectory(monitor) add_subdirectory(nodes) add_subdirectory(scalar) -add_subdirectory(command) +add_subdirectory(command) \ No newline at end of file diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 997397314f..64206fc10a 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -60,4 +60,5 @@ void dsScheduleProcess(void* ahandle, void* pItem) { void dsDestroyDataSinker(DataSinkHandle handle) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; pHandleImpl->fDestroy(pHandleImpl); + taosMemoryFree(pHandleImpl); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 321b7b68a5..34339d3a7f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5311,6 +5311,7 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); + destroyOperatorInfo(pTaskInfo->pRoot); taosMemoryFreeClear(pTaskInfo->sql); taosMemoryFreeClear(pTaskInfo->id.str); taosMemoryFreeClear(pTaskInfo); diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 3b62bdd664..86a28605b2 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -252,8 +252,8 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu return TSDB_CODE_QRY_INVALID_INPUT; } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, - strlen(left->colName), right->condValue, strlen(right->condValue)); + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), + right->condValue, strlen(right->condValue)); if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 8fa4bd0c4d..b22a9866c3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -241,7 +241,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE }; + pFunc->node.resType = (SDataType) { .bytes = 512, .type = TSDB_DATA_TYPE_BINARY }; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3c9eca85dd..0808ec58b0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,13 +14,14 @@ */ #include "builtinsimpl.h" +#include "cJSON.h" #include "function.h" #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" #include "tpercentile.h" -#define HISTOGRAM_MAX_BINS_NUM 100 +#define HISTOGRAM_MAX_BINS_NUM 100 typedef struct SSumRes { union { @@ -106,6 +107,13 @@ typedef struct SHistoFuncInfo { SHistoFuncBin bins[]; } SHistoFuncInfo; +typedef enum { + UNKNOWN_BIN = 0, + USER_INPUT_BIN, + LINEAR_BIN, + LOG_BIN +} EHistoBinType; + #define SET_VAL(_info, numOfElem, res) \ do { \ @@ -1801,28 +1809,241 @@ bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) return true; } -bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static int8_t getHistogramBinType(char *binTypeStr) { + int8_t binType; + if (strcasecmp(binTypeStr, "user_input") == 0) { + binType = USER_INPUT_BIN; + } else if (strcasecmp(binTypeStr, "linear_bin") == 0) { + binType = LINEAR_BIN; + } else if (strcasecmp(binTypeStr, "log_bin") == 0) { + binType = LOG_BIN; + } else { + binType = UNKNOWN_BIN; + } + + return binType; +} + +static bool getHistogramBinDesc(SHistoFuncInfo *pInfo, char *binDescStr, int8_t binType, bool normalized) { + cJSON* binDesc = cJSON_Parse(binDescStr); + int32_t numOfBins; + double* intervals; + if (cJSON_IsObject(binDesc)) { /* linaer/log bins */ + int32_t numOfParams = cJSON_GetArraySize(binDesc); + int32_t startIndex; + if (numOfParams != 4) { + return false; + } + + cJSON* start = cJSON_GetObjectItem(binDesc, "start"); + cJSON* factor = cJSON_GetObjectItem(binDesc, "factor"); + cJSON* width = cJSON_GetObjectItem(binDesc, "width"); + cJSON* count = cJSON_GetObjectItem(binDesc, "count"); + cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); + + if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { + return false; + } + + if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 + return false; + } + + if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || + (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { + return false; + } + + int32_t counter = (int32_t)count->valueint; + if (infinity->valueint == false) { + startIndex = 0; + numOfBins = counter + 1; + } else { + startIndex = 1; + numOfBins = counter + 3; + } + + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { + // linear bin process + if (width->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble + i * width->valuedouble; + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { + // log bin process + if (start->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else { + taosMemoryFree(intervals); + return false; + } + + if (infinity->valueint == true) { + intervals[0] = -INFINITY; + intervals[numOfBins - 1] = INFINITY; + // in case of desc bin orders, -inf/inf should be swapped + ASSERT(numOfBins >= 4); + if (intervals[1] > intervals[numOfBins - 2]) { + TSWAP(intervals[0], intervals[numOfBins - 1]); + } + } + } else if (cJSON_IsArray(binDesc)) { /* user input bins */ + if (binType != USER_INPUT_BIN) { + return false; + } + numOfBins = cJSON_GetArraySize(binDesc); + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + cJSON* bin = binDesc->child; + if (bin == NULL) { + taosMemoryFree(intervals); + return false; + } + int i = 0; + while (bin) { + intervals[i] = bin->valuedouble; + if (!cJSON_IsNumber(bin)) { + taosMemoryFree(intervals); + return false; + } + if (i != 0 && intervals[i] <= intervals[i - 1]) { + taosMemoryFree(intervals); + return false; + } + bin = bin->next; + i++; + } + } else { + return false; + } + + pInfo->numOfBins = numOfBins - 1; + pInfo->normalized = normalized; + for (int32_t i = 0; i < pInfo->numOfBins; ++i) { + pInfo->bins[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; + pInfo->bins[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; + pInfo->bins[i].count = 0; + } + + taosMemoryFree(intervals); + return true; +} + +bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; } - SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - char* binType = pCtx->param[1].param.pz; - char* binDesc = pCtx->param[2].param.pz; - int64_t nornalized = pCtx->param[3].param.i; + SHistoFuncInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + int8_t binType = getHistogramBinType(varDataVal(pCtx->param[1].param.pz)); + if (binType == UNKNOWN_BIN) { + return false; + } + char* binDesc = varDataVal(pCtx->param[2].param.pz); + int64_t normalized = pCtx->param[3].param.i; + if (normalized != 0 && normalized != 1) { + return false; + } + if (!getHistogramBinDesc(pInfo, binDesc, binType, (bool)normalized)) { + return false; + } return true; } int32_t histogramFunction(SqlFunctionCtx *pCtx) { + SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t type = pInput->pData[0]->info.type; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + int32_t numOfElems = 0; + int32_t totalElems = 0; + for (int32_t i = start; i < numOfRows + start; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElems++; + + char* data = colDataGetData(pCol, i); + double v; + GET_TYPED_DATA(v, double, type, data); + + for (int32_t k = 0; k < pInfo->numOfBins; ++k) { + if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) { + pInfo->bins[k].count++; + totalElems++; + break; + } + } + + } + + if (pInfo->normalized) { + for (int32_t k = 0; k < pInfo->numOfBins; ++k) { + if(totalElems != 0) { + pInfo->bins[k].percentage = pInfo->bins[k].count / (double)totalElems; + } else { + pInfo->bins[k].percentage = 0; + } + } + } + + SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); return TSDB_CODE_SUCCESS; } int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - //if (pInfo->hasResult == true) { - // SET_DOUBLE_VAL(&pInfo->result, pInfo->max - pInfo->min); - //} - return functionFinalize(pCtx, pBlock); + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + int32_t currentRow = pBlock->info.rows; + + for (int32_t i = 0; i < pResInfo->numOfRes; ++i) { + int32_t len; + char buf[512] = {0}; + if (!pInfo->normalized) { + len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}", + pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count); + } else { + len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].percentage); + } + varDataSetLen(buf, len); + colDataAppend(pCol, currentRow, buf, false); + currentRow++; + } + + return pResInfo->numOfRes; } diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 3ea986ad48..d474d87409 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -31,6 +31,7 @@ extern "C" { typedef struct MemTable { T_REF_DECLARE() SSkipList* mem; + void* pCache; } MemTable; typedef struct IndexCache { T_REF_DECLARE() @@ -47,6 +48,7 @@ typedef struct IndexCache { } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) + typedef struct CacheTerm { // key char* colVal; diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 0d8418ba65..3b07429089 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -24,7 +24,7 @@ extern char JSON_COLUMN[]; extern char JSON_VALUE_DELIM; char* indexPackJsonData(SIndexTerm* itm); - +char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 5c7b8b9afe..7b7050d80e 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -166,7 +166,9 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) + +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) + #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 83b5025ad0..e0c24ac3bd 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -244,8 +244,8 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde return 0; } -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType, - const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (tm == NULL) { return NULL; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 13768ce682..b3ae7b7dbe 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -34,44 +34,96 @@ static char* indexCacheTermGet(const void* pData); static MemTable* indexInternalCacheCreate(int8_t type); -static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); /*comm func of compare, used in (LE/LT/GE/GT compare)*/ -static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, +static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s, RangeType type); +static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); + +static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, + RangeType type); typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); -static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; } -static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } -static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } -static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { + int32_t ret = func(a, b); + switch (comType) { + case QUERY_LESS_THAN: { + if (ret < 0) return MATCH; + } break; + case QUERY_LESS_EQUAL: { + if (ret <= 0) return MATCH; + break; + } + case QUERY_GREATER_THAN: { + if (ret > 0) return MATCH; + break; + } + case QUERY_GREATER_EQUAL: { + if (ret >= 0) return MATCH; + } + } + return CONTINUE; +} +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_THAN, a, b); +} +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); +} +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_THAN, a, b); +} +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); +} static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual}; -static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { - cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, - cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}; +static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = { + {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual, + cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}, + {cacheSearchTerm_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON, + cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON, + cacheSearchRange_JSON}}; static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); -static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { if (cache == NULL) { return 0; } + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; - MemTable* mem = cache; - char* key = indexCacheTermGet(ct); + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* key = indexCacheTermGet(pCt); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { @@ -80,7 +132,7 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - if (0 == strcmp(c->colVal, ct->colVal)) { + if (0 == strcmp(c->colVal, pCt->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) // taosArrayPush(result, &c->uid); @@ -92,30 +144,39 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S break; } } + + taosMemoryFree(pCt); tSkipListDestroyIter(iter); return 0; } -static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchSuffix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchRegex(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, +static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, RangeType type) { if (cache == NULL) { return 0; } + _cache_range_compare cmpFn = rangeCompare[type]; - MemTable* mem = cache; - char* key = indexCacheTermGet(ct); + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* key = indexCacheTermGet(pCt); SSkipListIterator* iter = tSkipListCreateIter(mem->mem); while (tSkipListIterNext(iter)) { @@ -124,7 +185,7 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - TExeCond cond = cmpFn(c->colVal, ct->colVal, ct->colType); + TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) @@ -134,26 +195,153 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) } } else if (cond == CONTINUE) { + continue; } else if (cond == BREAK) { break; } } + taosMemoryFree(pCt); tSkipListDestroyIter(iter); return TSDB_CODE_SUCCESS; } -static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, LT); +static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, LT); } -static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, LE); +static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, LE); } -static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, GT); +static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, GT); } -static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, GE); +static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, GE); } -static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + +static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + if (cache == NULL) { + return 0; + } + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* exBuf = NULL; + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + exBuf = indexPackJsonData(term); + pCt->colVal = exBuf; + } + char* key = indexCacheTermGet(pCt); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + if (0 == strcmp(c->colVal, pCt->colVal)) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else { + break; + } + } + + taosMemoryFree(pCt); + taosMemoryFree(exBuf); + tSkipListDestroyIter(iter); + return 0; + + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT); +} +static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, LE); +} +static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, GT); +} +static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE); +} +static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} + +static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, + RangeType type) { + if (cache == NULL) { + return 0; + } + _cache_range_compare cmpFn = rangeCompare[type]; + + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + int8_t dType = INDEX_TYPE_GET_TYPE(term->colType); + int skip = 0; + char* exBuf = NULL; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + exBuf = indexPackJsonDataPrefix(term, &skip); + pCt->colVal = exBuf; + } + char* key = indexCacheTermGet(pCt); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + + TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType); + if (cond == MATCH) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else if (cond == CONTINUE) { + continue; + } else if (cond == BREAK) { + break; + } + } + + taosMemoryFree(pCt); + taosMemoryFree(exBuf); + tSkipListDestroyIter(iter); + + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } @@ -167,6 +355,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in }; cache->mem = indexInternalCacheCreate(type); + cache->mem->pCache = cache; cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; @@ -325,6 +514,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); + cache->mem->pCache = cache; cache->occupiedMem = 0; // sched to merge // unref cache in bgwork @@ -379,11 +569,19 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u return 0; } -static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SIdxTempResult* tr, STermValueType* s) { +static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTempResult* tr, STermValueType* s) { if (mem == NULL) { return 0; } - return cacheSearch[qtype](mem, ct, tr, s); + + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + return cacheSearch[1][qtype](mem, term, tr, s); + } else { + return cacheSearch[0][qtype](mem, term, tr, s); + } } int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); @@ -400,25 +598,24 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result indexMemRef(imm); taosThreadMutexUnlock(&pCache->mtx); - SIndexTerm* term = query->term; - EIndexQueryType qtype = query->qType; + // SIndexTerm* term = query->term; + // EIndexQueryType qtype = query->qType; - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); - char* p = term->colVal; - if (hasJson) { - p = indexPackJsonData(term); - } - CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; + // bool isJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); + // char* p = term->colVal; + // if (isJson) { + // p = indexPackJsonData(term); + //} + // CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; - int ret = indexQueryMem(mem, &ct, qtype, result, s); + int ret = indexQueryMem(mem, query, result, s); if (ret == 0 && *s != kTypeDeletion) { // continue search in imm - ret = indexQueryMem(imm, &ct, qtype, result, s); - } - - if (hasJson) { - taosMemoryFreeClear(p); + ret = indexQueryMem(imm, query, result, s); } + // if (isJson) { + // taosMemoryFreeClear(p); + //} indexMemUnRef(mem); indexMemUnRef(imm); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 4dea5fa011..cdd7b35675 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -46,3 +46,30 @@ char* indexPackJsonData(SIndexTerm* itm) { return buf; } +char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { + /* + * |<-----colname---->|<-----dataType---->|<--------colVal---------->| + * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| + */ + uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + + int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; + char* buf = (char*)taosMemoryCalloc(1, sz); + char* p = buf; + + memcpy(p, itm->colName, itm->nColName); + p += itm->nColName; + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, &ty, sizeof(ty)); + p += sizeof(ty); + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + *skip = p - buf; + + return buf; +} diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 5aed2bd6b0..b5551e825f 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -308,20 +308,20 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) } static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); - int ret = 0; - char* p = tem->colVal; - uint64_t sz = tem->nColVal; + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + int skip = 0; + if (hasJson) { - p = indexPackJsonData(tem); - sz = strlen(p); + p = indexPackJsonDataPrefix(tem, &skip); } SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); - FstSlice h = fstSliceCreate((uint8_t*)p, sz); + FstSlice h = fstSliceCreate((uint8_t*)p, skip); fstStreamBuilderSetRange(sb, &h, type); fstSliceDestroy(&h); @@ -606,16 +606,16 @@ static bool tfileIteratorNext(Iterate* iiter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter)); - if (tIter == NULL) { + TFileFstIter* iter = taosMemoryCalloc(1, sizeof(TFileFstIter)); + if (iter == NULL) { return NULL; } - tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); - tIter->fb = fstSearch(reader->fst, tIter->ctx); - tIter->st = streamBuilderIntoStream(tIter->fb); - tIter->rdr = reader; - return tIter; + iter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); + iter->fb = fstSearch(reader->fst, iter->ctx); + iter->st = streamBuilderIntoStream(iter->fb); + iter->rdr = reader; + return iter; } Iterate* tfileIteratorCreate(TFileReader* reader) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index d8ea6a8233..896451c686 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -483,7 +483,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colName("voltage"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; @@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); @@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -623,7 +623,7 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); @@ -638,7 +638,7 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); @@ -670,7 +670,7 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -679,7 +679,7 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -701,7 +701,7 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -737,7 +737,7 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -759,7 +759,7 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -784,7 +784,7 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); @@ -792,7 +792,7 @@ class IndexObj { } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); @@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e1e5004701..f789d23136 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); @@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 0bceb84f4a..319b680161 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1337,7 +1337,9 @@ void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - pVal->pz = pNode->datum.p; + pVal->pz = taosMemoryMalloc(pVal->nLen + VARSTR_HEADER_SIZE + 1); + memcpy(pVal->pz, pNode->datum.p, pVal->nLen + VARSTR_HEADER_SIZE); + pVal->pz[pVal->nLen + VARSTR_HEADER_SIZE] = 0; break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index 02be32a49e..0627ed1066 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -133,15 +133,15 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks * int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void destroyBlockArrayList(SArray *pDataBlockList); -void destroyBlockHashmap(SHashObj *pDataBlockHash); -int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); -int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows); -int32_t getDataBlockFromList(SHashObj *pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta *pTableMeta, STableDataBlocks **dataBlocks, SArray *pBlockList, - SVCreateTbReq *pCreateTbReq); -int32_t mergeTableDataBlocks(SHashObj *pHashObj, uint8_t payloadType, SArray **pVgDataBlocks); -int32_t buildCreateTbMsg(STableDataBlocks *pBlocks, SVCreateTbReq *pCreateTbReq); +void destroyBlockArrayList(SArray* pDataBlockList); +void destroyBlockHashmap(SHashObj* pDataBlockHash); +int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); +int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); +int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, + STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); +int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq); + int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 62d197d611..c146d42e05 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -41,7 +41,6 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); -STableMeta* tableMetaDup(const STableMeta* pTableMeta); SSchema* getTableColumnSchema(const STableMeta* pTableMeta); SSchema* getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index e82873b923..05d247c037 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -279,6 +279,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { int32_t numOfBlocks = blocks->numOfTables; while (numOfBlocks--) { int32_t dataLen = blk->dataLen; + int32_t schemaLen = blk->schemaLen; blk->uid = htobe64(blk->uid); blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); @@ -286,7 +287,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { blk->dataLen = htonl(blk->dataLen); blk->schemaLen = htonl(blk->schemaLen); blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); + blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); } } @@ -761,11 +762,9 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const SName* pName, SKVRow row, int64_t suid) { - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); +static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const char* tname, SKVRow row, int64_t suid) { pTbReq->type = TD_CHILD_TABLE; - pTbReq->name = strdup(pName->tname); + pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; pTbReq->ctb.pTag = row; @@ -773,7 +772,7 @@ static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const SName* pName, SKVRo } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const SName* pName) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -814,7 +813,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } tdSortKVRowByColIdx(row); - return buildCreateTbReq(&pCxt->createTblReq, pName, row, pCxt->pTableMeta->suid); + return buildCreateTbReq(&pCxt->createTblReq, tName, row, pCxt->pTableMeta->suid); } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { @@ -883,7 +882,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_NK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, &name)); + CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, name.tname)); NEXT_TOKEN(pCxt->pSql, sToken); if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); @@ -1037,11 +1036,6 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { - // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } taosMemoryFreeClear(pDataBlock); @@ -1269,10 +1263,9 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, - char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ + STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -1311,7 +1304,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* p tdSortKVRowByColIdx(row); SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); + CHECK_CODE(buildCreateTbReq(&tbReq, tName, row, suid)); CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); destroyCreateSubTbReq(&tbReq); @@ -1543,3 +1536,265 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields return TSDB_CODE_SUCCESS; } + +// schemaless logic start + +typedef struct SmlExecHandle { + SHashObj* pBlockHash; + + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SVCreateTbReq createTblReq; // each table + + SQuery* pQuery; +} SSmlExecHandle; + +static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) { + col_id_t nCols = pColList->numOfCols; + + pColList->numOfBound = 0; + pColList->boundNullLen = 0; + memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); + for (col_id_t i = 0; i < nCols; ++i) { + pColList->cols[i].valStat = VAL_STAT_NONE; + } + + bool isOrdered = true; + col_id_t lastColIdx = -1; // last column found + for (int i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = taosArrayGetP(cols, i); + SToken sToken = {.n=kv->keyLen, .z=(char*)kv->key}; + col_id_t t = lastColIdx + 1; + col_id_t index = findCol(&sToken, t, nCols, pSchema); + if (index < 0 && t > 0) { + index = findCol(&sToken, 0, t, pSchema); + isOrdered = false; + } + if (index < 0) { + return TSDB_CODE_SML_INVALID_DATA; + } + if (pColList->cols[index].valStat == VAL_STAT_HAS) { + return TSDB_CODE_SML_INVALID_DATA; + } + lastColIdx = index; + pColList->cols[index].valStat = VAL_STAT_HAS; + pColList->boundColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; + ++pColList->numOfBound; + switch (pSchema[t].type) { + case TSDB_DATA_TYPE_BINARY: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); + break; + case TSDB_DATA_TYPE_NCHAR: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + break; + default: + pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; + break; + } + } + + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (!isOrdered) { + pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + if (NULL == pColList->colIdxInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = pColList->colIdxInfo; + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = pColList->boundColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + if(pColList->numOfCols > pColList->numOfBound){ + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlBoundTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SKVRow *row, SMsgBuf *msg) { + if (tdInitKVRowBuilder(tagsBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SKvParam param = {.builder = tagsBuilder}; + for (int i = 0; i < tags->numOfBound; ++i) { + SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 + param.schema = pTagSchema; + SSmlKv *kv = taosArrayGetP(cols, i); + KvRowAppend(msg, kv->value, kv->valueLen, ¶m) ; + } + + + *row = tdGetKVRowFromBuilder(tagsBuilder); + if(*row == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + tdSortKVRowByColIdx(*row); + return TSDB_CODE_SUCCESS; +} + +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, + STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen) { + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); + int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "bound tags error"); + return ret; + } + SKVRow row = NULL; + ret = smlBoundTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); + + STableDataBlocks* pDataBlock = NULL; + ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, + &pDataBlock, NULL, &smlHandle->createTblReq); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "create data block error"); + return ret; + } + + SSchema* pSchema = getTableColumnSchema(pTableMeta); + + + if(format){ + ret = smlBoundColumns(taosArrayGetP(colsFormat, 0), &pDataBlock->boundColumnInfo, pSchema); + }else{ + SArray *columns = taosArrayInit(16, POINTER_BYTES); + void **p1 = taosHashIterate(colsHash, NULL); + while (p1) { + SSmlKv* kv = *p1; + taosArrayPush(columns, &kv); + p1 = taosHashIterate(colsHash, p1); + } + ret = smlBoundColumns(columns, &pDataBlock->boundColumnInfo, pSchema); + taosArrayDestroy(columns); + } + + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "bound cols error"); + return ret; + } + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + + initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + + int32_t rowNum = format ? taosArrayGetSize(colsFormat) : taosArrayGetSize(cols); + if(rowNum <= 0) { + return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + } + ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "allocate memory error"); + return ret; + } + for (int32_t r = 0; r < rowNum; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header + tdSRowResetBuf(pBuilder, row); + void *rowData = NULL; + if(format){ + rowData = taosArrayGetP(colsFormat, r); + }else{ + rowData = taosArrayGetP(cols, r); + } + + // 1. set the parsed value from sql string + for (int c = 0; c < spd->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); + + SSmlKv *kv = NULL; + if(format){ + kv = taosArrayGetP(rowData, c); + if (!kv){ + char msg[64] = {0}; + sprintf(msg, "cols num not the same like before:%d", r); + return buildInvalidOperationMsg(&pBuf, msg); + } + }else{ + void **p =taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + kv = *p; + } + + if (kv->length == 0) { + MemRowAppend(&pBuf, NULL, 0, ¶m); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = kv->length; + } + + MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + } + + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + + pDataBlock->size += extendedRowSize; + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, rowNum)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + + return TSDB_CODE_SUCCESS; +} + +void* smlInitHandle(SQuery *pQuery){ + SSmlExecHandle *handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); + if(!handle) return NULL; + handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + handle->pQuery = pQuery; + + return handle; +} + +void smlDestroyHandle(void *pHandle){ + if(!pHandle) return; + SSmlExecHandle *handle = (SSmlExecHandle *)pHandle; + destroyBlockHashmap(handle->pBlockHash); + taosMemoryFree(handle); +} + +int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { + SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; + return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); +} +// schemaless logic end + diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 4a96301d95..ac197077a4 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -115,7 +115,7 @@ void destroyBoundColumnInfo(void* pBoundInfo) { taosMemoryFreeClear(pColList->colIdxInfo); } -static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, const STableMeta* pTableMeta, +static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { @@ -137,8 +137,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - // Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tableMetaDup(pTableMeta); + dataBuf->pTableMeta = pTableMeta; SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); @@ -187,7 +186,7 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) } int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, + STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); @@ -238,9 +237,9 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } +// if (pDataBlock->pTableMeta != NULL) { +// taosMemoryFreeClear(pDataBlock->pTableMeta); +// } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } @@ -456,6 +455,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; + pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6874b5b7d4..ea5d373a73 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -481,6 +481,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } + *(int64_t*)&pVal->typeData = pVal->datum.i; } else { switch (pVal->node.resType.type) { case TSDB_DATA_TYPE_NULL: @@ -2596,7 +2597,7 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS pStmt->ignoreNotExists); } -static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) { +static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); if (NULL == pAlterReq->pFields) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2632,8 +2633,8 @@ static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterRe } static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { - SMAltertbReq alterReq = {0}; - SName tableName; + SMAlterStbReq alterReq = {0}; + SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), alterReq.name); alterReq.alterType = pStmt->alterType; alterReq.numOfFields = 1; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e37dbd1edd..715b9b97e6 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -191,27 +191,7 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } -static uint32_t getTableMetaSize(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - int32_t totalCols = 0; - if (pTableMeta->tableInfo.numOfColumns >= 0) { - totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; - } - - return sizeof(STableMeta) + totalCols * sizeof(SSchema); -} - -STableMeta* tableMetaDup(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - size_t size = getTableMetaSize(pTableMeta); - - STableMeta* p = taosMemoryMalloc(size); - memcpy(p, pTableMeta, size); - return p; -} - -SSchema* getTableColumnSchema(const STableMeta* pTableMeta) { +SSchema *getTableColumnSchema(const STableMeta *pTableMeta) { assert(pTableMeta != NULL); return (SSchema*)pTableMeta->schema; } diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index ca471262ff..3ecf987811 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -25,6 +25,7 @@ extern "C" { #include "tlockfree.h" #include "ttimer.h" #include "tref.h" +#include "plannodes.h" #define QW_DEFAULT_SCHEDULER_NUMBER 10000 #define QW_DEFAULT_TASK_NUMBER 10000 @@ -131,8 +132,9 @@ typedef struct SQWTaskCtx { int8_t events[QW_EVENT_MAX]; - void *taskHandle; - void *sinkHandle; + void *taskHandle; + void *sinkHandle; + SSubplan *plan; } SQWTaskCtx; typedef struct SQWSchStatus { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index bd7ee6321a..717958c033 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -424,6 +424,11 @@ void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { dsDestroyDataSinker(ctx->sinkHandle); ctx->sinkHandle = NULL; } + + if (ctx->plan) { + nodesDestroyNode(ctx->plan); + ctx->plan = NULL; + } } int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { @@ -440,6 +445,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { atomic_store_ptr(&ctx->taskHandle, NULL); atomic_store_ptr(&ctx->sinkHandle, NULL); + atomic_store_ptr(&ctx->plan, NULL); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_DROP); @@ -922,7 +928,7 @@ _return: int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { int32_t code = 0; bool queryRsped = false; - struct SSubplan *plan = NULL; + SSubplan* plan = NULL; SQWPhaseInput input = {0}; qTaskInfo_t pTaskInfo = NULL; DataSinkHandle sinkHandle = NULL; @@ -950,6 +956,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex QW_ERR_JRET(code); } + ctx->plan = plan; + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -1428,6 +1436,10 @@ void qwCloseRef(void) { taosWUnLockLatch(&gQwMgmt.lock); } +void qwDestroySchStatus(SQWSchStatus *pStatus) { + taosHashCleanup(pStatus->tasksHash); +} + void qwDestroyImpl(void *pMgmt) { SQWorker *mgmt = (SQWorker *)pMgmt; @@ -1439,6 +1451,13 @@ void qwDestroyImpl(void *pMgmt) { // TODO FREE ALL taosHashCleanup(mgmt->ctxHash); + + void *pIter = taosHashIterate(mgmt->schHash, NULL); + while (pIter) { + SQWSchStatus *sch = (SQWSchStatus *)pIter; + qwDestroySchStatus(sch); + pIter = taosHashIterate(mgmt->schHash, pIter); + } taosHashCleanup(mgmt->schHash); taosMemoryFree(mgmt); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index ed52e48f6d..49a5f5b9a4 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -166,7 +166,6 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.pData = (char *)taosMemoryCalloc(1, size); colInfoDataEnsureCapacity(&idata, 0, rowNum); taosArrayPush(res->pDataBlock, &idata); - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index ae153251c3..031722ab3c 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,7 +57,13 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - assert(code == 0); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(code == 0); walFsync(pWal, true); return code; @@ -69,7 +75,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - assert(walReadWithHandle(pWalHandle, index) == 0); + int32_t code = walReadWithHandle(pWalHandle, index); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walReadWithHandle error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); assert(pEntry != NULL); diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 18f8ddec5c..90b07fb6ae 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -49,6 +49,7 @@ int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, vo // TDBC int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); int tdbDbcClose(TDBC *pDbc); +int tdbDbcIsValid(TDBC *pDbc); int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); int tdbDbcMoveToFirst(TDBC *pDbc); int tdbDbcMoveToLast(TDBC *pDbc); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0d13ec4d2b..43822cd311 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1678,6 +1678,14 @@ int tdbBtcClose(SBTC *pBtc) { return 0; } + +int tdbBtcIsValid(SBTC *pBtc) { + if (pBtc->idx < 0) { + return 0; + } else { + return 1; + } +} // TDB_BTREE_CURSOR // TDB_BTREE_DEBUG ===================== diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 553bb2c646..ceaac6dff1 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -141,3 +141,5 @@ int tdbDbcClose(TDBC *pDbc) { return 0; } + +int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 76dacf7b84..ee431ac638 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -136,6 +136,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL // SBTC int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); int tdbBtcClose(SBTC *pBtc); +int tdbBtcIsValid(SBTC *pBtc); int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); int tdbBtcMoveToFirst(SBTC *pBtc); int tdbBtcMoveToLast(SBTC *pBtc); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index ad3f520210..6047643881 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -784,8 +784,17 @@ static void uvDestroyConn(uv_handle_t* handle) { tDebug("server conn %p destroy", conn); // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); + + if (conn->regArg.init == 1) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); + if (conn->regArg.init == 1) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 70a3559cd9..4fe07029f1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -49,7 +49,10 @@ void walCloseReadHandle(SWalReadHandle *pRead) { taosMemoryFree(pRead); } -int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { + // TODO + return 0; +} static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index dc31086e9f..da1c36dcc4 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -198,12 +198,14 @@ int walRoll(SWal *pWal) { if (pWal->pWriteIdxTFile != NULL) { code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } if (pWal->pWriteLogTFile != NULL) { code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } @@ -263,14 +265,19 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog if (index == pWal->vers.lastVer + 1) { if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->vers.firstVer = index; - code = walRoll(pWal); - ASSERT(code == 0); + if (walRoll(pWal) < 0) { + return -1; + } } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { - walRoll(pWal); + if (walRoll(pWal) < 0) { + return -1; + } } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { - walRoll(pWal); + if (walRoll(pWal) < 0) { + return -1; + } } } } else { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 1470496c68..4b3c99d321 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -326,6 +326,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, "Invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, "Invalid table schema version") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST, "Table not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, "No diskspace for tsdb") @@ -449,6 +452,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TABLE_NOT_EXIST, "Table does not exist" //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error") +//schemaless +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") + #ifdef TAOS_ERROR_C }; #endif diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 828216d1d3..99e546fcd0 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -398,7 +398,7 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32 } } -void generateQuerySQL(BindData *data, int32_t tblIdx) { +void generateQueryCondSQL(BindData *data, int32_t tblIdx) { int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx); if (!gCurCase->fullCol) { for (int c = 0; c < gCurCase->bindColNum; ++c) { @@ -462,6 +462,72 @@ void generateQuerySQL(BindData *data, int32_t tblIdx) { } } + +void generateQueryMiscSQL(BindData *data, int32_t tblIdx) { + int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx); + if (!gCurCase->fullCol) { + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, " and "); + } + switch (data->pBind[c].buffer_type) { + case TSDB_DATA_TYPE_BOOL: + len += sprintf(data->sql + len, "booldata"); + break; + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(data->sql + len, "tinydata"); + break; + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(data->sql + len, "smalldata"); + break; + case TSDB_DATA_TYPE_INT: + len += sprintf(data->sql + len, "intdata"); + break; + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(data->sql + len, "bigdata"); + break; + case TSDB_DATA_TYPE_FLOAT: + len += sprintf(data->sql + len, "floatdata"); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += sprintf(data->sql + len, "doubledata"); + break; + case TSDB_DATA_TYPE_VARCHAR: + len += sprintf(data->sql + len, "binarydata"); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(data->sql + len, "ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + len += sprintf(data->sql + len, "nchardata"); + break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(data->sql + len, "utinydata"); + break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(data->sql + len, "usmalldata"); + break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(data->sql + len, "uintdata"); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(data->sql + len, "ubigdata"); + break; + default: + printf("!!!invalid col type:%d", data->pBind[c].buffer_type); + exit(1); + } + + bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); + } + } + + if (gCaseCtrl.printStmtSql) { + printf("\tSTMT SQL: %s\n", data->sql); + } +} + + void generateErrorSQL(BindData *data, int32_t tblIdx) { int32_t len = 0; data->sql = taosMemoryCalloc(1, 1024); @@ -677,7 +743,7 @@ int32_t prepareInsertData(BindData *data) { return 0; } -int32_t prepareQueryData(BindData *data, int32_t tblIdx) { +int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { static int64_t tsData = 1591060628000; uint64_t bindNum = gCurCase->rowNum / gCurCase->bindRowNum; @@ -735,6 +801,63 @@ int32_t prepareQueryData(BindData *data, int32_t tblIdx) { } +int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { + static int64_t tsData = 1591060628000; + uint64_t bindNum = gCurCase->rowNum / gCurCase->bindRowNum; + + data->colNum = 0; + data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); + data->sql = taosMemoryCalloc(1, 1024); + data->pBind = taosMemoryCalloc(bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); + data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t)); + data->boolData = taosMemoryMalloc(bindNum * sizeof(bool)); + data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t)); + data->utinyData = taosMemoryMalloc(bindNum * sizeof(uint8_t)); + data->smallData = taosMemoryMalloc(bindNum * sizeof(int16_t)); + data->usmallData = taosMemoryMalloc(bindNum * sizeof(uint16_t)); + data->intData = taosMemoryMalloc(bindNum * sizeof(int32_t)); + data->uintData = taosMemoryMalloc(bindNum * sizeof(uint32_t)); + data->bigData = taosMemoryMalloc(bindNum * sizeof(int64_t)); + data->ubigData = taosMemoryMalloc(bindNum * sizeof(uint64_t)); + data->floatData = taosMemoryMalloc(bindNum * sizeof(float)); + data->doubleData = taosMemoryMalloc(bindNum * sizeof(double)); + data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize); + data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t)); + if (gCurCase->bindNullNum) { + data->isNull = taosMemoryCalloc(bindNum, sizeof(char)); + } + + for (int32_t i = 0; i < bindNum; ++i) { + data->tsData[i] = tsData + tblIdx*gCurCase->rowNum + rand()%gCurCase->rowNum; + data->boolData[i] = (bool)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->tinyData[i] = (int8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->utinyData[i] = (uint8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->smallData[i] = (int16_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->usmallData[i] = (uint16_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->intData[i] = (int32_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->uintData[i] = (uint32_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->bigData[i] = (int64_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->ubigData[i] = (uint64_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->floatData[i] = (float)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); + if (gCurCase->bindNullNum) { + data->isNull[i] = i % 2; + } + data->binaryLen[i] = gVarCharLen; + } + + for (int b = 0; b < bindNum; b++) { + for (int c = 0; c < gCurCase->bindColNum; ++c) { + prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + } + } + + generateQueryMiscSQL(data, tblIdx); + + return 0; +} + void destroyData(BindData *data) { @@ -1385,7 +1508,7 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { for (int32_t t = 0; t< gCurCase->tblNum; ++t) { memset(&data, 0, sizeof(data)); - prepareQueryData(&data, t); + prepareQueryCondData(&data, t); int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ @@ -1431,7 +1554,7 @@ int querySUBTTest2(TAOS_STMT *stmt, TAOS *taos) { for (int32_t t = 0; t< gCurCase->tblNum; ++t) { memset(&data, 0, sizeof(data)); - prepareQueryData(&data, t); + prepareQueryMiscData(&data, t); int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ diff --git a/tests/script/general/stable/testSuite.sim b/tests/script/general/stable/testSuite.sim deleted file mode 100644 index e786ac9ca4..0000000000 --- a/tests/script/general/stable/testSuite.sim +++ /dev/null @@ -1,5 +0,0 @@ -run general/stable/disk.sim -run general/stable/dnode3.sim -run general/stable/metrics.sim -run general/stable/values.sim -run general/stable/vnode3.sim diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim index 80d2dafa84..2946a89ff6 100644 --- a/tests/script/tsim/query/scalarFunction.sim +++ b/tests/script/tsim/query/scalarFunction.sim @@ -102,20 +102,26 @@ print ====> $data60 $data61 $data62 $data63 $data64 $data65 print ====> $data70 $data71 $data72 $data73 $data74 $data75 print ====> $data80 $data81 $data82 $data83 $data84 $data85 print ====> $data90 $data91 $data92 $data93 $data94 $data95 +print ====> rows = $rows and rowNum = $rowNum for ct1 if $rows != $rowNum then return -1 endi + print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb +print ====> rows = $rows and totalRows = $totalRows for stb if $rows != $totalRows then return -1 endi + print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb +print ====> rows = $rows and rowNum = $rowNum for ntb if $rows != $rowNum then return -1 endi + print ====> log sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py new file mode 100644 index 0000000000..3ab39f9e7b --- /dev/null +++ b/tests/system-test/0-others/telemetry.py @@ -0,0 +1,197 @@ +import taos +import sys +import time +import socket +import pexpect +import os +import http.server +import gzip +import threading +import json + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +telemetryPort = '80' + +#{ +# "instanceId": "5cf4cd7a-acd4-43ba-8b0d-e84395b76a65", +# "reportVersion": 1, +# "os": "Ubuntu 20.04.3 LTS", +# "cpuModel": "Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz", +# "numOfCpu": 6, +# "memory": "65860292 kB", +# "version": "3.0.0.0", +# "buildInfo": "Built at 2022-05-07 14:09:02", +# "gitInfo": "2139ccceb0946cde86b6b553b11e338f1ba437e5", +# "email": "user@taosdata.com", +# "numOfDnode": 1, +# "numOfMnode": 1, +# "numOfVgroup": 32, +# "numOfDatabase": 2, +# "numOfSuperTable": 0, +# "numOfChildTable": 100, +# "numOfColumn": 200, +# "numOfPoint": 300, +# "totalStorage": 400, +# "compStorage": 500 +#} + +def telemetryInfoCheck(infoDict=''): + if "instanceId" not in infoDict or len(infoDict["instanceId"]) == 0: + tdLog.exit("instanceId is null!") + + if "reportVersion" not in infoDict or infoDict["reportVersion"] != 1: + tdLog.exit("reportVersion is null!") + + if "os" not in infoDict: + tdLog.exit("os is null!") + + if "cpuModel" not in infoDict: + tdLog.exit("cpuModel is null!") + + if "numOfCpu" not in infoDict or infoDict["numOfCpu"] == 0: + tdLog.exit("numOfCpu is null!") + + if "memory" not in infoDict: + tdLog.exit("memory is null!") + + if "version" not in infoDict: + tdLog.exit("version is null!") + + if "buildInfo" not in infoDict: + tdLog.exit("buildInfo is null!") + + if "gitInfo" not in infoDict: + tdLog.exit("gitInfo is null!") + + if "email" not in infoDict: + tdLog.exit("email is not exists!") + + if "numOfDnode" not in infoDict or infoDict["numOfDnode"] < 1: + tdLog.exit("numOfDnode is null!") + + if "numOfMnode" not in infoDict or infoDict["numOfMnode"] < 1: + tdLog.exit("numOfMnode is null!") + + if "numOfVgroup" not in infoDict or infoDict["numOfVgroup"] <= 0: + tdLog.exit("numOfVgroup is null!") + + if "numOfDatabase" not in infoDict or infoDict["numOfDatabase"] <= 0: + tdLog.exit("numOfDatabase is null!") + + if "numOfSuperTable" not in infoDict or infoDict["numOfSuperTable"] < 0: + tdLog.exit("numOfSuperTable is null!") + + if "numOfChildTable" not in infoDict or infoDict["numOfChildTable"] < 0: + tdLog.exit("numOfChildTable is null!") + + if "numOfColumn" not in infoDict or infoDict["numOfColumn"] < 0: + tdLog.exit("numOfColumn is null!") + + if "numOfPoint" not in infoDict or infoDict["numOfPoint"] < 0: + tdLog.exit("numOfPoint is null!") + + if "totalStorage" not in infoDict or infoDict["totalStorage"] < 0: + tdLog.exit("totalStorage is null!") + + if "compStorage" not in infoDict or infoDict["compStorage"] < 0: + tdLog.exit("compStorage is null!") + + +class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): + def do_GET(self): + """ + process GET request + """ + + def do_POST(self): + """ + process POST request + """ + contentEncoding = self.headers["Content-Encoding"] + + if contentEncoding == 'gzip': + req_body = self.rfile.read(int(self.headers["Content-Length"])) + plainText = gzip.decompress(req_body).decode() + else: + plainText = self.rfile.read(int(self.headers["Content-Length"])).decode() + + print("monitor info:\n%s"%plainText) + + # 1. send response code and header + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + + # 2. send response content + #self.wfile.write(("Hello World: " + req_body + "\n").encode("utf-8")) + + # 3. check request body info + infoDict = json.loads(plainText) + #print("================") + #print(infoDict) + telemetryInfoCheck(infoDict) + + # 4. shutdown the server and exit case + assassin = threading.Thread(target=httpServer.shutdown) + assassin.daemon = True + assassin.start() + print ("==== shutdown http server ====") + +class TDTestCase: + hostname = socket.gethostname() + serverPort = '7080' + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + updatecfgDict["telemetryReporting"] = '1' + updatecfgDict["telemetryServer"] = hostname + updatecfgDict["telemetryPort"] = telemetryPort + updatecfgDict["telemetryInterval"] = "3" + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + vgroups = "30" + sql = "create database db3 vgroups " + vgroups + tdSql.query(sql) + + # loop to wait request + httpServer.serve_forever() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +# create http server: bing ip/port , and request processor +serverAddress = ("", int(telemetryPort)) +httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl) + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) + + + + + diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 99fca2e88d..9d073eb4f7 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -144,6 +144,12 @@ class TDTestCase: tdSql.error("select now()+1.5 from ntb") tdSql.error("select now()+1.5 from db.ntb") + tdSql.error("select now()-1.5 from ntb") + tdSql.error("select now()-1.5 from db.ntb") + tdSql.error("select now()*1.5 from ntb") + tdSql.error("select now()*1.5 from db.ntb") + tdSql.error("select now()/1.5 from ntb") + tdSql.error("select now()/1.5 from db.ntb") tdSql.error("select now()+'abc' from ntb") tdSql.error("select now()+'abc' from db.ntb") tdSql.error("select now()+abc from ntb") @@ -262,7 +268,10 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select now() +1 from db.stb") tdSql.checkRows(3) - + tdSql.error("select now() +1.5 from stb") + tdSql.error("select now() -1.5 from stb") + tdSql.error("select now() *1.5 from stb") + tdSql.error("select now() /1.5 from stb") tdSql.error("select now() +'abc' from stb") tdSql.error("select now() +'abc' from db.stb") tdSql.error("select now() + ! from stb") @@ -272,6 +281,7 @@ class TDTestCase: tdSql.error("select now() -today() from stb") tdSql.error("select now() - today() from db.stb") + tdSql.query("select now()+null from stb") tdSql.checkData(0,0,None) tdSql.query("select now()+null from db.stb") diff --git a/tests/system-test/2-query/Timediff.py b/tests/system-test/2-query/Timediff.py new file mode 100644 index 0000000000..2824fea5a2 --- /dev/null +++ b/tests/system-test/2-query/Timediff.py @@ -0,0 +1,196 @@ +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + tdSql.prepare() + tdLog.printNoPrefix("==========step1:create tables==========") + tdSql.execute( + '''create table if not exists ntb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) + ''' + ) + tdSql.execute( + '''create table if not exists stb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) + ''' + ) + tdSql.execute( + '''create table if not exists stb_1 using stb tags(100) + ''' + ) + tdLog.printNoPrefix("==========step2:insert data into ntb==========") + + # RFC3339:2020-01-01T00:00:00+8:00 + # ISO8601:2020-01-01T00:00:00.000+0800 + tdSql.execute( + 'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.execute( + 'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from ntb") + tdSql.checkRows(3) + tdSql.query("select timediff(1,0,1d) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1d) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1s) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff(1,0,1s) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff(1,0,1w) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1w) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1h) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1h) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1m) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1m) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1a) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000) + tdSql.query("select timediff(1,0,1a) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000) + tdSql.query("select timediff(1,0,1u) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000000) + tdSql.query("select timediff(1,0,1u) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000000) + + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from stb") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from db.stb") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1d) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1d) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1h) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,24) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1h) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,24) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1w) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1m) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1440) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1m) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1440) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1s) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1s) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1a) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1a) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000000) + + + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00') from stb_1") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00') from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1w) from stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1w) from db.stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1d) from stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1d) from db.stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1h) from stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,12) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1h) from db.stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,12) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1m) from stb_1" ) + tdSql.checkRows(3) + tdSql.checkData(0,0,720) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1m) from db.stb_1" ) + tdSql.checkRows(3) + tdSql.checkData(0,0,720) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1s) from stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1s) from db.stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1a) from stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1a) from db.stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000000) + + tdSql.query("select timediff('a','b') from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.checkData(1,0,None) + tdSql.checkData(2,0,None) + tdSql.error("select timediff(1.5,1.5) from stb") + tdSql.error("select timediff(1) from stb") + tdSql.error("select timediff(10,1,1.5) from stb") + # tdSql.error("select timediff(10,1,2s) from stb") + # tdSql.error("select timedifff(10,1,c1) from stb") + + + + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/To_iso8061.py b/tests/system-test/2-query/To_iso8601.py similarity index 88% rename from tests/system-test/2-query/To_iso8061.py rename to tests/system-test/2-query/To_iso8601.py index 86322ed076..cd22ffb90c 100644 --- a/tests/system-test/2-query/To_iso8061.py +++ b/tests/system-test/2-query/To_iso8601.py @@ -92,6 +92,21 @@ class TDTestCase: tdSql.query("select to_iso8601(today()) /null from db.ntb") tdSql.checkRows(3) tdSql.checkData(0,0,None) + # tdSql.query("select to_iso8601(-1) from ntb") + tdSql.query("select to_iso8601(9223372036854775807) from ntb") + tdSql.checkRows(3) + + # tdSql.query("select to_iso8601(10000000000) from ntb") + # tdSql.checkData(0,0,None) + # tdSql.query("select to_iso8601(-1) from ntb") + # tdSql.checkRows(3) + # tdSql.query("select to_iso8601(-10000000000) from ntb") + # tdSql.checkData(0,0,None) + tdSql.error("select to_iso8601(1.5) from ntb") + tdSql.error("select to_iso8601(1.5) from db.ntb") + tdSql.error("select to_iso8601('a') from ntb") + tdSql.error("select to_iso8601(c2) from ntb") + @@ -137,6 +152,8 @@ class TDTestCase: tdSql.checkRows(3) tdSql.checkData(0,0,None) + # tdSql.query("select to_iso8601(-1) from ntb") + def stop(self): diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py new file mode 100644 index 0000000000..43315ff0d8 --- /dev/null +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -0,0 +1,73 @@ +from time import sleep + +from util.log import * +from util.sql import * +from util.cases import * + + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + tdSql.prepare() + tdLog.printNoPrefix("==========step1:create tables==========") + tdSql.execute( + '''create table if not exists ntb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) + ''' + ) + tdSql.execute( + '''create table if not exists stb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) + ''' + ) + tdSql.execute( + '''create table if not exists stb_1 using stb tags(100) + ''' + ) + tdLog.printNoPrefix("==========step2:insert data into ntb==========") + + # RFC3339:2020-01-01T00:00:00+8:00 + # ISO8601:2020-01-01T00:00:00.000+0800 + tdSql.execute( + 'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.execute( + 'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.query("select to_unixtimestamp('1970-01-01T08:00:00+0800') from ntb") + tdSql.checkData(0,0,0) + tdSql.checkData(1,0,0) + tdSql.checkData(2,0,0) + tdSql.checkRows(3) + tdSql.query("select to_unixtimestamp('1970-01-01T08:00:00+08:00') from ntb") + tdSql.checkData(0,0,0) + tdSql.checkRows(3) + tdSql.query("select to_unixtimestamp('1900-01-01T08:00:00+08:00') from ntb") + tdSql.checkRows(3) + tdSql.query("select to_unixtimestamp('2020-01-32T08:00:00') from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.query("select to_unixtimestamp('2020-13-32T08:00:00') from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.query("select to_unixtimestamp('acd') from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.error("select to_unixtimestamp(1) from ntb") + tdSql.error("select to_unixtimestamp(1.5) from ntb") + tdSql.error("select to_unixtimestamp(ts) from ntb") + + tdSql.query("select ts from ntb where to_unixtimestamp('1970-01-01T08:00:00+08:00')=0") + tdSql.checkRows(3) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 09f018dc11..9eb06de9fb 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -134,6 +134,35 @@ class TDTestCase: tdSql.checkRows(3) tdSql.checkData(0, 0, str(today_date)) + tdSql.error("select today()+1.5 from ntb") + tdSql.error("select today()-1.5 from ntb") + tdSql.error("select today()*1.5 from ntb") + tdSql.error("select today()/1.5 from ntb") + tdSql.error("select today()+1.5 from db.ntb") + tdSql.error("select today()-1.5 from db.ntb") + tdSql.error("select today()*1.5 from db.ntb") + tdSql.error("select today()/1.5 from db.ntb") + tdSql.query("select today()+null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()+null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/0 from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.checkData(1,0,None) + tdSql.checkData(2,0,None) # stable tdSql.query("select today() from stb") tdSql.checkRows(3) @@ -208,6 +237,26 @@ class TDTestCase: tdSql.query("select ts from stb where ts<=today()") tdSql.checkRows(2) + tdSql.error("select today()+1.5 from stb") + tdSql.error("select today()-1.5 from stb") + tdSql.error("select today()*1.5 from stb") + tdSql.error("select today()/1.5 from stb") + tdSql.query("select today()+null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()+null from db.stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from db.stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from db.stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from db.stb") + tdSql.checkData(0,0,None) # # tdSql.query("select * from ntb where ts=-1 and elem <=1: + elem = math.acos(elem) + else: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("acos function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("acos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select acos from t1", + # "select acos(-+--+c1 ) from t1", + # "select +-acos(c1) from t1", + # "select ++-acos(c1) from t1", + # "select ++--acos(c1) from t1", + # "select - -acos(c1)*0 from t1", + # "select acos(tbname+1) from t1 ", + "select acos(123--123)==1 from t1", + "select acos(c1) as 'd1' from t1", + "select acos(c1 ,c2) from t1", + "select acos(c1 ,NULL ) from t1", + "select acos(,) from t1;", + "select acos(acos(c1) ab from t1)", + "select acos(c1 ) as int from t1", + "select acos from stb1", + # "select acos(-+--+c1) from stb1", + # "select +-acos(c1) from stb1", + # "select ++-acos(c1) from stb1", + # "select ++--acos(c1) from stb1", + # "select - -acos(c1)*0 from stb1", + # "select acos(tbname+1) from stb1 ", + "select acos(123--123)==1 from stb1", + "select acos(c1) as 'd1' from stb1", + "select acos(c1 ,c2 ) from stb1", + "select acos(c1 ,NULL) from stb1", + "select acos(,) from stb1;", + "select acos(acos(c1) ab from stb1)", + "select acos(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select acos(ts) from t1" , + "select acos(c7) from t1", + "select acos(c8) from t1", + "select acos(c9) from t1", + "select acos(ts) from ct1" , + "select acos(c7) from ct1", + "select acos(c8) from ct1", + "select acos(c9) from ct1", + "select acos(ts) from ct3" , + "select acos(c7) from ct3", + "select acos(c8) from ct3", + "select acos(c9) from ct3", + "select acos(ts) from ct4" , + "select acos(c7) from ct4", + "select acos(c8) from ct4", + "select acos(c9) from ct4", + "select acos(ts) from stb1" , + "select acos(c7) from stb1", + "select acos(c8) from stb1", + "select acos(c9) from stb1" , + + "select acos(ts) from stbbb1" , + "select acos(c7) from stbbb1", + + "select acos(ts) from tbname", + "select acos(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select acos(c1) from t1", + "select acos(c2) from t1", + "select acos(c3) from t1", + "select acos(c4) from t1", + "select acos(c5) from t1", + "select acos(c6) from t1", + + "select acos(c1) from ct1", + "select acos(c2) from ct1", + "select acos(c3) from ct1", + "select acos(c4) from ct1", + "select acos(c5) from ct1", + "select acos(c6) from ct1", + + "select acos(c1) from ct3", + "select acos(c2) from ct3", + "select acos(c3) from ct3", + "select acos(c4) from ct3", + "select acos(c5) from ct3", + "select acos(c6) from ct3", + + "select acos(c1) from stb1", + "select acos(c2) from stb1", + "select acos(c3) from stb1", + "select acos(c4) from stb1", + "select acos(c5) from stb1", + "select acos(c6) from stb1", + + "select acos(c6) as alisb from stb1", + "select acos(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_acos_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select acos(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select acos(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.000000000) + tdSql.checkData(3 , 0, None) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_acos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select acos(abs(c1)), acos(abs(c2)) ,acos(abs(c3)), acos(abs(c4)), acos(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,acos(c2) from ct1") + tdSql.checkData(0, 1, None) + tdSql.checkData(1 , 1, None) + tdSql.checkData(3 , 1, None) + tdSql.checkData(4 , 1, 1.570796327) + + tdSql.query("select c1, c5 ,acos(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, None) + tdSql.checkData(2 , 2, None) + tdSql.checkData(3 , 2, None) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_acos( "select c1, c2, c3 , c4, c5 from ct1", "select acos(c1), acos(c2) ,acos(c3), acos(c4), acos(c5) from ct1") + + # nest query for acos functions + tdSql.query("select c4 , acos(c4) ,acos(acos(c4)) , acos(acos(acos(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , None) + tdSql.checkData(0 , 2 , None) + tdSql.checkData(0 , 3 , None) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , None) + tdSql.checkData(1 , 2 , None) + tdSql.checkData(1 , 3 , None) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select acos(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select acos(c1) from stbbb1") + tdSql.error("select acos(c1) from tbname") + tdSql.error("select acos(c1) from ct5") + + # mix with common col + tdSql.query("select c1, acos(c1) from ct1") + tdSql.query("select c2, acos(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, acos(c1),acos(c1), acos(acos(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,None) + tdSql.checkData(3 , 2 ,None) + tdSql.checkData(3 , 3 ,None) + + tdSql.query("select c1, acos(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, acos(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, acos(c1),c5, count(c5) from ct1 ") + tdSql.error("select acos(c1), count(c5) from stb1 ") + tdSql.error("select acos(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, acos(c1) -0 ,acos(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query(" select c1, acos(c1) -0 ,acos(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query("select c1, acos(c1), c2, acos(c2), c3, acos(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, acos(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + + tdSql.query("select c1, acos(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, acos(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, acos(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, acos(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(acos(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(acos(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(acos(c1)-0.5) from ct4 where c1=-1 and elem <=1: + elem = math.asin(elem) + else: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("asin function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("asin value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select asin from t1", + # "select asin(-+--+c1 ) from t1", + # "select +-asin(c1) from t1", + # "select ++-asin(c1) from t1", + # "select ++--asin(c1) from t1", + # "select - -asin(c1)*0 from t1", + # "select asin(tbname+1) from t1 ", + "select asin(123--123)==1 from t1", + "select asin(c1) as 'd1' from t1", + "select asin(c1 ,c2) from t1", + "select asin(c1 ,NULL ) from t1", + "select asin(,) from t1;", + "select asin(asin(c1) ab from t1)", + "select asin(c1 ) as int from t1", + "select asin from stb1", + # "select asin(-+--+c1) from stb1", + # "select +-asin(c1) from stb1", + # "select ++-asin(c1) from stb1", + # "select ++--asin(c1) from stb1", + # "select - -asin(c1)*0 from stb1", + # "select asin(tbname+1) from stb1 ", + "select asin(123--123)==1 from stb1", + "select asin(c1) as 'd1' from stb1", + "select asin(c1 ,c2 ) from stb1", + "select asin(c1 ,NULL) from stb1", + "select asin(,) from stb1;", + "select asin(asin(c1) ab from stb1)", + "select asin(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select asin(ts) from t1" , + "select asin(c7) from t1", + "select asin(c8) from t1", + "select asin(c9) from t1", + "select asin(ts) from ct1" , + "select asin(c7) from ct1", + "select asin(c8) from ct1", + "select asin(c9) from ct1", + "select asin(ts) from ct3" , + "select asin(c7) from ct3", + "select asin(c8) from ct3", + "select asin(c9) from ct3", + "select asin(ts) from ct4" , + "select asin(c7) from ct4", + "select asin(c8) from ct4", + "select asin(c9) from ct4", + "select asin(ts) from stb1" , + "select asin(c7) from stb1", + "select asin(c8) from stb1", + "select asin(c9) from stb1" , + + "select asin(ts) from stbbb1" , + "select asin(c7) from stbbb1", + + "select asin(ts) from tbname", + "select asin(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select asin(c1) from t1", + "select asin(c2) from t1", + "select asin(c3) from t1", + "select asin(c4) from t1", + "select asin(c5) from t1", + "select asin(c6) from t1", + + "select asin(c1) from ct1", + "select asin(c2) from ct1", + "select asin(c3) from ct1", + "select asin(c4) from ct1", + "select asin(c5) from ct1", + "select asin(c6) from ct1", + + "select asin(c1) from ct3", + "select asin(c2) from ct3", + "select asin(c3) from ct3", + "select asin(c4) from ct3", + "select asin(c5) from ct3", + "select asin(c6) from ct3", + + "select asin(c1) from stb1", + "select asin(c2) from stb1", + "select asin(c3) from stb1", + "select asin(c4) from stb1", + "select asin(c5) from stb1", + "select asin(c6) from stb1", + + "select asin(c6) as alisb from stb1", + "select asin(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_asin_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select asin(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select asin(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.570796327) + tdSql.checkData(3 , 0, None) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_asin( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select asin(abs(c1)), asin(abs(c2)) ,asin(abs(c3)), asin(abs(c4)), asin(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,asin(c2) from ct1") + tdSql.checkData(0, 1, None) + tdSql.checkData(1 , 1, None) + tdSql.checkData(3 , 1, None) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,asin(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, None) + tdSql.checkData(2 , 2, None) + tdSql.checkData(3 , 2, None) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_asin( "select c1, c2, c3 , c4, c5 from ct1", "select asin(c1), asin(c2) ,asin(c3), asin(c4), asin(c5) from ct1") + + # nest query for asin functions + tdSql.query("select c4 , asin(c4) ,asin(asin(c4)) , asin(asin(asin(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , None) + tdSql.checkData(0 , 2 , None) + tdSql.checkData(0 , 3 , None) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , None) + tdSql.checkData(1 , 2 , None) + tdSql.checkData(1 , 3 , None) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select asin(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select asin(c1) from stbbb1") + tdSql.error("select asin(c1) from tbname") + tdSql.error("select asin(c1) from ct5") + + # mix with common col + tdSql.query("select c1, asin(c1) from ct1") + tdSql.query("select c2, asin(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, asin(c1),asin(c1), asin(asin(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,None) + tdSql.checkData(3 , 2 ,None) + tdSql.checkData(3 , 3 ,None) + + tdSql.query("select c1, asin(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, asin(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, asin(c1),c5, count(c5) from ct1 ") + tdSql.error("select asin(c1), count(c5) from stb1 ") + tdSql.error("select asin(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, asin(c1) -0 ,asin(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query(" select c1, asin(c1) -0 ,asin(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query("select c1, asin(c1), c2, asin(c2), c3, asin(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, asin(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + + tdSql.query("select c1, asin(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, asin(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, asin(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, asin(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(asin(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(asin(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(asin(c1)-0.5) from ct4 where c1 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("atan function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("atan value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select atan from t1", + # "select atan(-+--+c1 ) from t1", + # "select +-atan(c1) from t1", + # "select ++-atan(c1) from t1", + # "select ++--atan(c1) from t1", + # "select - -atan(c1)*0 from t1", + # "select atan(tbname+1) from t1 ", + "select atan(123--123)==1 from t1", + "select atan(c1) as 'd1' from t1", + "select atan(c1 ,c2) from t1", + "select atan(c1 ,NULL ) from t1", + "select atan(,) from t1;", + "select atan(atan(c1) ab from t1)", + "select atan(c1 ) as int from t1", + "select atan from stb1", + # "select atan(-+--+c1) from stb1", + # "select +-atan(c1) from stb1", + # "select ++-atan(c1) from stb1", + # "select ++--atan(c1) from stb1", + # "select - -atan(c1)*0 from stb1", + # "select atan(tbname+1) from stb1 ", + "select atan(123--123)==1 from stb1", + "select atan(c1) as 'd1' from stb1", + "select atan(c1 ,c2 ) from stb1", + "select atan(c1 ,NULL) from stb1", + "select atan(,) from stb1;", + "select atan(atan(c1) ab from stb1)", + "select atan(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select atan(ts) from t1" , + "select atan(c7) from t1", + "select atan(c8) from t1", + "select atan(c9) from t1", + "select atan(ts) from ct1" , + "select atan(c7) from ct1", + "select atan(c8) from ct1", + "select atan(c9) from ct1", + "select atan(ts) from ct3" , + "select atan(c7) from ct3", + "select atan(c8) from ct3", + "select atan(c9) from ct3", + "select atan(ts) from ct4" , + "select atan(c7) from ct4", + "select atan(c8) from ct4", + "select atan(c9) from ct4", + "select atan(ts) from stb1" , + "select atan(c7) from stb1", + "select atan(c8) from stb1", + "select atan(c9) from stb1" , + + "select atan(ts) from stbbb1" , + "select atan(c7) from stbbb1", + + "select atan(ts) from tbname", + "select atan(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select atan(c1) from t1", + "select atan(c2) from t1", + "select atan(c3) from t1", + "select atan(c4) from t1", + "select atan(c5) from t1", + "select atan(c6) from t1", + + "select atan(c1) from ct1", + "select atan(c2) from ct1", + "select atan(c3) from ct1", + "select atan(c4) from ct1", + "select atan(c5) from ct1", + "select atan(c6) from ct1", + + "select atan(c1) from ct3", + "select atan(c2) from ct3", + "select atan(c3) from ct3", + "select atan(c4) from ct3", + "select atan(c5) from ct3", + "select atan(c6) from ct3", + + "select atan(c1) from stb1", + "select atan(c2) from stb1", + "select atan(c3) from stb1", + "select atan(c4) from stb1", + "select atan(c5) from stb1", + "select atan(c6) from stb1", + + "select atan(c6) as alisb from stb1", + "select atan(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_atan_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select atan(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select atan(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.785398163) + tdSql.checkData(3 , 0, 1.249045772) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_atan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select atan(abs(c1)), atan(abs(c2)) ,atan(abs(c3)), atan(abs(c4)), atan(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,atan(c2) from ct1") + tdSql.checkData(0, 1, 1.570785077) + tdSql.checkData(1 , 1, 1.570783470) + tdSql.checkData(3 , 1, 1.570778327) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,atan(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 1.458656162) + tdSql.checkData(2 , 2, 1.442799803) + tdSql.checkData(3 , 2, 1.421759533) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_atan( "select c1, c2, c3 , c4, c5 from ct1", "select atan(c1), atan(c2) ,atan(c3), atan(c4), atan(c5) from ct1") + + # nest query for atan functions + tdSql.query("select c4 , atan(c4) ,atan(atan(c4)) , atan(atan(atan(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 1.559433180) + tdSql.checkData(0 , 2 , 1.000590740) + tdSql.checkData(0 , 3 , 0.785693446) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 1.557810044) + tdSql.checkData(1 , 2 , 1.000117426) + tdSql.checkData(1 , 3 , 0.785456873) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , -1.560695660) + tdSql.checkData(11 , 2 , -1.000958403) + tdSql.checkData(11 , 3 , -0.785877135) + + # used for stable table + + tdSql.query("select atan(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select atan(c1) from stbbb1") + tdSql.error("select atan(c1) from tbname") + tdSql.error("select atan(c1) from ct5") + + # mix with common col + tdSql.query("select c1, atan(c1) from ct1") + tdSql.query("select c2, atan(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, atan(c1),atan(c1), atan(atan(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,1.405647649) + tdSql.checkData(3 , 2 ,1.405647649) + tdSql.checkData(3 , 3 ,0.952449745) + + tdSql.query("select c1, atan(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, atan(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, atan(c1),c5, count(c5) from ct1 ") + tdSql.error("select atan(c1), count(c5) from stb1 ") + tdSql.error("select atan(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, atan(c1) -0 ,atan(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 1.446441332) + tdSql.checkData(1, 2, 1.325817664) + + tdSql.query(" select c1, atan(c1) -0 ,atan(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 1.446441332) + tdSql.checkData(1, 2, 1.344883701) + + tdSql.query("select c1, atan(c1), c2, atan(c2), c3, atan(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, atan(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.atan(100000000)) + + + tdSql.query("select c1, atan(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.atan(10000000000000)) + + tdSql.query("select c1, atan(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, atan(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.atan(10000000000000000000000000.0)) + + tdSql.query("select c1, atan(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, atan(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.atan(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, atan(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, atan(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.atan(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, atan(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(atan(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,1.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(atan(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,1.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(atan(c1)-0.5) from ct4 where c1=atan(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(0,2,0.000000000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,-0.100000000) + tdSql.checkData(0,5,0.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_atan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select atan(abs(c1)), atan(abs(c2)) ,atan(abs(c3)), atan(abs(c4)), atan(abs(c5)) from sub1_bound") + + self.check_result_auto_atan( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select atan(c1), atan(c2) ,atan(c3), atan(c3), atan(c2) ,atan(c1) from sub1_bound") + + self.check_result_auto_atan("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select atan(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select atan(abs(c1)) ,atan(abs(c2)) , atan(abs(c3)) , atan(abs(c4)), atan(abs(c5)), atan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.atan(2147483647)) + tdSql.checkData(0,1,math.atan(9223372036854775807)) + tdSql.checkData(0,2,math.atan(32767)) + tdSql.checkData(0,3,math.atan(127)) + tdSql.checkData(0,4,math.atan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.atan(2147483647)) + tdSql.checkData(1,1,math.atan(9223372036854775807)) + tdSql.checkData(1,2,math.atan(32767)) + tdSql.checkData(1,3,math.atan(127)) + tdSql.checkData(1,4,math.atan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.atan(2147483646)) + tdSql.checkData(3,1,math.atan(9223372036854775806)) + tdSql.checkData(3,2,math.atan(32766)) + tdSql.checkData(3,3,math.atan(126)) + tdSql.checkData(3,4,math.atan(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select atan(abs(c1+1)) ,atan(abs(c2)) , atan(abs(c3*1)) , atan(abs(c4/2)), atan(abs(c5))/2, atan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.atan(2147483648.000000000)) + tdSql.checkData(0,1,math.atan(9223372036854775807)) + tdSql.checkData(0,2,math.atan(32767.000000000)) + tdSql.checkData(0,3,math.atan(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + + self.check_result_auto_atan("select num1,num2 from tb3;" , "select atan(num1),atan(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: atan basic query ============") + + self.basic_atan_function() + + tdLog.printNoPrefix("==========step5: big number atan query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: atan boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: atan filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py new file mode 100644 index 0000000000..0cb9f66937 --- /dev/null +++ b/tests/system-test/2-query/cos.py @@ -0,0 +1,518 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_cos(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.cos(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("cos function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("cos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select cos from t1", + # "select cos(-+--+c1 ) from t1", + # "select +-cos(c1) from t1", + # "select ++-cos(c1) from t1", + # "select ++--cos(c1) from t1", + # "select - -cos(c1)*0 from t1", + # "select cos(tbname+1) from t1 ", + "select cos(123--123)==1 from t1", + "select cos(c1) as 'd1' from t1", + "select cos(c1 ,c2) from t1", + "select cos(c1 ,NULL ) from t1", + "select cos(,) from t1;", + "select cos(cos(c1) ab from t1)", + "select cos(c1 ) as int from t1", + "select cos from stb1", + # "select cos(-+--+c1) from stb1", + # "select +-cos(c1) from stb1", + # "select ++-cos(c1) from stb1", + # "select ++--cos(c1) from stb1", + # "select - -cos(c1)*0 from stb1", + # "select cos(tbname+1) from stb1 ", + "select cos(123--123)==1 from stb1", + "select cos(c1) as 'd1' from stb1", + "select cos(c1 ,c2 ) from stb1", + "select cos(c1 ,NULL) from stb1", + "select cos(,) from stb1;", + "select cos(cos(c1) ab from stb1)", + "select cos(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select cos(ts) from t1" , + "select cos(c7) from t1", + "select cos(c8) from t1", + "select cos(c9) from t1", + "select cos(ts) from ct1" , + "select cos(c7) from ct1", + "select cos(c8) from ct1", + "select cos(c9) from ct1", + "select cos(ts) from ct3" , + "select cos(c7) from ct3", + "select cos(c8) from ct3", + "select cos(c9) from ct3", + "select cos(ts) from ct4" , + "select cos(c7) from ct4", + "select cos(c8) from ct4", + "select cos(c9) from ct4", + "select cos(ts) from stb1" , + "select cos(c7) from stb1", + "select cos(c8) from stb1", + "select cos(c9) from stb1" , + + "select cos(ts) from stbbb1" , + "select cos(c7) from stbbb1", + + "select cos(ts) from tbname", + "select cos(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select cos(c1) from t1", + "select cos(c2) from t1", + "select cos(c3) from t1", + "select cos(c4) from t1", + "select cos(c5) from t1", + "select cos(c6) from t1", + + "select cos(c1) from ct1", + "select cos(c2) from ct1", + "select cos(c3) from ct1", + "select cos(c4) from ct1", + "select cos(c5) from ct1", + "select cos(c6) from ct1", + + "select cos(c1) from ct3", + "select cos(c2) from ct3", + "select cos(c3) from ct3", + "select cos(c4) from ct3", + "select cos(c5) from ct3", + "select cos(c6) from ct3", + + "select cos(c1) from stb1", + "select cos(c2) from stb1", + "select cos(c3) from stb1", + "select cos(c4) from stb1", + "select cos(c5) from stb1", + "select cos(c6) from stb1", + + "select cos(c6) as alisb from stb1", + "select cos(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_cosin_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select cos(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select cos(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.540302306) + tdSql.checkData(3 , 0, -0.989992497) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_cos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,cos(c2) from ct1") + tdSql.checkData(0, 1, 0.975339851) + tdSql.checkData(1 , 1, -0.830564903) + tdSql.checkData(3 , 1, 0.602244939) + tdSql.checkData(4 , 1, 1.000000000) + + tdSql.query("select c1, c5 ,cos(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, -0.855242438) + tdSql.checkData(2 , 2, 0.083882969) + tdSql.checkData(3 , 2, 0.929841474) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_cos( "select c1, c2, c3 , c4, c5 from ct1", "select cos(c1), cos(c2) ,cos(c3), cos(c4), cos(c5) from ct1") + + # nest query for cos functions + tdSql.query("select c4 , cos(c4) ,cos(cos(c4)) , cos(cos(cos(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 0.999373284) + tdSql.checkData(0 , 2 , 0.540829563) + tdSql.checkData(0 , 3 , 0.857281878) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , -0.030975032) + tdSql.checkData(1 , 2 , 0.999520312) + tdSql.checkData(1 , 3 , 0.540705887) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , 0.039820880) + tdSql.checkData(11 , 2 , 0.999207254) + tdSql.checkData(11 , 3 , 0.540969209) + + # used for stable table + + tdSql.query("select cos(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select cos(c1) from stbbb1") + tdSql.error("select cos(c1) from tbname") + tdSql.error("select cos(c1) from ct5") + + # mix with common col + tdSql.query("select c1, cos(c1) from ct1") + tdSql.query("select c2, cos(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, cos(c1),cos(c1), cos(cos(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,0.960170287) + tdSql.checkData(3 , 2 ,0.960170287) + tdSql.checkData(3 , 3 ,0.573380480) + + tdSql.query("select c1, cos(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, cos(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, cos(c1),c5, count(c5) from ct1 ") + tdSql.error("select cos(c1), count(c5) from stb1 ") + tdSql.error("select cos(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, cos(c1) -0 ,cos(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -0.145500034) + tdSql.checkData(1, 2, -0.653643621) + + tdSql.query(" select c1, cos(c1) -0 ,cos(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -0.145500034) + tdSql.checkData(1, 2, -0.146002126) + + tdSql.query("select c1, cos(c1), c2, cos(c2), c3, cos(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, cos(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.cos(100000000)) + + + tdSql.query("select c1, cos(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.cos(10000000000000)) + + tdSql.query("select c1, cos(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, cos(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.cos(10000000000000000000000000.0)) + + tdSql.query("select c1, cos(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, cos(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.cos(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, cos(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, cos(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.cos(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, cos(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,0.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,0.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1>cos(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,0.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_cos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from sub1_bound") + + self.check_result_auto_cos( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select cos(c1), cos(c2) ,cos(c3), cos(c3), cos(c2) ,cos(c1) from sub1_bound") + + self.check_result_auto_cos("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select cos(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select cos(abs(c1)) ,cos(abs(c2)) , cos(abs(c3)) , cos(abs(c4)), cos(abs(c5)), cos(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.cos(2147483647)) + tdSql.checkData(0,1,math.cos(9223372036854775807)) + tdSql.checkData(0,2,math.cos(32767)) + tdSql.checkData(0,3,math.cos(127)) + tdSql.checkData(0,4,math.cos(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.cos(2147483647)) + tdSql.checkData(1,1,math.cos(9223372036854775807)) + tdSql.checkData(1,2,math.cos(32767)) + tdSql.checkData(1,3,math.cos(127)) + tdSql.checkData(1,4,math.cos(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.cos(2147483646)) + tdSql.checkData(3,1,math.cos(9223372036854775806)) + tdSql.checkData(3,2,math.cos(32766)) + tdSql.checkData(3,3,math.cos(126)) + tdSql.checkData(3,4,math.cos(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select cos(abs(c1+1)) ,cos(abs(c2)) , cos(abs(c3*1)) , cos(abs(c4/2)), cos(abs(c5))/2, cos(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.cos(2147483648.000000000)) + tdSql.checkData(0,1,math.cos(9223372036854775807)) + tdSql.checkData(0,2,math.cos(32767.000000000)) + tdSql.checkData(0,3,math.cos(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + + self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: cos basic query ============") + + self.basic_cosin_function() + + tdLog.printNoPrefix("==========step5: big number cos query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: cos boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: cos filter query ============") + + self.abs_func_filter() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py new file mode 100644 index 0000000000..a70a2e72f2 --- /dev/null +++ b/tests/system-test/2-query/count.py @@ -0,0 +1,196 @@ +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute("create table stb_2 using stb tags('shanghai')") + + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + tdSql.execute("insert into stb_2 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.query("select count(*) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(*) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(ts) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(ts) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col1) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col1) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col2) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col2) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col3) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col3) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col4) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col4) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col5) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col5) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col6) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col6) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col7) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col7) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col8) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col8) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col9) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col9) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col11) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col11) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col12) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col12) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col13) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col13) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col14) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col14) from db.stb") + tdSql.checkData(0,0,20) + + + + tdSql.query("select count(ts) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(ts) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col2) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col2) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col3) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col3) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col4) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col4) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col5) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col5) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col6) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col6) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col7) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col7) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col8) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col8) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col9) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col9) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col11) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col11) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col12) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col12) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col13) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col13) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col14) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col14) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from stb_1 group by col7") + tdSql.checkRows(2) + + tdSql.execute("insert into stb_1 values(now,null,null,null,null,null,null,null,null,null,null,null,null,null)") + tdSql.query("select count(col1) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1),count(ts) from stb_1") + tdSql.checkData(0,0,10) + tdSql.checkData(0,1,11) + + tdSql.query("select count(col1) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1),count(ts) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.checkData(0,1,11) + + tdSql.query("select count(col1) from stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col1),count(ts) from stb") + tdSql.checkData(0,0,20) + tdSql.checkData(0,1,21) + + tdSql.query("select count(col1) from db.stb") + tdSql.checkData(0,0,20) + tdSql.query("select count(col1),count(ts) from db.stb") + tdSql.checkData(0,0,20) + tdSql.checkData(0,1,21) + tdSql.query("select count(col1) from stb_1 group by col7") + tdSql.checkRows(3) + tdSql.query("select count(col1) from stb_2 group by col7") + tdSql.checkRows(2) + tdSql.query("select count(col1) from stb group by col7") + tdSql.checkRows(3) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py new file mode 100644 index 0000000000..b5c0498e40 --- /dev/null +++ b/tests/system-test/2-query/last.py @@ -0,0 +1,331 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute("insert into stb_1(ts) values(%d)" % (self.ts - 1)) + + # last verifacation + # tdSql.query("select last(*) from stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) + # tdSql.query("select last(*) from db.stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) + tdSql.query("select last(col1) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col1) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col2) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col2) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col3) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col3) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col4) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col4) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col11) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col11) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col12) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col12) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col13) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col13) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col14) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col14) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col5) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col5) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col6) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col6) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col7) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col7) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col8) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col8) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col9) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col9) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select count(col1) from stb_1 group by col7") + tdSql.checkRows(1) + + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + # tdSql.query("select last(*) from stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, 10) + # tdSql.query("select last(*) from db.stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, 10) + tdSql.query("select last(col1) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col1) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col2) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col2) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col3) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col3) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col4) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col4) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col5) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col5) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col6) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col6) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col7) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + tdSql.query("select last(col7) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + tdSql.query("select last(col8) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + tdSql.query("select last(col8) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + tdSql.query("select last(col9) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + tdSql.query("select last(col9) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + tdSql.execute("insert into ntb(ts) values(%d)" % (self.ts - 1)) + # tdSql.query("select last(*) from ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) + # tdSql.query("select last(*) from db.ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) + tdSql.query("select last(col1) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col1) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col2) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col2) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col3) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col3) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col4) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col4) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col11) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col11) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col12) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col12) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col13) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col13) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col14) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col14) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col5) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col5) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col6) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col6) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col7) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col7) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col8) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col8) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col9) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col9) from db.ntb") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.query("select last(*) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + tdSql.query("select last(*) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + tdSql.query("select last(col1) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col1) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col2) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col2) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col3) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col3) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col4) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col4) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col5) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col5) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col6) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col6) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col7) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + tdSql.query("select last(col7) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + tdSql.query("select last(col8) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + tdSql.query("select last(col8) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + # tdSql.query("select last(col9) from ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 0, '涛思数据10') + # tdSql.query("select last(col9) from db.ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py new file mode 100644 index 0000000000..5342c7d449 --- /dev/null +++ b/tests/system-test/2-query/max.py @@ -0,0 +1,205 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def prepare_data(self): + + pass + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # max verifacation + tdSql.error("select max(ts) from stb_1") + tdSql.error("select max(ts) from db.stb_1") + tdSql.error("select max(col7) from stb_1") + tdSql.error("select max(col7) from db.stb_1") + tdSql.error("select max(col8) from stb_1") + tdSql.error("select max(col8) from db.stb_1") + tdSql.error("select max(col9) from stb_1") + tdSql.error("select max(col9) from db.stb_1") + + tdSql.query("select max(col1) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col1) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col5) from stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col5) from db.stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from db.stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from stb_1 where col2<=5") + tdSql.checkData(0,0,5) + + + + tdSql.error("select max(ts) from stb") + tdSql.error("select max(ts) from db.stb") + tdSql.error("select max(col7) from stb") + tdSql.error("select max(col7) from db.stb") + tdSql.error("select max(col8) from stb") + tdSql.error("select max(col8) from db.stb") + tdSql.error("select max(col9) from stb") + tdSql.error("select max(col9) from db.stb") + + tdSql.query("select max(col1) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col1) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col5) from stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col5) from db.stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from db.stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from stb where col2<=5") + tdSql.checkData(0,0,5) + + + + tdSql.error("select max(ts) from ntb") + tdSql.error("select max(ts) from db.ntb") + tdSql.error("select max(col7) from ntb") + tdSql.error("select max(col7) from db.ntb") + tdSql.error("select max(col8) from ntb") + tdSql.error("select max(col8) from db.ntb") + tdSql.error("select max(col9) from ntb") + tdSql.error("select max(col9) from db.ntb") + + tdSql.query("select max(col1) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col1) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col5) from ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col5) from db.ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from db.ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from stb_1 where col2<=5") + tdSql.checkData(0,0,5) + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/min.py b/tests/system-test/2-query/min.py new file mode 100644 index 0000000000..60a8c105f1 --- /dev/null +++ b/tests/system-test/2-query/min.py @@ -0,0 +1,215 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # max verifacation + tdSql.error("select min(ts) from stb_1") + tdSql.error("select min(ts) from db.stb_1") + tdSql.error("select min(col7) from stb_1") + tdSql.error("select min(col7) from db.stb_1") + tdSql.error("select min(col8) from stb_1") + tdSql.error("select min(col8) from db.stb_1") + tdSql.error("select min(col9) from stb_1") + tdSql.error("select min(col9) from db.stb_1") + # tdSql.error("select min(a) from stb_1") + # tdSql.error("select min(1) from stb_1") + tdSql.error("select min(now()) from stb_1") + tdSql.error("select min(count(c1),count(c2)) from stb_1") + + tdSql.query("select min(col1) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col1) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col5) from stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col5) from db.stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from db.stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col1) from stb_1 where col2>=5") + tdSql.checkData(0,0,5) + + + tdSql.error("select min(ts) from stb_1") + tdSql.error("select min(ts) from db.stb_1") + tdSql.error("select min(col7) from stb_1") + tdSql.error("select min(col7) from db.stb_1") + tdSql.error("select min(col8) from stb_1") + tdSql.error("select min(col8) from db.stb_1") + tdSql.error("select min(col9) from stb_1") + tdSql.error("select min(col9) from db.stb_1") + # tdSql.error("select min(a) from stb_1") + # tdSql.error("select min(1) from stb_1") + tdSql.error("select min(now()) from stb_1") + tdSql.error("select min(count(c1),count(c2)) from stb_1") + + tdSql.query("select min(col1) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col1) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col5) from stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col5) from db.stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from db.stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col1) from stb where col2>=5") + tdSql.checkData(0,0,5) + + + tdSql.error("select min(ts) from ntb") + tdSql.error("select min(ts) from db.ntb") + tdSql.error("select min(col7) from ntb") + tdSql.error("select min(col7) from db.ntb") + tdSql.error("select min(col8) from ntb") + tdSql.error("select min(col8) from db.ntb") + tdSql.error("select min(col9) from ntb") + tdSql.error("select min(col9) from db.ntb") + # tdSql.error("select min(a) from stb_1") + # tdSql.error("select min(1) from stb_1") + tdSql.error("select min(now()) from ntb") + tdSql.error("select min(count(c1),count(c2)) from ntb") + + tdSql.query("select min(col1) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col1) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col5) from ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col5) from db.ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from db.ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col1) from ntb where col2>=5") + tdSql.checkData(0,0,5) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py new file mode 100644 index 0000000000..62d42a991e --- /dev/null +++ b/tests/system-test/2-query/sin.py @@ -0,0 +1,519 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_sin(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.sin(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("sin function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("sin value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select sin from t1", + # "select sin(-+--+c1 ) from t1", + # "select +-sin(c1) from t1", + # "select ++-sin(c1) from t1", + # "select ++--sin(c1) from t1", + # "select - -sin(c1)*0 from t1", + # "select sin(tbname+1) from t1 ", + "select sin(123--123)==1 from t1", + "select sin(c1) as 'd1' from t1", + "select sin(c1 ,c2) from t1", + "select sin(c1 ,NULL ) from t1", + "select sin(,) from t1;", + "select sin(sin(c1) ab from t1)", + "select sin(c1 ) as int from t1", + "select sin from stb1", + # "select sin(-+--+c1) from stb1", + # "select +-sin(c1) from stb1", + # "select ++-sin(c1) from stb1", + # "select ++--sin(c1) from stb1", + # "select - -sin(c1)*0 from stb1", + # "select sin(tbname+1) from stb1 ", + "select sin(123--123)==1 from stb1", + "select sin(c1) as 'd1' from stb1", + "select sin(c1 ,c2 ) from stb1", + "select sin(c1 ,NULL) from stb1", + "select sin(,) from stb1;", + "select sin(sin(c1) ab from stb1)", + "select sin(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select sin(ts) from t1" , + "select sin(c7) from t1", + "select sin(c8) from t1", + "select sin(c9) from t1", + "select sin(ts) from ct1" , + "select sin(c7) from ct1", + "select sin(c8) from ct1", + "select sin(c9) from ct1", + "select sin(ts) from ct3" , + "select sin(c7) from ct3", + "select sin(c8) from ct3", + "select sin(c9) from ct3", + "select sin(ts) from ct4" , + "select sin(c7) from ct4", + "select sin(c8) from ct4", + "select sin(c9) from ct4", + "select sin(ts) from stb1" , + "select sin(c7) from stb1", + "select sin(c8) from stb1", + "select sin(c9) from stb1" , + + "select sin(ts) from stbbb1" , + "select sin(c7) from stbbb1", + + "select sin(ts) from tbname", + "select sin(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select sin(c1) from t1", + "select sin(c2) from t1", + "select sin(c3) from t1", + "select sin(c4) from t1", + "select sin(c5) from t1", + "select sin(c6) from t1", + + "select sin(c1) from ct1", + "select sin(c2) from ct1", + "select sin(c3) from ct1", + "select sin(c4) from ct1", + "select sin(c5) from ct1", + "select sin(c6) from ct1", + + "select sin(c1) from ct3", + "select sin(c2) from ct3", + "select sin(c3) from ct3", + "select sin(c4) from ct3", + "select sin(c5) from ct3", + "select sin(c6) from ct3", + + "select sin(c1) from stb1", + "select sin(c2) from stb1", + "select sin(c3) from stb1", + "select sin(c4) from stb1", + "select sin(c5) from stb1", + "select sin(c6) from stb1", + + "select sin(c6) as alisb from stb1", + "select sin(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_sin_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select sin(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select sin(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.841470985) + tdSql.checkData(3 , 0, 0.141120008) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_sin( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sin(abs(c1)), sin(abs(c2)) ,sin(abs(c3)), sin(abs(c4)), sin(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,sin(c2) from ct1") + tdSql.checkData(0, 1, -0.220708349) + tdSql.checkData(1 , 1, -0.556921845) + tdSql.checkData(3 , 1, -0.798311364) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,sin(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 0.518228108) + tdSql.checkData(2 , 2, 0.996475613) + tdSql.checkData(3 , 2, 0.367960369) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_sin( "select c1, c2, c3 , c4, c5 from ct1", "select sin(c1), sin(c2) ,sin(c3), sin(c4), sin(c5) from ct1") + + # nest query for sin functions + tdSql.query("select c4 , sin(c4) ,sin(sin(c4)) , sin(sin(sin(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 0.035398303) + tdSql.checkData(0 , 2 , 0.035390911) + tdSql.checkData(0 , 3 , 0.035383523) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 0.999520159) + tdSql.checkData(1 , 2 , 0.841211629) + tdSql.checkData(1 , 3 , 0.745451290) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , 0.999206834) + tdSql.checkData(11 , 2 , 0.841042171) + tdSql.checkData(11 , 3 , 0.745338326) + + # used for stable table + + tdSql.query("select sin(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select sin(c1) from stbbb1") + tdSql.error("select sin(c1) from tbname") + tdSql.error("select sin(c1) from ct5") + + # mix with common col + tdSql.query("select c1, sin(c1) from ct1") + tdSql.query("select c2, sin(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, sin(c1),sin(c1), sin(sin(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,-0.279415498) + tdSql.checkData(3 , 2 ,-0.279415498) + tdSql.checkData(3 , 3 ,-0.275793863) + + tdSql.query("select c1, sin(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, sin(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, sin(c1),c5, count(c5) from ct1 ") + tdSql.error("select sin(c1), count(c5) from stb1 ") + tdSql.error("select sin(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, sin(c1) -0 ,sin(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 0.989358247) + tdSql.checkData(1, 2, -0.756802495) + + tdSql.query(" select c1, sin(c1) -0 ,sin(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 0.989358247) + tdSql.checkData(1, 2, 0.898941342) + + tdSql.query("select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, sin(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.sin(100000000)) + + + tdSql.query("select c1, sin(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.sin(10000000000000)) + + tdSql.query("select c1, sin(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sin(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.sin(10000000000000000000000000.0)) + + tdSql.query("select c1, sin(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sin(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.sin(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, sin(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sin(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.sin(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, sin(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sin(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,1.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sin(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,-1.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sin(c1)-0.5) from ct4 where c1=sin(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(0,2,0.000000000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,-0.100000000) + tdSql.checkData(0,5,0.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_sin( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sin(abs(c1)), sin(abs(c2)) ,sin(abs(c3)), sin(abs(c4)), sin(abs(c5)) from sub1_bound") + + self.check_result_auto_sin( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sin(c1), sin(c2) ,sin(c3), sin(c3), sin(c2) ,sin(c1) from sub1_bound") + + self.check_result_auto_sin("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select sin(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select sin(abs(c1)) ,sin(abs(c2)) , sin(abs(c3)) , sin(abs(c4)), sin(abs(c5)), sin(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sin(2147483647)) + tdSql.checkData(0,1,math.sin(9223372036854775807)) + tdSql.checkData(0,2,math.sin(32767)) + tdSql.checkData(0,3,math.sin(127)) + tdSql.checkData(0,4,math.sin(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.sin(2147483647)) + tdSql.checkData(1,1,math.sin(9223372036854775807)) + tdSql.checkData(1,2,math.sin(32767)) + tdSql.checkData(1,3,math.sin(127)) + tdSql.checkData(1,4,math.sin(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.sin(2147483646)) + tdSql.checkData(3,1,math.sin(9223372036854775806)) + tdSql.checkData(3,2,math.sin(32766)) + tdSql.checkData(3,3,math.sin(126)) + tdSql.checkData(3,4,math.sin(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select sin(abs(c1+1)) ,sin(abs(c2)) , sin(abs(c3*1)) , sin(abs(c4/2)), sin(abs(c5))/2, sin(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sin(2147483648.000000000)) + tdSql.checkData(0,1,math.sin(9223372036854775807)) + tdSql.checkData(0,2,math.sin(32767.000000000)) + tdSql.checkData(0,3,math.sin(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + + self.check_result_auto_sin("select num1,num2 from tb3;" , "select sin(num1),sin(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: sin basic query ============") + + self.basic_sin_function() + + tdLog.printNoPrefix("==========step5: big number sin query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: sin boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: sin filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py new file mode 100644 index 0000000000..c8914d18c0 --- /dev/null +++ b/tests/system-test/2-query/tan.py @@ -0,0 +1,518 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_tan(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.tan(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("tan function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("tan value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select tan from t1", + # "select tan(-+--+c1 ) from t1", + # "select +-tan(c1) from t1", + # "select ++-tan(c1) from t1", + # "select ++--tan(c1) from t1", + # "select - -tan(c1)*0 from t1", + # "select tan(tbname+1) from t1 ", + "select tan(123--123)==1 from t1", + "select tan(c1) as 'd1' from t1", + "select tan(c1 ,c2) from t1", + "select tan(c1 ,NULL ) from t1", + "select tan(,) from t1;", + "select tan(tan(c1) ab from t1)", + "select tan(c1 ) as int from t1", + "select tan from stb1", + # "select tan(-+--+c1) from stb1", + # "select +-tan(c1) from stb1", + # "select ++-tan(c1) from stb1", + # "select ++--tan(c1) from stb1", + # "select - -tan(c1)*0 from stb1", + # "select tan(tbname+1) from stb1 ", + "select tan(123--123)==1 from stb1", + "select tan(c1) as 'd1' from stb1", + "select tan(c1 ,c2 ) from stb1", + "select tan(c1 ,NULL) from stb1", + "select tan(,) from stb1;", + "select tan(tan(c1) ab from stb1)", + "select tan(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select tan(ts) from t1" , + "select tan(c7) from t1", + "select tan(c8) from t1", + "select tan(c9) from t1", + "select tan(ts) from ct1" , + "select tan(c7) from ct1", + "select tan(c8) from ct1", + "select tan(c9) from ct1", + "select tan(ts) from ct3" , + "select tan(c7) from ct3", + "select tan(c8) from ct3", + "select tan(c9) from ct3", + "select tan(ts) from ct4" , + "select tan(c7) from ct4", + "select tan(c8) from ct4", + "select tan(c9) from ct4", + "select tan(ts) from stb1" , + "select tan(c7) from stb1", + "select tan(c8) from stb1", + "select tan(c9) from stb1" , + + "select tan(ts) from stbbb1" , + "select tan(c7) from stbbb1", + + "select tan(ts) from tbname", + "select tan(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select tan(c1) from t1", + "select tan(c2) from t1", + "select tan(c3) from t1", + "select tan(c4) from t1", + "select tan(c5) from t1", + "select tan(c6) from t1", + + "select tan(c1) from ct1", + "select tan(c2) from ct1", + "select tan(c3) from ct1", + "select tan(c4) from ct1", + "select tan(c5) from ct1", + "select tan(c6) from ct1", + + "select tan(c1) from ct3", + "select tan(c2) from ct3", + "select tan(c3) from ct3", + "select tan(c4) from ct3", + "select tan(c5) from ct3", + "select tan(c6) from ct3", + + "select tan(c1) from stb1", + "select tan(c2) from stb1", + "select tan(c3) from stb1", + "select tan(c4) from stb1", + "select tan(c5) from stb1", + "select tan(c6) from stb1", + + "select tan(c6) as alisb from stb1", + "select tan(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_tan_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select tan(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select tan(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.557407725) + tdSql.checkData(3 , 0, -0.142546543) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_tan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select tan(abs(c1)), tan(abs(c2)) ,tan(abs(c3)), tan(abs(c4)), tan(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,tan(c2) from ct1") + tdSql.checkData(0, 1, -0.226288661) + tdSql.checkData(1 , 1, 0.670533806) + tdSql.checkData(3 , 1, -1.325559275) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,tan(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, -0.605942929) + tdSql.checkData(2 , 2, 11.879355609) + tdSql.checkData(3 , 2, 0.395723765) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_tan( "select c1, c2, c3 , c4, c5 from ct1", "select tan(c1), tan(c2) ,tan(c3), tan(c4), tan(c5) from ct1") + + # nest query for tan functions + tdSql.query("select c4 , tan(c4) ,tan(tan(c4)) , tan(tan(tan(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 0.035420501) + tdSql.checkData(0 , 2 , 0.035435322) + tdSql.checkData(0 , 3 , 0.035450161) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , -32.268575776) + tdSql.checkData(1 , 2 , -1.144433254) + tdSql.checkData(1 , 3 , -2.201545113) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , 25.092534980) + tdSql.checkData(11 , 2 , -0.040227928) + tdSql.checkData(11 , 3 , -0.040249642) + + # used for stable table + + tdSql.query("select tan(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select tan(c1) from stbbb1") + tdSql.error("select tan(c1) from tbname") + tdSql.error("select tan(c1) from ct5") + + # mix with common col + tdSql.query("select c1, tan(c1) from ct1") + tdSql.query("select c2, tan(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, tan(c1),tan(c1), tan(tan(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,-0.291006191) + tdSql.checkData(3 , 2 ,-0.291006191) + tdSql.checkData(3 , 3 ,-0.299508909) + + tdSql.query("select c1, tan(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, tan(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, tan(c1),c5, count(c5) from ct1 ") + tdSql.error("select tan(c1), count(c5) from stb1 ") + tdSql.error("select tan(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, tan(c1) -0 ,tan(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -6.799711455) + tdSql.checkData(1, 2, 1.157821282) + + tdSql.query(" select c1, tan(c1) -0 ,tan(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -6.799711455) + tdSql.checkData(1, 2, -21.815112681) + + tdSql.query("select c1, tan(c1), c2, tan(c2), c3, tan(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, tan(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.tan(100000000)) + + + tdSql.query("select c1, tan(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.tan(10000000000000)) + + tdSql.query("select c1, tan(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, tan(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.tan(10000000000000000000000000.0)) + + tdSql.query("select c1, tan(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, tan(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.tan(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, tan(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, tan(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.tan(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, tan(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(tan(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,-7.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(tan(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,-3.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(tan(c1)-0.5) from ct4 where c1>tan(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,-7.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_tan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select tan(abs(c1)), tan(abs(c2)) ,tan(abs(c3)), tan(abs(c4)), tan(abs(c5)) from sub1_bound") + + self.check_result_auto_tan( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select tan(c1), tan(c2) ,tan(c3), tan(c3), tan(c2) ,tan(c1) from sub1_bound") + + self.check_result_auto_tan("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select tan(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select tan(abs(c1)) ,tan(abs(c2)) , tan(abs(c3)) , tan(abs(c4)), tan(abs(c5)), tan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.tan(2147483647)) + tdSql.checkData(0,1,math.tan(9223372036854775807)) + tdSql.checkData(0,2,math.tan(32767)) + tdSql.checkData(0,3,math.tan(127)) + tdSql.checkData(0,4,math.tan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.tan(2147483647)) + tdSql.checkData(1,1,math.tan(9223372036854775807)) + tdSql.checkData(1,2,math.tan(32767)) + tdSql.checkData(1,3,math.tan(127)) + tdSql.checkData(1,4,math.tan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.tan(2147483646)) + tdSql.checkData(3,1,math.tan(9223372036854775806)) + tdSql.checkData(3,2,math.tan(32766)) + tdSql.checkData(3,3,math.tan(126)) + tdSql.checkData(3,4,math.tan(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select tan(abs(c1+1)) ,tan(abs(c2)) , tan(abs(c3*1)) , tan(abs(c4/2)), tan(abs(c5))/2, tan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.tan(2147483648.000000000)) + tdSql.checkData(0,1,math.tan(9223372036854775807)) + tdSql.checkData(0,2,math.tan(32767.000000000)) + tdSql.checkData(0,3,math.tan(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + + self.check_result_auto_tan("select num1,num2 from tb3;" , "select tan(num1),tan(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: tan basic query ============") + + self.basic_tan_function() + + tdLog.printNoPrefix("==========step5: big number tan query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: tan boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: tan filter query ============") + + self.abs_func_filter() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py new file mode 100644 index 0000000000..a48851b251 --- /dev/null +++ b/tests/system-test/2-query/timetruncate.py @@ -0,0 +1,220 @@ +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 # 2018-9-17 09:00:00.000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select timetruncate(1,1d) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1u) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1a) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1m) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1h) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(ts,1d) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 08:00:00.000") + tdSql.query("select timetruncate(ts,1h) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1m) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1s) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1a) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.checkData(1,0,"2018-09-17 09:00:00.001") + tdSql.checkData(2,0,"2018-09-17 09:00:00.002") + tdSql.checkData(3,0,"2018-09-17 09:00:00.003") + tdSql.checkData(4,0,"2018-09-17 09:00:00.004") + tdSql.checkData(5,0,"2018-09-17 09:00:00.005") + tdSql.checkData(6,0,"2018-09-17 09:00:00.006") + tdSql.checkData(7,0,"2018-09-17 09:00:00.007") + tdSql.checkData(8,0,"2018-09-17 09:00:00.008") + tdSql.checkData(9,0,"2018-09-17 09:00:00.009") + # tdSql.query("select timetruncate(ts,1u) from ntb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000") + # tdSql.query("select timetruncate(ts,1b) from ntb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000") + + + tdSql.query("select timetruncate(1,1d) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1u) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1a) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1m) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1h) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(ts,1d) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 08:00:00.000") + tdSql.query("select timetruncate(ts,1h) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1m) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1s) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1a) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.checkData(1,0,"2018-09-17 09:00:00.001") + tdSql.checkData(2,0,"2018-09-17 09:00:00.002") + tdSql.checkData(3,0,"2018-09-17 09:00:00.003") + tdSql.checkData(4,0,"2018-09-17 09:00:00.004") + tdSql.checkData(5,0,"2018-09-17 09:00:00.005") + tdSql.checkData(6,0,"2018-09-17 09:00:00.006") + tdSql.checkData(7,0,"2018-09-17 09:00:00.007") + tdSql.checkData(8,0,"2018-09-17 09:00:00.008") + tdSql.checkData(9,0,"2018-09-17 09:00:00.009") + # tdSql.query("select timetruncate(ts,1u) from stb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000") + # tdSql.query("select timetruncate(ts,1b) from stb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000") + + tdSql.query("select timetruncate(1,1d) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1u) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1a) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1m) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1h) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(ts,1d) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 08:00:00.000") + tdSql.query("select timetruncate(ts,1h) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1m) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1s) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1a) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.checkData(1,0,"2018-09-17 09:00:00.001") + tdSql.checkData(2,0,"2018-09-17 09:00:00.002") + tdSql.checkData(3,0,"2018-09-17 09:00:00.003") + tdSql.checkData(4,0,"2018-09-17 09:00:00.004") + tdSql.checkData(5,0,"2018-09-17 09:00:00.005") + tdSql.checkData(6,0,"2018-09-17 09:00:00.006") + tdSql.checkData(7,0,"2018-09-17 09:00:00.007") + tdSql.checkData(8,0,"2018-09-17 09:00:00.008") + tdSql.checkData(9,0,"2018-09-17 09:00:00.009") + # tdSql.query("select timetruncate(ts,1u) from stb_1") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000") + # tdSql.query("select timetruncate(ts,1b) from stb_1") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000") + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 51f1649cc8..22dc59365a 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -5,16 +5,25 @@ set -x python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py +python3 ./test.py -f 0-others/telemetry.py #python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py -#python3 ./test.py -f 2-query/timezone.py +python3 ./test.py -f 2-query/timezone.py python3 ./test.py -f 2-query/Now.py python3 ./test.py -f 2-query/Today.py +python3 ./test.py -f 2-query/max.py +python3 ./test.py -f 2-query/min.py +python3 ./test.py -f 2-query/count.py +python3 ./test.py -f 2-query/last.py +#python3 ./test.py -f 2-query/To_iso8601.py +python3 ./test.py -f 2-query/To_unixtimestamp.py +python3 ./test.py -f 2-query/timetruncate.py +# python3 ./test.py -f 2-query/Timediff.py #python3 ./test.py -f 2-query/cast.py @@ -25,3 +34,9 @@ python3 ./test.py -f 2-query/round.py python3 ./test.py -f 2-query/log.py python3 ./test.py -f 2-query/pow.py python3 ./test.py -f 2-query/sqrt.py +python3 ./test.py -f 2-query/sin.py +python3 ./test.py -f 2-query/cos.py +python3 ./test.py -f 2-query/tan.py +python3 ./test.py -f 2-query/arcsin.py +python3 ./test.py -f 2-query/arccos.py +python3 ./test.py -f 2-query/arctan.py \ No newline at end of file diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 6813e9b2f7..f4f7c893c4 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -151,8 +151,8 @@ void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t * if (str[pos] > 0 || shellCountPrefixOnes((uint8_t)str[pos]) > 1) break; } - int32_t rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - assert(rc == *size); + taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk *width = taosWcharWidth(wc); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 08fe2cbc32..21fd3d0359 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -118,66 +118,27 @@ int32_t shellRunCommand(char *command) { } } - bool esc = false; - char quote = 0, *cmd = command, *p = command; + char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { - if (esc) { - switch (c) { - case 'n': - c = '\n'; - break; - case 'r': - c = '\r'; - break; - case 't': - c = '\t'; - break; - case 'G': - *p++ = '\\'; - break; - case '\'': - case '"': - if (quote) { - *p++ = '\\'; - } - break; - default: - *p++ = '\\'; - break; - } - *p++ = c; - esc = false; + if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) { + command ++; continue; } - if (c == '\\') { - if (quote != 0 && (*command == '_' || *command == '\\')) { - // DO nothing - } else { - esc = true; - continue; - } - } - if (quote == c) { quote = 0; - } else if (quote == 0 && (c == '\'' || c == '"')) { + } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) { quote = c; - } - - *p++ = c; - if (c == ';' && quote == 0) { - c = *p; - *p = 0; + } else if (c == ';' && quote == 0) { + c = *command; + *command = 0; if (shellRunSingleCommand(cmd) < 0) { return -1; } - *p = c; - p = cmd; + *command = c; + cmd = command; } } - - *p = 0; return shellRunSingleCommand(cmd); } diff --git a/tools/taos-tools b/tools/taos-tools index 59e0ebaf49..2f3dfddd4d 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20 +Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c