diff --git a/cmake/cmake.define b/cmake/cmake.define index e5ef08acb8..f90024e032 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -75,6 +75,14 @@ IF (TD_WINDOWS) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}") ELSE () + IF (${COVER} MATCHES "true") + MESSAGE(STATUS "Test coverage mode, add extra flags") + SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage") + SET(GCC_COVERAGE_LINK_FLAGS "-lgcov --coverage") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") + ENDIF () + IF (${SANITIZER} MATCHES "true") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") diff --git a/include/common/tdata.h b/include/common/tdata.h new file mode 100644 index 0000000000..9c4d33ed02 --- /dev/null +++ b/include/common/tdata.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDATA_H_ +#define _TD_TDATA_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +typedef struct STaosData TDATA, tdata_t; + +typedef enum { + TAOS_META_STABLE_DATA = 0, // super table meta + TAOS_META_TABLE_DATA, // non-super table meta + TAOS_TS_ROW_DATA, // row time-series data + TAOS_TS_COL_DATA, // col time-series data + TAOS_DATA_MAX +} ETaosDataT; + +struct STaosData { + ETaosDataT type; + uint32_t nPayload; + uint8_t *pPayload; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDATA_H_*/ \ No newline at end of file diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index e94be797b9..1f3b787538 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -150,29 +150,6 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition -#ifdef TD_2_0 - -typedef uint64_t TKEY; - -#define TKEY_INVALID UINT64_MAX -#define TKEY_NULL TKEY_INVALID -#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) -#define TKEY_DELETE_FLAG (((TKEY)1) << 62) -#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) - -#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) -#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) -#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) -#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) -#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) - -#define MIN_TS_KEY ((TSKEY)0x8000000000000001) -#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff) - -#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key)) - -#else - // typedef uint64_t TKEY; #define TKEY TSKEY @@ -192,8 +169,6 @@ typedef uint64_t TKEY; #define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key)) -#endif - static FORCE_INLINE TKEY keyToTkey(TSKEY key) { TSKEY lkey = key; if (key > MAX_TS_KEY) { @@ -218,157 +193,6 @@ static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) { } } -#if 0 -// ----------------- Data row structure - -/* A data row, the format is like below: - * |<------------------------------------------------ len ---------------------------------->| - * |<-- Head -->|<--------- flen -------------->| | - * +---------------------+---------------------------------+---------------------------------+ - * | uint16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | sversion | First part | Second part | - * +----------+----------+---------------------------------+---------------------------------+ - * - * NOTE: timestamp in this row structure is TKEY instead of TSKEY - */ -typedef void *SDataRow; - -#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) - -#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 -#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r)) -#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) -#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) -#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) -#define dataRowKey(r) tdGetKey(dataRowTKey(r)) -#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) -#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) -#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) -#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) -#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) - -SDataRow tdNewDataRowFromSchema(STSchema *pSchema); -void tdFreeDataRow(SDataRow row); -void tdInitDataRow(SDataRow row, STSchema *pSchema); -SDataRow tdDataRowDup(SDataRow row); - -// offset here not include dataRow header length -static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, - int32_t offset) { - assert(value != NULL); - int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; - - if (IS_VAR_DATA_TYPE(type)) { - *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value)); - } - dataRowLen(row) += varDataTLen(value); - } else { - if (offset == 0) { - assert(type == TSDB_DATA_TYPE_TIMESTAMP); - TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]); - } else { - memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); - } - } - - return 0; -} - -// offset here not include dataRow header length -static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { - return tdAppendDataColVal(row, value, true, type, offset); -} - -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { - if (IS_VAR_DATA_TYPE(type)) { - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - } else { - return POINTER_SHIFT(row, offset); - } -} - -static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) { - return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset); -} - -static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { - int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; - int8_t type = pSchema->columns[idx].type; - - return tdGetRowDataOfCol(row, type, offset); -} - -static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { - int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; - int8_t type = pSchema->columns[idx].type; - - return isNull(tdGetRowDataOfCol(row, type, offset), type); -} - -static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { - int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; - int8_t type = pSchema->columns[idx].type; - int16_t bytes = pSchema->columns[idx].bytes; - - setNull(tdGetRowDataOfCol(row, type, offset), type, bytes); -} - -static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src, - STSchema *pSrcSchema, int32_t srcIdx) { - int8_t type = pDstSchema->columns[dstIdx].type; - assert(type == pSrcSchema->columns[srcIdx].type); - void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx); - void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx); - - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - *(VarDataOffsetT *)pData = *(VarDataOffsetT *)value; - pData = POINTER_SHIFT(dst, *(VarDataOffsetT *)pData); - value = POINTER_SHIFT(src, *(VarDataOffsetT *)value); - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pSrcSchema->columns[srcIdx].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - *(TSKEY *)pData = tdGetKey(*(TKEY *)value); - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes); - } -} -#endif // ----------------- Data column structure // SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { @@ -398,29 +222,6 @@ void *dataColSetOffset(SDataCol *pCol, int32_t nEle); bool isNEleNull(SDataCol *pCol, int32_t nEle); -#if 0 -// Get the data pointer from a column-wised data -static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) { - if (isAllRowsNull(pCol)) { - return getNullValue(pCol->type); - } - if (IS_VAR_DATA_TYPE(pCol->type)) { - return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - } else { - return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - } -} - -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) { - assert(rows > 0); - - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); - } else { - return TYPE_BYTES[pDataCol->type] * rows; - } -} -#endif typedef struct { col_id_t maxCols; // max number of columns col_id_t numOfCols; // Total number of cols @@ -479,7 +280,8 @@ void tdResetDataCols(SDataCols *pCols); int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, + TDRowVerT maxVer); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| @@ -542,54 +344,6 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ); } -#if 0 -// offset here not include kvRow header length -static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, - int32_t offset) { - assert(value != NULL); - int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; - SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset); - char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row)); - - pColIdx->colId = colId; - pColIdx->offset = kvRowLen(row); // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE - - if (IS_VAR_DATA_TYPE(type)) { - if (isCopyValData) { - memcpy(ptr, value, varDataTLen(value)); - } - kvRowLen(row) += varDataTLen(value); - } else { - if (offset == 0) { - assert(type == TSDB_DATA_TYPE_TIMESTAMP); - TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]); - } else { - memcpy(ptr, value, TYPE_BYTES[type]); - } - kvRowLen(row) += TYPE_BYTES[type]; - } - - return 0; -} -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); } - -static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_t *nIdx) { - while (*nIdx < kvRowNCols(row)) { - SColIdx *pColIdx = kvRowColIdxAt(row, *nIdx); - if (pColIdx->colId == colId) { - ++(*nIdx); - return tdGetKvRowDataOfCol(row, pColIdx->offset); - } else if (pColIdx->colId > colId) { - return NULL; - } else { - ++(*nIdx); - } - } - return NULL; -} -#endif // ----------------- K-V data row builder typedef struct { int16_t tCols; @@ -632,166 +386,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co return 0; } -#if 0 -// ----------------- SMemRow appended with tuple row structure -/* - * |---------|------------------------------------------------- len ---------------------------------->| - * |<-------- Head ------>|<--------- flen -------------->| | - * |---------+---------------------+---------------------------------+---------------------------------+ - * | uint8_t | uint16_t | int16_t | | | - * |---------+----------+----------+---------------------------------+---------------------------------+ - * | flag | len | sversion | First part | Second part | - * +---------+----------+----------+---------------------------------+---------------------------------+ - * - * NOTE: timestamp in this row structure is TKEY instead of TSKEY - */ - -// ----------------- SMemRow appended with extended K-V data row structure -/* |--------------------|------------------------------------------------ len ---------------------------------->| - * |<------------- Head ------------>|<--------- flen -------------->| | - * |--------------------+----------+--------------------------------------------+---------------------------------+ - * | uint8_t | int16_t | uint16_t | int16_t | | | - * |---------+----------+----------+----------+---------------------------------+---------------------------------+ - * | flag | sversion | len | ncols | cols index | data part | - * |---------+----------+----------+----------+---------------------------------+---------------------------------+ - */ - -typedef void *SMemRow; - -#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) -#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) -#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) -#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) -#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) - -#define SMEM_ROW_DATA 0x0U // SDataRow -#define SMEM_ROW_KV 0x01U // SKVRow - -#define KVRatioConvert (0.9f) - -#define memRowType(r) ((*(uint8_t *)(r)) & 0x01) - -#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory -#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) -#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) -#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) -#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) -#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) - -#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag -#define memRowKvBody(r) \ - POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse SKVRow - -#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535 -#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 - -#define memRowDataTLen(r) \ - ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE)) // using uint32_t/int32_t to store the TLen - -#define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE)) - -#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) -#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen - -static FORCE_INLINE char *memRowEnd(SMemRow row) { - if (isDataRow(row)) { - return (char *)dataRowEnd(memRowDataBody(row)); - } else { - return (char *)kvRowEnd(memRowKvBody(row)); - } -} - -#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) -#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) -#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version -#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) -#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) - -#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) -#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) -#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r))) -#define memRowSetTKey(r, k) \ - do { \ - if (isDataRow(r)) { \ - dataRowTKey(memRowDataBody(r)) = (k); \ - } else { \ - kvRowTKey(memRowKvBody(r)) = (k); \ - } \ - } while (0) - -#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) -#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) -#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) -#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) -#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) - -SMemRow tdMemRowDup(SMemRow row); -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull); - -// NOTE: offset here including the header size -static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) { - if (isDataRow(row)) { - return tdGetRowDataOfCol(memRowDataBody(row), colType, offset); - } else { - return tdGetKVRowValOfCol(memRowKvBody(row), colId); - } -} - -/** - * NOTE: - * 1. Applicable to scan columns one by one - * 2. offset here including the header size - */ -static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_t colType, int32_t offset, - int32_t *kvNIdx) { - if (isDataRow(row)) { - return tdGetRowDataOfCol(memRowDataBody(row), colType, offset); - } else { - return tdGetKVRowValOfColEx(memRowKvBody(row), colId, kvNIdx); - } -} - -static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, - int8_t type, int32_t offset) { - if (isDataRow(row)) { - tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset); - } else { - tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset); - } - return 0; -} - -// make sure schema->flen appended for SDataRow -static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value, int8_t colType) { - int32_t len = 0; - if (IS_VAR_DATA_TYPE(colType)) { - len += varDataTLen(value); - if (rowType == SMEM_ROW_KV) { - len += sizeof(SColIdx); - } - } else { - if (rowType == SMEM_ROW_KV) { - len += TYPE_BYTES[colType]; - len += sizeof(SColIdx); - } - } - return len; -} - -typedef struct { - int16_t colId; - uint8_t colType; - char * colVal; -} SColInfo; - -static FORCE_INLINE void setSColInfo(SColInfo *colInfo, int16_t colId, uint8_t colType, char *colVal) { - colInfo->colId = colId; - colInfo->colType = colType; - colInfo->colVal = colVal; -} - -SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2); -#endif #ifdef __cplusplus } diff --git a/include/common/tglobal.h b/include/common/tglobal.h index da5158abb5..97e6491b98 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -121,6 +121,9 @@ extern char tsCompressor[]; extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; +// udf +extern bool tsStartUdfd; + // internal extern int32_t tsTransPullupInterval; extern int32_t tsMqRebalanceInterval; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5b0aaece99..909b7d0877 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -328,6 +328,9 @@ typedef struct { int8_t alterType; int32_t numOfFields; SArray* pFields; + int32_t ttl; + int32_t commentLen; + char* comment; } SMAlterStbReq; int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq); diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 4002db06ea..b6c96bb2d1 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -57,6 +57,7 @@ typedef struct { RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; + void* clientRpc; } SMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); diff --git a/include/common/trow.h b/include/common/trow.h index 0d34c6e49f..4031946ee8 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -30,6 +30,22 @@ extern "C" { #endif +typedef struct { + TSKEY ts; + union { + uint32_t info; + struct { + uint16_t type : 2; + uint16_t del : 1; + uint16_t endian : 1; + uint16_t reserve : 12; + uint16_t sver; + }; + }; + uint32_t len; + char data[]; +} STSRow; + // Target of tdataformat.h: // 1. Row related definition in dataformat.h of 2.0 could be replaced with tdataformat.h of 3.0. // 2. The basic definition in dataformat.h is shared with tdataformat.h of 3.0. @@ -95,8 +111,6 @@ static FORCE_INLINE bool tdValIsNull(TDRowValT valType, const void *val, int32_t #endif } -typedef void *SRow; - typedef struct { TDRowValT valType; void *val; @@ -119,31 +133,6 @@ typedef struct { SKvRowIdx cidx[]; } SKvRow; -typedef struct { - /// timestamp - TSKEY ts; - union { - /// union field for encode and decode - uint32_t info; - struct { - /// row type - uint16_t type : 2; - /// is delete row(0 not delete, 1 delete) - uint16_t del : 1; - /// endian(0 little endian, 1 big endian) - uint16_t endian : 1; - /// reserved for back compatibility - uint16_t reserve : 12; - /// row schema version - uint16_t sver; - }; - }; - /// row total length - uint32_t len; - /// the inline data, maybe a tuple or a k-v tuple - char data[]; -} STSRow; - typedef struct { // basic info int8_t rowType; @@ -165,15 +154,15 @@ typedef struct { #define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) -#define TD_ROW_INFO(r) ((r)->info) -#define TD_ROW_TYPE(r) ((r)->type) -#define TD_ROW_DELETE(r) ((r)->del) -#define TD_ROW_ENDIAN(r) ((r)->endian) -#define TD_ROW_SVER(r) ((r)->sver) -#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow -#define TD_ROW_DATA(r) ((r)->data) -#define TD_ROW_LEN(r) ((r)->len) -#define TD_ROW_KEY(r) ((r)->ts) +#define TD_ROW_INFO(r) ((r)->info) +#define TD_ROW_TYPE(r) ((r)->type) +#define TD_ROW_DELETE(r) ((r)->del) +#define TD_ROW_ENDIAN(r) ((r)->endian) +#define TD_ROW_SVER(r) ((r)->sver) +#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow +#define TD_ROW_DATA(r) ((r)->data) +#define TD_ROW_LEN(r) ((r)->len) +#define TD_ROW_KEY(r) ((r)->ts) // #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) @@ -228,24 +217,19 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap); static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); } -static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); -static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); -static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); -static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, int8_t bitmapMode, bool isMerge); -static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); -static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, - col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); + +int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); +int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); +int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); /** * @brief @@ -264,257 +248,11 @@ static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { // The primary TS key is stored separatedly and is Norm value, thus should minus 1 firstly return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), (--nKvCols) * sizeof(SKvRowIdx)); } -static FORCE_INLINE void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { -#ifdef TD_SUPPORT_BITMAP - switch (rowType) { - case TD_ROW_TP: - return tdGetBitmapAddrTp(pRow, flen); - case TD_ROW_KV: - return tdGetBitmapAddrKv(pRow, nKvCols); - default: - break; - } -#endif - return NULL; -} - -static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { - switch (bitmapMode) { - case 0: - tdSetBitmapValTypeII(pBitmap, colIdx, valType); - break; - case -1: - case 1: - tdSetBitmapValTypeI(pBitmap, colIdx, valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Use 2 bits at default - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param valType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS; - int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); - // set the value and clear other partitions for offset 0 - // *pDestByte |= (valType << 6); - break; - case 1: - *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); - // *pDestByte |= (valType << 4); - break; - case 2: - *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); - // *pDestByte |= (valType << 2); - break; - case 3: - *pDestByte = ((*pDestByte) & 0xFC) | valType; - // *pDestByte |= (valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { - TDRowValT valType = 0; - tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); - if (tdValTypeIsNorm(valType)) { - return true; - } - return false; -} - -static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, - int8_t bitmapMode) { - switch (bitmapMode) { - case 0: - tdGetBitmapValTypeII(pBitmap, colIdx, pValType); - break; - case -1: - case 1: - tdGetBitmapValTypeI(pBitmap, colIdx, pValType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Use 2 bits at default - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param pValType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS; - int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0xC0) >> 6); - break; - case 1: - *pValType = (((*pDestByte) & 0x30) >> 4); - break; - case 2: - *pValType = (((*pDestByte) & 0x0C) >> 2); - break; - case 3: - *pValType = ((*pDestByte) & 0x03); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param valType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); - // set the value and clear other partitions for offset 0 - // *pDestByte |= (valType << 7); - break; - case 1: - *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); - // *pDestByte |= (valType << 6); - break; - case 2: - *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); - // *pDestByte |= (valType << 5); - break; - case 3: - *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); - // *pDestByte |= (valType << 4); - break; - case 4: - *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); - // *pDestByte |= (valType << 3); - break; - case 5: - *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); - // *pDestByte |= (valType << 2); - break; - case 6: - *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); - // *pDestByte |= (valType << 1); - break; - case 7: - *pDestByte = ((*pDestByte) & 0xFE) | valType; - // *pDestByte |= (valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param pValType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0x80) >> 7); - break; - case 1: - *pValType = (((*pDestByte) & 0x40) >> 6); - break; - case 2: - *pValType = (((*pDestByte) & 0x20) >> 5); - break; - case 3: - *pValType = (((*pDestByte) & 0x10) >> 4); - break; - case 4: - *pValType = (((*pDestByte) & 0x08) >> 3); - break; - case 5: - *pValType = (((*pDestByte) & 0x04) >> 2); - break; - case 6: - *pValType = (((*pDestByte) & 0x02) >> 1); - break; - case 7: - *pValType = ((*pDestByte) & 0x01); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} +void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); +int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); +int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); +bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); +int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); // ----------------- Tuple row structure(STpRow) /* @@ -539,482 +277,28 @@ static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t col * */ -/** - * @brief - * - * @param pBuilder - * @param sver schema version - * @return FORCE_INLINE - */ static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { pBuilder->rowType = TD_ROW_TP; // default STpRow pBuilder->sver = sver; } - -/** - * @brief Not recommended to use - * - * @param pBuilder - * @param rowType - * @return FORCE_INLINE - */ -static FORCE_INLINE void tdSRowSetRowType(SRowBuilder *pBuilder, int8_t rowType) { pBuilder->rowType = rowType; } - -/** - * @brief - * - * @param pBuilder - * @param nCols - * @param nBoundCols use -1 if not available - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) { - pBuilder->flen = flen; - pBuilder->nCols = nCols; - pBuilder->nBoundCols = nBoundCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); - if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); - } else { - pBuilder->nBoundBitmaps = 0; - } -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBuilder - * @param nCols - * @param nBoundCols use -1 if not available - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { - pBuilder->flen = flen; - pBuilder->nCols = nCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief To judge row type: STpRow/SKvRow - * - * @param pBuilder - * @param nCols - * @param nBoundCols - * @param flen - * @param allNullLen use -1 if not available - * @param boundNullLen use -1 if not available - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, - int32_t flen, int32_t allNullLen, int32_t boundNullLen) { - if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { - uint32_t tpLen = allNullLen; - uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen; - if (isSelectKVRow(kvLen, tpLen)) { - pBuilder->rowType = TD_ROW_KV; - } else { - pBuilder->rowType = TD_ROW_TP; - } - - } else { - pBuilder->rowType = TD_ROW_TP; - } - - pBuilder->flen = flen; - pBuilder->nCols = nCols; - pBuilder->nBoundCols = nBoundCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); - if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); - } else { - pBuilder->nBoundBitmaps = 0; - } -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief The invoker is responsible for memory alloc/dealloc. - * - * @param pBuilder - * @param pBuf Output buffer of STSRow - */ -static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { - pBuilder->pBuf = (STSRow *)pBuf; - if (!pBuilder->pBuf) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - TD_ROW_SET_INFO(pBuilder->pBuf, 0); - TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); - - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); - - uint32_t len = 0; - switch (pBuilder->rowType) { - case TD_ROW_TP: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); - memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); -#endif - // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; - TD_ROW_SET_LEN(pBuilder->pBuf, len); - TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - break; - case TD_ROW_KV: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); - memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); -#endif - len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + - pBuilder->nBoundBitmaps; // add - TD_ROW_SET_LEN(pBuilder->pBuf, len); - TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief The invoker is responsible for memory alloc/dealloc. - * - * @param pBuilder - * @param pBuf Output buffer of STSRow - */ -static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { - pBuilder->pBuf = (STSRow *)pBuf; - if (!pBuilder->pBuf) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); - - uint32_t len = 0; - switch (pBuilder->rowType) { - case TD_ROW_TP: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); -#endif - break; - case TD_ROW_KV: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); -#endif - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief 由调用方管理存储空间的分配及释放,一次输入多个参数 - * - * @param pBuilder - * @param pBuf - * @param allNullLen - * @param boundNullLen - * @param nCols - * @param nBoundCols - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, - int32_t nCols, int32_t nBoundCols, int32_t flen) { - if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { - return terrno; - } - return tdSRowResetBuf(pBuilder, pBuf); -} - -/** - * @brief - * - * @param pBuilder - */ -static FORCE_INLINE void tdSRowReset(SRowBuilder *pBuilder) { - pBuilder->rowType = TD_ROW_TP; - pBuilder->pBuf = NULL; - pBuilder->nBoundCols = -1; - pBuilder->nCols = -1; - pBuilder->flen = -1; - pBuilder->pBitmap = NULL; -} - -// internal func -static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - offset -= sizeof(TSKEY); - --colIdx; - -#ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } -#endif - - STSRow *row = pBuilder->pBuf; - - // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap - // should be updated simultaneously if Norm val overwrite Null/None cols. - // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. - if (tdValIsNorm(valType, val, colType)) { - // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val)); - } - TD_ROW_LEN(row) += varDataTLen(val); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); - } - } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - // TODO: Null value for new data types imported since 3.0 need to be defined. - const void *nullVal = getNullValue(colType); - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); - } - } -#endif - - return 0; -} - -// internal func -static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, - col_id_t colId) { - if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - offset -= sizeof(SKvRowIdx); - --colIdx; - -#ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } -#endif - - STSRow *row = pBuilder->pBuf; - // No need to store None/Null values. - if (tdValIsNorm(valType, val, colType)) { - // ts key stored in STSRow.ts - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, val, varDataTLen(val)); - } - TD_ROW_LEN(row) += varDataTLen(val); - } else { - memcpy(ptr, val, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - const void *nullVal = getNullValue(colType); - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(ptr, nullVal, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#endif - - return 0; -} - -/** - * @brief exposed func - * - * @param pBuilder - * @param colId start from PRIMARYKEY_TIMESTAMP_COL_ID - * @param colType - * @param valType - * @param val - * @param isCopyVarData - * @param offset - * @param colIdx sorted column index, start from 0 - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, - TDRowValT valType, const void *val, bool isCopyVarData, int32_t offset, - col_id_t colIdx) { - STSRow *pRow = pBuilder->pBuf; - if (!val) { -#ifdef TD_SUPPORT_BITMAP - if (tdValTypeIsNorm(valType)) { - terrno = TSDB_CODE_INVALID_PTR; - return terrno; - } -#else - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; -#endif - } - // TS KEY is stored in STSRow.ts and not included in STSRow.data field. - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - TD_ROW_KEY(pRow) = *(TSKEY *)val; - // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. - return TSDB_CODE_SUCCESS; - } - // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. - if (TD_IS_TP_ROW(pRow)) { - tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); - } else { - tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); - } - return TSDB_CODE_SUCCESS; -} - -// internal -static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, - int32_t offset, int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - } -#else - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { - if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { - return 1; - } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { - return -1; - } else { - return 0; - } -} -// internal -static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, - int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow) - 1); - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { - if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; - output->valType = TD_VTYPE_NONE; - return terrno; - } - output->val = POINTER_SHIFT(pRow, offset); - } -#else - TASSERT(0); - if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; - output->valType = TD_VTYPE_NONE; - return terrno; - } - output->val = POINTER_SHIFT(pRow, offset); - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - return TSDB_CODE_SUCCESS; -} +int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); +int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); +int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, + int32_t allNullLen, int32_t boundNullLen); +int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, + int32_t nBoundCols, int32_t flen); +void tdSRowReset(SRowBuilder *pBuilder); +int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset); +int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); +int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val, + bool isCopyVarData, int32_t offset, col_id_t colIdx); +int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, + int16_t colIdx); +int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx); typedef struct { STSchema *pSchema; @@ -1026,448 +310,20 @@ typedef struct { col_id_t kvIdx; // [0, nKvCols) } STSRowIter; -static FORCE_INLINE void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { - pIter->pRow = pRow; - pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); - pIter->offset = 0; - pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; - pIter->kvIdx = 0; -} - -static FORCE_INLINE void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { - pIter->pSchema = pSchema; - pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; -} - -static int32_t tdCompareColId(const void *arg1, const void *arg2) { - int32_t colId = *(int32_t *)arg1; - STColumn *pCol = (STColumn *)arg2; - - if (colId < pCol->colId) { - return -1; - } else if (colId == pCol->colId) { - return 0; - } else { - return 1; - } -} - -/** - * @brief STSRow method to get value of specified colId/colType by bsearch - * - * @param pIter - * @param colId Start from PRIMARYKEY_TIMESTAMP_COL_ID(1) - * @param colType - * @param pVal - * @return true Not reach end and pVal is set(None/Null/Norm). - * @return false Reach end and pVal not set. - */ -static FORCE_INLINE bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - STSRow *pRow = pIter->pRow; - int16_t colIdx = -1; - if (TD_IS_TP_ROW(pRow)) { - STSchema *pSchema = pIter->pSchema; - STColumn *pCol = - (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); - if (!pCol) { - pVal->valType = TD_VTYPE_NONE; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - return true; - } -#ifdef TD_SUPPORT_BITMAP - colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); -#endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); - } else if (TD_IS_KV_ROW(pRow)) { - SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), - compareKvRowColId, TD_EQ); -#ifdef TD_SUPPORT_BITMAP - if (pIdx) { - colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); - } -#endif - tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); - } else { - if (COL_REACH_END(colId, pIter->maxColId)) return false; - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - -// internal -static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - - return true; -} - -// internal -static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, - SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - SKvRowIdx *pKvIdx = NULL; - bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; - while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); - if (pKvIdx->colId == colId) { - ++(*nIdx); - pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); - colFound = true; - break; - } else if (pKvIdx->colId > colId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - ++(*nIdx); - } - } - - if (!colFound) { - if (colId <= pIter->maxColId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - return false; - } - } - -#ifdef TD_SUPPORT_BITMAP - int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); - if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - - return true; -} - -/** - * @brief STSRow Iter to get value from colId 1 to maxColId ascendingly - * - * @param pIter - * @param pVal - * @param colId - * @param colType - * @param pVal output - * @return true Not reach end and pVal is set(None/Null/Norm). - * @return false Reach end of row and pVal not set. - */ -static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - if (TD_IS_TP_ROW(pIter->pRow)) { - STColumn *pCol = NULL; - STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx < pSchema->numOfCols) { - pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key - if (colId == pCol->colId) { - break; - } else if (pCol->colId < colId) { - ++pIter->colIdx; - continue; - } else { - return false; - } - } - tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); - ++pIter->colIdx; - } else if (TD_IS_KV_ROW(pIter->pRow)) { - return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); - } else { - pVal->valType = TD_VTYPE_NONE; - terrno = TSDB_CODE_INVALID_PARA; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - } - return true; -} - +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2); - -// Get the data pointer from a column-wised data -static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { - if (isAllRowsNone(pCol)) { - pVal->valType = TD_VTYPE_NULL; -#ifdef TD_SUPPORT_READ2 - pVal->val = (void *)getNullValue(pCol->type); -#else - pVal->val = NULL; -#endif - return TSDB_CODE_SUCCESS; - } - - if (TD_COL_ROWS_NORM(pCol)) { - pVal->valType = TD_VTYPE_NORM; - } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) { - return terrno; - } - - if (tdValTypeIsNorm(pVal->valType)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - } else { - pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - } - } else { - pVal->valType = TD_VTYPE_NULL; -#ifdef TD_SUPPORT_READ2 - pVal->val = (void *)getNullValue(pCol->type); -#else - pVal->val = NULL; -#endif - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pRow - * @param colId - * @param colType - * @param flen - * @param offset - * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE - */ -static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, - col_id_t colIdx, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); - return true; - } - void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); - return true; -} - -/** - * @brief - * - * @param pRow - * @param colId - * @param offset - * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE - */ -static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, - SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); - return true; - } - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx); - return true; -} - -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) { - ASSERT(rows > 0); - int32_t result = 0; - - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - result += pDataCol->dataOff[rows - 1]; - SCellVal val = {0}; - if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) { - TASSERT(0); - } - - // Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4 - result += varDataTLen(val.val); - // TODO: later on, don't save Null/None for VarDataT for 3.0 - // if (tdValTypeIsNorm(val.valType)) { - // result += varDataTLen(val.val); - // } - } else { - result += TYPE_BYTES[pDataCol->type] * rows; - } - - ASSERT(pDataCol->len == result); - - return result; -} - -static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { - if (tdValTypeIsNull(pVal->valType)) { - printf("NULL "); - return; - } else if (tdValTypeIsNone(pVal->valType)) { - printf("NONE "); - return; - } - switch (colType) { - case TSDB_DATA_TYPE_BOOL: - printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%" PRIi8 " ", *(int8_t *)pVal->val); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRIi16 " ", *(int16_t *)pVal->val); - break; - case TSDB_DATA_TYPE_INT: - printf("%" PRIi32 " ", *(int32_t *)pVal->val); - break; - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRIi64 " ", *(int64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f ", *(float *)pVal->val); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf ", *(double *)pVal->val); - break; - case TSDB_DATA_TYPE_VARCHAR: - printf("VARCHAR "); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRIi64 " ", *(int64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_NCHAR: - printf("NCHAR "); - break; - case TSDB_DATA_TYPE_UTINYINT: - printf("%" PRIu8 " ", *(uint8_t *)pVal->val); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf("%" PRIu16 " ", *(uint16_t *)pVal->val); - break; - case TSDB_DATA_TYPE_UINT: - printf("%" PRIu32 " ", *(uint32_t *)pVal->val); - break; - case TSDB_DATA_TYPE_UBIGINT: - printf("%" PRIu64 " ", *(uint64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_JSON: - printf("JSON "); - break; - case TSDB_DATA_TYPE_VARBINARY: - printf("VARBIN "); - break; - case TSDB_DATA_TYPE_DECIMAL: - printf("DECIMAL "); - break; - case TSDB_DATA_TYPE_BLOB: - printf("BLOB "); - break; - case TSDB_DATA_TYPE_MEDIUMBLOB: - printf("MedBLOB "); - break; - // case TSDB_DATA_TYPE_BINARY: - // printf("BINARY "); - // break; - case TSDB_DATA_TYPE_MAX: - printf("UNDEF "); - break; - default: - printf("UNDEF "); - break; - } -} - -static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pSchema); - tdSTSRowIterReset(&iter, row); - printf("%s >>>", tag); - for (int i = 0; i < pSchema->numOfCols; ++i) { - STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {255, NULL}; - if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { - break; - } - ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); - tdSCellValPrint(&sVal, stCol->type); - } - printf("\n"); -} -#ifdef TROW_ORIGIN_HZ -typedef struct { - uint32_t nRows; - char rows[]; -} STSRowBatch; - -static void tdSRowPrint(STSRow *row) { - printf("type:%d, del:%d, sver:%d\n", row->type, row->del, row->sver); - printf("isDeleted:%s, isTpRow:%s, isKvRow:%s\n", TD_BOOL_STR(TD_ROW_IS_DELETED(row)), TD_BOOL_STR(TD_IS_TP_ROW(row)), - TD_BOOL_STR(TD_IS_KV_ROW(row))); -} - -typedef enum { - /// tuple row builder - TD_TP_ROW_BUILDER = 0, - /// kv row builder - TD_KV_ROW_BUILDER, - /// self-determined row builder - TD_SD_ROW_BUILDER -} ERowBbuilderT; - -typedef struct { - /// row builder type - ERowBbuilderT type; - /// buffer writer - SBufferWriter bw; - /// target row - STSRow *pRow; -} STSRowBuilder; - -typedef struct { - STSchema *pSchema; - STSRow *pRow; -} STSRowReader; - -typedef struct { - uint32_t it; - STSRowBatch *pRowBatch; -} STSRowBatchIter; - -// STSRowBuilder -#define trbInit(rt, allocator, endian, target, size) \ - { .type = (rt), .bw = tbufInitWriter(allocator, endian), .pRow = (target) } -void trbSetRowInfo(STSRowBuilder *pRB, bool del, uint16_t sver); -void trbSetRowVersion(STSRowBuilder *pRB, uint64_t ver); -void trbSetRowTS(STSRowBuilder *pRB, TSKEY ts); -int32_t trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid); - -// STSRowReader -#define tRowReaderInit(schema, row) \ - { .schema = (schema), .row = (row) } -int32_t tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size); - -// STSRowBatchIter -#define tRowBatchIterInit(pRB) \ - { .it = 0, .pRowBatch = (pRB) } -const STSRow *tRowBatchIterNext(STSRowBatchIter *pRowBatchIter); -#endif +int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode); +bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, + SCellVal *pVal); +bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal); +int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode); +void tdSCellValPrint(SCellVal *pVal, int8_t colType); +void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); #ifdef __cplusplus } diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index e0fce2aa6a..162b6fb2ed 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -22,6 +22,7 @@ extern "C" { #include "query.h" #include "tcommon.h" +#include "tmsgcb.h" typedef void* qTaskInfo_t; typedef void* DataSinkHandle; @@ -29,11 +30,12 @@ struct SRpcMsg; struct SSubplan; typedef struct SReadHandle { - void* reader; - void* meta; - void* config; - void* vnode; - void* mnd; + void* reader; + void* meta; + void* config; + void* vnode; + void* mnd; + SMsgCb* pMsgCb; } SReadHandle; #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1 diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 4a37283ee5..1cc4d9839e 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -31,13 +31,13 @@ typedef enum EFunctionType { FUNCTION_TYPE_ELAPSED, FUNCTION_TYPE_IRATE, FUNCTION_TYPE_LAST_ROW, - FUNCTION_TYPE_LEASTSQUARES, FUNCTION_TYPE_MAX, FUNCTION_TYPE_MIN, FUNCTION_TYPE_MODE, FUNCTION_TYPE_PERCENTILE, FUNCTION_TYPE_SPREAD, FUNCTION_TYPE_STDDEV, + FUNCTION_TYPE_LEASTSQUARES, FUNCTION_TYPE_SUM, FUNCTION_TYPE_TWA, FUNCTION_TYPE_HISTOGRAM, @@ -154,6 +154,7 @@ bool fmIsWindowClauseFunc(int32_t funcId); bool fmIsSpecialDataRequiredFunc(int32_t funcId); bool fmIsDynamicScanOptimizedFunc(int32_t funcId); bool fmIsMultiResFunc(int32_t funcId); +bool fmIsRepeatScanFunc(int32_t funcId); bool fmIsUserDefinedFunc(int32_t funcId); typedef enum EFuncDataRequired { @@ -170,7 +171,7 @@ int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet); -bool fmIsInvertible(int32_t funcId); +bool fmIsInvertible(int32_t funcId); #ifdef __cplusplus } diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index ad8a472d08..05c9da1a8a 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -179,6 +179,7 @@ typedef enum ENodeType { QUERY_NODE_KILL_CONNECTION_STMT, QUERY_NODE_KILL_QUERY_STMT, QUERY_NODE_KILL_TRANSACTION_STMT, + QUERY_NODE_QUERY, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN, @@ -206,6 +207,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index a756217c7a..d0d10b2761 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -274,6 +274,8 @@ typedef struct SIntervalPhysiNode { int8_t slidingUnit; } SIntervalPhysiNode; +typedef SIntervalPhysiNode SStreamIntervalPhysiNode; + typedef struct SFillPhysiNode { SPhysiNode node; EFillMode mode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 91c80f6cf5..c0e2ec91e0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -233,6 +233,7 @@ typedef struct SSelectStmt { uint8_t precision; bool isEmptyResult; bool hasAggFuncs; + bool hasRepeatScanFuncs; bool isTimeOrderQuery; } SSelectStmt; @@ -294,6 +295,37 @@ typedef struct SExplainStmt { SNode* pQuery; } SExplainStmt; +typedef struct SCmdMsgInfo { + int16_t msgType; + SEpSet epSet; + void* pMsg; + int32_t msgLen; + void* pExtension; // todo remove it soon +} SCmdMsgInfo; + +typedef enum EQueryExecMode { + QUERY_EXEC_MODE_LOCAL = 1, + QUERY_EXEC_MODE_RPC, + QUERY_EXEC_MODE_SCHEDULE, + QUERY_EXEC_MODE_EMPTY_RESULT +} EQueryExecMode; + +typedef struct SQuery { + ENodeType type; + EQueryExecMode execMode; + bool haveResultSet; + SNode* pRoot; + int32_t numOfResCols; + SSchema* pResSchema; + int8_t precision; + SCmdMsgInfo* pCmdMsg; + int32_t msgType; + SArray* pDbList; + SArray* pTableList; + bool showRewrite; + int32_t placeholderNum; +} SQuery; + void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 998d45aee1..a8bcac1c42 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -48,36 +48,6 @@ typedef struct SParseContext { bool isSuperUser; } SParseContext; -typedef struct SCmdMsgInfo { - int16_t msgType; - SEpSet epSet; - void* pMsg; - int32_t msgLen; - void* pExtension; // todo remove it soon -} SCmdMsgInfo; - -typedef enum EQueryExecMode { - QUERY_EXEC_MODE_LOCAL = 1, - QUERY_EXEC_MODE_RPC, - QUERY_EXEC_MODE_SCHEDULE, - QUERY_EXEC_MODE_EMPTY_RESULT -} EQueryExecMode; - -typedef struct SQuery { - EQueryExecMode execMode; - bool haveResultSet; - SNode* pRoot; - int32_t numOfResCols; - SSchema* pResSchema; - int8_t precision; - SCmdMsgInfo* pCmdMsg; - int32_t msgType; - SArray* pDbList; - SArray* pTableList; - bool showRewrite; - int32_t placeholderNum; -} SQuery; - int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); bool isInsertSql(const char* pStr, size_t length); @@ -103,9 +73,10 @@ void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen); -void* smlInitHandle(SQuery *pQuery); -void smlDestroyHandle(void *pHandle); -int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SArray *colsSchema, SArray *cols, bool format, STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen); +void* smlInitHandle(SQuery* pQuery); +void smlDestroyHandle(void* pHandle); +int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* colsSchema, SArray* cols, bool format, + STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c1e53fa805..c390f67153 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -24,6 +24,7 @@ extern "C" { #include "thash.h" #include "tlog.h" #include "tmsg.h" +#include "tmsgcb.h" typedef enum { JOB_TASK_STATUS_NULL = 0, @@ -149,7 +150,8 @@ int32_t cleanupTaskQueue(); */ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); -int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *ctx); +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, + bool persistHandle, void* ctx); /** * Asynchronously send message to server, after the response received, the callback will be incured. @@ -182,7 +184,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE -#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) +#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) #define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6a57b8d53e..52b2f0c670 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -88,6 +88,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0114) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0115) #define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0116) +#define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0117) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0140) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0141) @@ -632,6 +633,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN TAOS_DEF_ERROR_CODE(0, 0x2642) #define TSDB_CODE_PAR_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x2643) #define TSDB_CODE_PAR_PERMISSION_DENIED TAOS_DEF_ERROR_CODE(0, 0x2644) +#define TSDB_CODE_PAR_INVALID_STREAM_QUERY TAOS_DEF_ERROR_CODE(0, 0x2645) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h deleted file mode 100644 index 83e6ad0868..0000000000 --- a/include/util/tskiplist2.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_UTIL_SKIPLIST2_H_ -#define _TD_UTIL_SKIPLIST2_H_ - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define SL_MAX_LEVEL 15 - -typedef struct SSkipList2 SSkipList2; -typedef struct SSLCursor SSLCursor; -typedef struct SSLCfg SSLCfg; -typedef struct SSLNode SSLNode; - -typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); - -// SSkipList2 -int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); -int32_t slClose(SSkipList2 *pSl); -int32_t slClear(SSkipList2 *pSl); - -// SSLCursor -int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); -int32_t slcClose(SSLCursor *pSlc); -int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey); -int32_t slcMoveToNext(SSLCursor *pSlc); -int32_t slcMoveToPrev(SSLCursor *pSlc); -int32_t slcMoveToFirst(SSLCursor *pSlc); -int32_t slcMoveToLast(SSLCursor *pSlc); -int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData); -int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData); -int32_t slcDrop(SSLCursor *pSlc); - -// struct -struct SSLCfg { - int8_t maxLevel; - int32_t nKey; - int32_t nData; - tslCmprFn cmprFn; - void *pPool; - void *(*xMalloc)(void *, int32_t size); - void (*xFree)(void *, void *); -}; - -struct SSLCursor { - SSkipList2 *pSl; - SSLNode **forwards[SL_MAX_LEVEL]; -}; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_SKIPLIST2_H_*/ \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ce5b101b4a..57d4031835 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -254,7 +254,7 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void* createTscObj(const char* user, const char* auth, const char* db, int32_t connType, SAppInstInfo* pAppInfo); void destroyTscObj(void* pObj); STscObj* acquireTscObj(int64_t rid); int32_t releaseTscObj(int64_t rid); @@ -310,6 +310,7 @@ void hbMgrInitMqHbRspHandle(); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); +int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); #ifdef __cplusplus } diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 43e886faf5..32da75fb1e 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -60,6 +60,7 @@ typedef struct SStmtBindInfo { int32_t sBindRowNum; int32_t sBindLastIdx; int8_t tbType; + bool tagsCached; void* boundTags; char* tbName; SName sname; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 596c3d3fbf..819c50275b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -131,7 +131,7 @@ void destroyTscObj(void *pObj) { taosMemoryFreeClear(pTscObj); } -void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { +void *createTscObj(const char *user, const char *auth, const char *db, int32_t connType, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -145,6 +145,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI return NULL; } + pObj->connType = connType; pObj->pAppInfo = pAppInfo; tstrncpy(pObj->user, user, sizeof(pObj->user)); memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 97c7d2bad1..53aebe751b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,7 +25,7 @@ #include "tref.h" static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); -static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType); +static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static bool stringLengthCheck(const char* str, size_t maxsize) { @@ -491,7 +491,7 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, SAppInstInfo* pAppInfo, int connType) { - STscObj* pTscObj = createTscObj(user, auth, db, pAppInfo); + STscObj* pTscObj = createTscObj(user, auth, db, connType, pAppInfo); if (NULL == pTscObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return pTscObj; @@ -504,7 +504,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t return NULL; } - SMsgSendInfo* body = buildConnectMsg(pRequest, connType); + SMsgSendInfo* body = buildConnectMsg(pRequest); int64_t transporterId = 0; asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); @@ -527,7 +527,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t return pTscObj; } -static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) { +static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -550,9 +550,10 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) { } taosMemoryFreeClear(db); - connectReq.connType = connType; - connectReq.pid = htonl(appInfo.pid); + connectReq.connType = pObj->connType; + connectReq.pid = htonl(appInfo.pid); connectReq.startTime = htobe64(appInfo.startTime); + tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user)); tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd)); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 9d75586917..ae6aee13d5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -563,7 +563,11 @@ const char *taos_get_server_info(TAOS *taos) { } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { - // TODO + if (taos == NULL || sql == NULL) { + // todo directly call fp + } + + taos_query_l(taos, sql, (int32_t) strlen(sql)); } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 15d1500860..59a7cbee1a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -405,7 +405,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_MND_INVALID_STB) { + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 3c94de1cef..c804213d89 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -29,6 +29,11 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) { code = TSDB_CODE_TSC_STMT_API_ERROR; } +/* + if ((pStmt->sql.type == STMT_TYPE_MULTI_INSERT) && ()) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } +*/ break; case STMT_BIND_COL: if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) { @@ -123,6 +128,7 @@ int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.boundTags = tags; + pStmt->bInfo.tagsCached = false; return TSDB_CODE_SUCCESS; } @@ -207,8 +213,6 @@ int32_t stmtParseSql(STscStmt* pStmt) { STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR); } - STMT_ERR_RET(stmtCacheBlock(pStmt)); - return TSDB_CODE_SUCCESS; } @@ -219,8 +223,10 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.needParse = true; taosMemoryFreeClear(pStmt->bInfo.tbName); - destroyBoundColumnInfo(pStmt->bInfo.boundTags); - taosMemoryFreeClear(pStmt->bInfo.boundTags); + if (!pStmt->bInfo.tagsCached) { + destroyBoundColumnInfo(pStmt->bInfo.boundTags); + taosMemoryFreeClear(pStmt->bInfo.boundTags); + } return TSDB_CODE_SUCCESS; } @@ -275,6 +281,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { qDestroyStmtDataBlock(pCache->pDataBlock); destroyBoundColumnInfo(pCache->boundTags); + taosMemoryFreeClear(pCache->boundTags); pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); } @@ -302,7 +309,15 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { STableMeta *pTableMeta = NULL; SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); + int32_t code = catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta); + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + + return TSDB_CODE_SUCCESS; + } + + STMT_ERR_RET(code); + uint64_t uid = pTableMeta->uid; uint64_t suid = pTableMeta->suid; int8_t tableType = pTableMeta->tableType; @@ -328,6 +343,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.tbSuid = suid; pStmt->bInfo.tbType = tableType; pStmt->bInfo.boundTags = pCache->boundTags; + pStmt->bInfo.tagsCached = true; return TSDB_CODE_SUCCESS; } @@ -340,6 +356,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.tbSuid = suid; pStmt->bInfo.tbType = tableType; pStmt->bInfo.boundTags = pCache->boundTags; + pStmt->bInfo.tagsCached = true; STableDataBlocks* pNewBlock = NULL; STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock)); @@ -448,10 +465,12 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); - if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + if (!pStmt->bInfo.needParse) { + return TSDB_CODE_SUCCESS; } + STMT_ERR_RET(stmtParseSql(pStmt)); + STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); if (NULL == pDataBlock) { tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); @@ -501,8 +520,6 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); - if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { pStmt->bInfo.needParse = false; } @@ -520,6 +537,8 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) { STMT_ERR_RET(stmtParseSql(pStmt)); } + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); + if (STMT_TYPE_QUERY == pStmt->sql.type) { if (NULL == pStmt->sql.pQueryPlan) { STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); @@ -586,6 +605,16 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); } + + if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { + code = refreshMeta(pStmt->exec.pRequest->pTscObj, pStmt->exec.pRequest); + if (code) { + pStmt->exec.pRequest->code = code; + } else { + STMT_ERR_RET(stmtResetStmt(pStmt)); + STMT_ERR_RET(TSDB_CODE_NEED_RETRY); + } + } STMT_ERR_JRET(pStmt->exec.pRequest->code); @@ -613,13 +642,11 @@ int stmtClose(TAOS_STMT *stmt) { const char *stmtErrstr(TAOS_STMT *stmt) { STscStmt* pStmt = (STscStmt*)stmt; - if (stmt == NULL) { + if (stmt == NULL || NULL == pStmt->exec.pRequest) { return (char*) tstrerror(terrno); } - if (pStmt->exec.pRequest) { - pStmt->exec.pRequest->code = terrno; - } + pStmt->exec.pRequest->code = terrno; return taos_errstr(pStmt->exec.pRequest); } diff --git a/source/common/src/tdata.c b/source/common/src/tdata.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/common/src/tdata.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0d0bbb07be..b58e4bd1dd 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -493,7 +493,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { bool isNull = false; if (pBlock->pBlockAgg == NULL) { - isNull = colDataIsNull_s(pColData, pBlock->info.rows); + isNull = colDataIsNull_s(pColData, j); } else { isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index e659b9ec04..5d893fe398 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -129,50 +129,6 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { return buf; } -#if 0 -int32_t tEncodeSTColumn(SCoder *pEncoder, const STColumn *pCol) { - if (tEncodeI16(pEncoder, pCol->colId) < 0) return -1; - if (tEncodeI8(pEncoder, pCol->type) < 0) return -1; - if (tEncodeI8(pEncoder, pCol->sma) < 0) return -1; - if (tEncodeI32(pEncoder, pCol->bytes) < 0) return -1; - if (tEncodeI32(pEncoder, pCol->offset) < 0) return -1; - return pEncoder->pos; -} - -int32_t tDecodeSTColumn(SCoder *pDecoder, STColumn *pCol) { - if (tDecodeI16(pDecoder, &pCol->colId) < 0) return -1; - if (tDecodeI8(pDecoder, &pCol->type) < 0) return -1; - if (tDecodeI8(pDecoder, &pCol->sma) < 0) return -1; - if (tDecodeI32(pDecoder, &pCol->bytes) < 0) return -1; - if (tDecodeI32(pDecoder, &pCol->offset) < 0) return -1; - return 0; -} - -int32_t tEncodeSchema(SCoder *pEncoder, const STSchema *pSchema) { - if (tEncodeI32(pEncoder, pSchema->numOfCols) < 0) return -1; - if (tEncodeI16(pEncoder, pSchema->version) < 0) return -1; - if (tEncodeU16(pEncoder, pSchema->flen) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->vlen) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->tlen) < 0) return -1; - - for (int32_t i = 0; i < schemaNCols(pSchema); i++) { - const STColumn *pCol = schemaColAt(pSchema, i); - if (tEncodeSTColumn(pEncoder, pCol) < 0) return -1; - } - return 0; -} - -int32_t tDecodeSchema(SCoder *pDecoder, STSchema *pSchema) { - if (tDecodeI32(pDecoder, &pSchema->numOfCols) < 0) return -1; - if (tDecodeI16(pDecoder, &pSchema->version) < 0) return -1; - if (tDecodeU16(pDecoder, &pSchema->flen) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->vlen) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->tlen) < 0) return -1; - - return 0; -} -#endif - int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { if (pBuilder == NULL) return -1; @@ -260,49 +216,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { return pSchema; } -#if 0 -/** - * Initialize a data row - */ -void tdInitDataRow(SDataRow row, STSchema *pSchema) { - dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema)); - dataRowSetVersion(row, schemaVersion(pSchema)); -} - -SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { - int32_t size = dataRowMaxBytesFromSchema(pSchema); - - SDataRow row = taosMemoryMalloc(size); - if (row == NULL) return NULL; - - tdInitDataRow(row, pSchema); - return row; -} - -/** - * Free the SDataRow object - */ -void tdFreeDataRow(SDataRow row) { - if (row) taosMemoryFree(row); -} - -SDataRow tdDataRowDup(SDataRow row) { - SDataRow trow = taosMemoryMalloc(dataRowLen(row)); - if (trow == NULL) return NULL; - - dataRowCpy(trow, row); - return trow; -} - -SMemRow tdMemRowDup(SMemRow row) { - SMemRow trow = taosMemoryMalloc(memRowTLen(row)); - if (trow == NULL) return NULL; - - memRowCpy(trow, row); - return trow; -} -#endif - void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); @@ -312,39 +225,6 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->len = 0; } -#if 0 -// value from timestamp should be TKEY here instead of TSKEY -int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { - ASSERT(pCol != NULL && value != NULL); - - if (isAllRowsNull(pCol)) { - if (isNull(value, pCol->type)) { - // all null value yet, just return - return 0; - } - - if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1; - if (numOfRows > 0) { - // Find the first not null value, fill all previouse values as NULL - dataColSetNEleNull(pCol, numOfRows); - } - } - - if (IS_VAR_DATA_TYPE(pCol->type)) { - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); - // Update the length - pCol->len += varDataTLen(value); - } else { - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); - pCol->len += pCol->bytes; - } - return 0; -} -#endif static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) { if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); @@ -361,31 +241,6 @@ bool isNEleNull(SDataCol *pCol, int nEle) { return true; } -#if 0 -static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pCol->dataOff[index] = pCol->len; - char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); - setVardataNull(ptr, pCol->type); - pCol->len += varDataTLen(ptr); - } else { - setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes); - pCol->len += TYPE_BYTES[pCol->type]; - } -} - -static void dataColSetNEleNull(SDataCol *pCol, int nEle, int8_t bitmapMode) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pCol->len = 0; - for (int i = 0; i < nEle; ++i) { - dataColSetNullAt(pCol, i); - } - } else { - setNullN(pCol->pData, pCol->type, pCol->bytes, nEle); - pCol->len = TYPE_BYTES[pCol->type] * nEle; - } -} -#endif void *dataColSetOffset(SDataCol *pCol, int nEle) { ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR))); @@ -483,42 +338,6 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { return NULL; } -#if 0 -SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { - SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); - if (pRet == NULL) return NULL; - - pRet->numOfCols = pDataCols->numOfCols; - pRet->sversion = pDataCols->sversion; - if (keepData) pRet->numOfRows = pDataCols->numOfRows; - - for (int i = 0; i < pDataCols->numOfCols; i++) { - pRet->cols[i].type = pDataCols->cols[i].type; - pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; - pRet->cols[i].colId = pDataCols->cols[i].colId; - pRet->cols[i].bytes = pDataCols->cols[i].bytes; - pRet->cols[i].offset = pDataCols->cols[i].offset; - - if (keepData) { - if (pDataCols->cols[i].len > 0) { - if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { - tdFreeDataCols(pRet); - return NULL; - } - pRet->cols[i].len = pDataCols->cols[i].len; - memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); - if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { - int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints; - memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); - } - } - } - } - - return pRet; -} -#endif - void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { pCols->numOfRows = 0; @@ -528,180 +347,6 @@ void tdResetDataCols(SDataCols *pCols) { } } } -#if 0 -static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { - ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); - - int rcol = 0; - int dcol = 0; - - while (dcol < pCols->numOfCols) { - bool setCol = 0; - SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - dcol++; - continue; - } - - STColumn *pRowCol = schemaColAt(pSchema, rcol); - if (pRowCol->colId == pDataCol->colId) { - void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE); - if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - dcol++; - rcol++; - } else if (pRowCol->colId < pDataCol->colId) { - rcol++; - } else { - if(forceSetNull || setCol) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - } - dcol++; - } - } - pCols->numOfRows++; -} - -static void tdAppendKVRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { - ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row)); - - int rcol = 0; - int dcol = 0; - - int nRowCols = kvRowNCols(row); - - while (dcol < pCols->numOfCols) { - bool setCol = 0; - SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - ++dcol; - continue; - } - - SColIdx *colIdx = kvRowColIdxAt(row, rcol); - - if (colIdx->colId == pDataCol->colId) { - void *value = tdGetKvRowDataOfCol(row, colIdx->offset); - if(!isNull(value, pDataCol->type)) setCol = 1; - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - ++dcol; - ++rcol; - } else if (colIdx->colId < pDataCol->colId) { - ++rcol; - } else { - if(forceSetNull || setCol) { - dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints); - } - ++dcol; - } - } - pCols->numOfRows++; -} - -void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) { - if (isDataRow(row)) { - tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull); - } else if (isKvRow(row)) { - tdAppendKVRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull); - } else { - ASSERT(0); - } -} - -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) { - ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); - ASSERT(target->numOfCols == source->numOfCols); - int offset = 0; - - if (pOffset == NULL) { - pOffset = &offset; - } - - SDataCols *pTarget = NULL; - - if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap - ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); - for (int i = 0; i < rowsToMerge; i++) { - for (int j = 0; j < source->numOfCols; j++) { - if (source->cols[j].len > 0 || target->cols[j].len > 0) { - dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i + (*pOffset)), target->numOfRows, - target->maxPoints); - } - } - target->numOfRows++; - } - (*pOffset) += rowsToMerge; - } else { - pTarget = tdDupDataCols(target, true); - if (pTarget == NULL) goto _err; - - int iter1 = 0; - tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows, - pTarget->numOfRows + rowsToMerge, forceSetNull); - } - - tdFreeDataCols(pTarget); - return 0; - -_err: - tdFreeDataCols(pTarget); - return -1; -} - -// src2 data has more priority than src1 -static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull) { - tdResetDataCols(target); - ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); - - while (target->numOfRows < tRows) { - if (*iter1 >= limit1 && *iter2 >= limit2) break; - - TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); - TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); - TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); - TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); - - ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - - if (key1 < key2) { - for (int i = 0; i < src1->numOfCols; i++) { - ASSERT(target->cols[i].type == src1->cols[i].type); - if (src1->cols[i].len > 0 || target->cols[i].len > 0) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, - target->maxPoints); - } - } - - target->numOfRows++; - (*iter1)++; - } else if (key1 >= key2) { - if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { - for (int i = 0; i < src2->numOfCols; i++) { - ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, - target->maxPoints); - } else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, - target->maxPoints); - } else if(target->cols[i].len > 0) { - dataColSetNullAt(&target->cols[i], target->numOfRows); - } - } - target->numOfRows++; - } - - (*iter2)++; - if (key1 == key2) (*iter1)++; - } - - ASSERT(target->numOfRows <= target->maxPoints); - } -} -#endif SKVRow tdKVRowDup(SKVRow row) { SKVRow trow = taosMemoryMalloc(kvRowLen(row)); @@ -859,98 +504,3 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { return row; } -#if 0 -SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { -#if 0 - ASSERT(memRowKey(row1) == memRowKey(row2)); - ASSERT(schemaVersion(pSchema1) == memRowVersion(row1)); - ASSERT(schemaVersion(pSchema2) == memRowVersion(row2)); - ASSERT(schemaVersion(pSchema1) >= schemaVersion(pSchema2)); -#endif - - SArray *stashRow = taosArrayInit(pSchema1->numOfCols, sizeof(SColInfo)); - if (stashRow == NULL) { - return NULL; - } - - SMemRow pRow = buffer; - SDataRow dataRow = memRowDataBody(pRow); - memRowSetType(pRow, SMEM_ROW_DATA); - dataRowSetVersion(dataRow, schemaVersion(pSchema1)); // use latest schema version - dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pSchema1->flen)); - - TDRowLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE; - - int32_t i = 0; // row1 - int32_t j = 0; // row2 - int32_t nCols1 = schemaNCols(pSchema1); - int32_t nCols2 = schemaNCols(pSchema2); - SColInfo colInfo = {0}; - int32_t kvIdx1 = 0, kvIdx2 = 0; - - while (i < nCols1) { - STColumn *pCol = schemaColAt(pSchema1, i); - void * val1 = tdGetMemRowDataOfColEx(row1, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx1); - // if val1 != NULL, use val1; - if (val1 != NULL && !isNull(val1, pCol->type)) { - tdAppendColVal(dataRow, val1, pCol->type, pCol->offset); - kvLen += tdGetColAppendLen(SMEM_ROW_KV, val1, pCol->type); - setSColInfo(&colInfo, pCol->colId, pCol->type, val1); - taosArrayPush(stashRow, &colInfo); - ++i; // next col - continue; - } - - void *val2 = NULL; - while (j < nCols2) { - STColumn *tCol = schemaColAt(pSchema2, j); - if (tCol->colId < pCol->colId) { - ++j; - continue; - } - if (tCol->colId == pCol->colId) { - val2 = tdGetMemRowDataOfColEx(row2, tCol->colId, tCol->type, TD_DATA_ROW_HEAD_SIZE + tCol->offset, &kvIdx2); - } else if (tCol->colId > pCol->colId) { - // set NULL - } - break; - } // end of while(jtype); - } - tdAppendColVal(dataRow, val2, pCol->type, pCol->offset); - if (!isNull(val2, pCol->type)) { - kvLen += tdGetColAppendLen(SMEM_ROW_KV, val2, pCol->type); - setSColInfo(&colInfo, pCol->colId, pCol->type, val2); - taosArrayPush(stashRow, &colInfo); - } - - ++i; // next col - } - - dataLen = memRowTLen(pRow); - - if (kvLen < dataLen) { - // scan stashRow and generate SKVRow - memset(buffer, 0, sizeof(dataLen)); - SMemRow tRow = buffer; - memRowSetType(tRow, SMEM_ROW_KV); - SKVRow kvRow = (SKVRow)memRowKvBody(tRow); - int16_t nKvNCols = (int16_t) taosArrayGetSize(stashRow); - kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nKvNCols)); - kvRowSetNCols(kvRow, nKvNCols); - memRowSetKvVersion(tRow, pSchema1->version); - - int32_t toffset = 0; - int16_t k; - for (k = 0; k < nKvNCols; ++k) { - SColInfo *pColInfo = taosArrayGet(stashRow, k); - tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset); - toffset += sizeof(SColIdx); - } - ASSERT(kvLen == memRowTLen(tRow)); - } - taosArrayDestroy(stashRow); - return buffer; -} -#endif diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c878109711..f017d22b39 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -169,6 +169,9 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR +// udf +bool tsStartUdfd = true; + // internal int32_t tsTransPullupInterval = 6; int32_t tsMqRebalanceInterval = 2; @@ -441,6 +444,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1; + if (cfgAddBool(pCfg, "startUdfd", tsStartUdfd, 0) != 0) return -1; return 0; } @@ -581,6 +585,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; + tsStartUdfd = cfgGetItem(pCfg, "startUdfd")->bval; + if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 56bb93faa4..9524fb1f44 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -607,6 +607,11 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; + if (pReq->commentLen > 0) { + if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -639,6 +644,14 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq } } + if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; + if (pReq->commentLen > 0) { + pReq->comment = taosMemoryMalloc(pReq->commentLen); + if (pReq->comment == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1233,21 +1246,21 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs for (int32_t i = 0; i < numOfCreatedDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(pDecoder, db) < 0) return -1; - int32_t len = strlen(db) + 1; + int32_t len = strlen(db); taosHashPut(pRsp->createdDbs, db, len, db, len); } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(pDecoder, db) < 0) return -1; - int32_t len = strlen(db) + 1; + int32_t len = strlen(db); taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; if (tDecodeCStrTo(pDecoder, db) < 0) return -1; - int32_t len = strlen(db) + 1; + int32_t len = strlen(db); taosHashPut(pRsp->writeDbs, db, len, db, len); } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 50e51a40dd..44bcd72a33 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -33,6 +33,7 @@ const uint8_t tdVTypeByte[2][3] = {{ // declaration static uint8_t tdGetBitmapByte(uint8_t byte); +static int32_t tdCompareColId(const void *arg1, const void *arg2); // static void dataColSetNEleNull(SDataCol *pCol, int nEle); @@ -916,4 +917,937 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { } return pRet; +} + +void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { + STSRowIter iter = {0}; + tdSTSRowIterInit(&iter, pSchema); + tdSTSRowIterReset(&iter, row); + printf("%s >>>", tag); + for (int i = 0; i < pSchema->numOfCols; ++i) { + STColumn *stCol = pSchema->columns + i; + SCellVal sVal = {255, NULL}; + if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + break; + } + ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); + tdSCellValPrint(&sVal, stCol->type); + } + printf("\n"); +} + +void tdSCellValPrint(SCellVal *pVal, int8_t colType) { + if (tdValTypeIsNull(pVal->valType)) { + printf("NULL "); + return; + } else if (tdValTypeIsNone(pVal->valType)) { + printf("NONE "); + return; + } + switch (colType) { + case TSDB_DATA_TYPE_BOOL: + printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); + break; + case TSDB_DATA_TYPE_TINYINT: + printf("%" PRIi8 " ", *(int8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%" PRIi16 " ", *(int16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_INT: + printf("%" PRIi32 " ", *(int32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_BIGINT: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f ", *(float *)pVal->val); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%lf ", *(double *)pVal->val); + break; + case TSDB_DATA_TYPE_VARCHAR: + printf("VARCHAR "); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_NCHAR: + printf("NCHAR "); + break; + case TSDB_DATA_TYPE_UTINYINT: + printf("%" PRIu8 " ", *(uint8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf("%" PRIu16 " ", *(uint16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UINT: + printf("%" PRIu32 " ", *(uint32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf("%" PRIu64 " ", *(uint64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_JSON: + printf("JSON "); + break; + case TSDB_DATA_TYPE_VARBINARY: + printf("VARBIN "); + break; + case TSDB_DATA_TYPE_DECIMAL: + printf("DECIMAL "); + break; + case TSDB_DATA_TYPE_BLOB: + printf("BLOB "); + break; + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("MedBLOB "); + break; + // case TSDB_DATA_TYPE_BINARY: + // printf("BINARY "); + // break; + case TSDB_DATA_TYPE_MAX: + printf("UNDEF "); + break; + default: + printf("UNDEF "); + break; + } +} + +int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) { + ASSERT(rows > 0); + int32_t result = 0; + + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + result += pDataCol->dataOff[rows - 1]; + SCellVal val = {0}; + if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) { + TASSERT(0); + } + + // Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4 + result += varDataTLen(val.val); + // TODO: later on, don't save Null/None for VarDataT for 3.0 + // if (tdValTypeIsNorm(val.valType)) { + // result += varDataTLen(val.val); + // } + } else { + result += TYPE_BYTES[pDataCol->type] * rows; + } + + ASSERT(pDataCol->len == result); + + return result; +} + +bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); + return true; + } + void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); + tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx); + return true; +} + +bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, + SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); + return true; + } + void *pBitmap = tdGetBitmapAddrTp(pRow, flen); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); + return true; +} + +int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { + if (isAllRowsNone(pCol)) { + pVal->valType = TD_VTYPE_NULL; +#ifdef TD_SUPPORT_READ2 + pVal->val = (void *)getNullValue(pCol->type); +#else + pVal->val = NULL; +#endif + return TSDB_CODE_SUCCESS; + } + + if (TD_COL_ROWS_NORM(pCol)) { + pVal->valType = TD_VTYPE_NORM; + } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) { + return terrno; + } + + if (tdValTypeIsNorm(pVal->valType)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); + } + } else { + pVal->valType = TD_VTYPE_NULL; +#ifdef TD_SUPPORT_READ2 + pVal->val = (void *)getNullValue(pCol->type); +#else + pVal->val = NULL; +#endif + } + return TSDB_CODE_SUCCESS; +} + +bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + if (TD_IS_TP_ROW(pIter->pRow)) { + STColumn *pCol = NULL; + STSchema *pSchema = pIter->pSchema; + while (pIter->colIdx < pSchema->numOfCols) { + pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key + if (colId == pCol->colId) { + break; + } else if (pCol->colId < colId) { + ++pIter->colIdx; + continue; + } else { + return false; + } + } + tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + ++pIter->colIdx; + } else if (TD_IS_KV_ROW(pIter->pRow)) { + return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); + } else { + pVal->valType = TD_VTYPE_NONE; + terrno = TSDB_CODE_INVALID_PARA; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + } + return true; +} + +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + SKvRowIdx *pKvIdx = NULL; + bool colFound = false; + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + while (*nIdx < kvNCols) { + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); + if (pKvIdx->colId == colId) { + ++(*nIdx); + pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); + colFound = true; + break; + } else if (pKvIdx->colId > colId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + ++(*nIdx); + } + } + + if (!colFound) { + if (colId <= pIter->maxColId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + return false; + } + } + +#ifdef TD_SUPPORT_BITMAP + int16_t colIdx = -1; + if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); + if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } +#else + pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + + return true; +} + +bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + +#ifdef TD_SUPPORT_BITMAP + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } +#else + pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + + return true; +} + +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { + if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { + return 1; + } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { + return -1; + } else { + return 0; + } +} + +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + STSRow *pRow = pIter->pRow; + int16_t colIdx = -1; + if (TD_IS_TP_ROW(pRow)) { + STSchema *pSchema = pIter->pSchema; + STColumn *pCol = + (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); + if (!pCol) { + pVal->valType = TD_VTYPE_NONE; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + return true; + } +#ifdef TD_SUPPORT_BITMAP + colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); +#endif + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + } else if (TD_IS_KV_ROW(pRow)) { + SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), + compareKvRowColId, TD_EQ); +#ifdef TD_SUPPORT_BITMAP + if (pIdx) { + colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); + } +#endif + tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); + } else { + if (COL_REACH_END(colId, pIter->maxColId)) return false; + pVal->valType = TD_VTYPE_NONE; + } + + return true; +} + +static int32_t tdCompareColId(const void *arg1, const void *arg2) { + int32_t colId = *(int32_t *)arg1; + STColumn *pCol = (STColumn *)arg2; + + if (colId < pCol->colId) { + return -1; + } else if (colId == pCol->colId) { + return 0; + } else { + return 1; + } +} + +int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS; + int16_t nOffset = colIdx & TD_VTYPE_OPTR; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0xC0) >> 6); + break; + case 1: + *pValType = (((*pDestByte) & 0x30) >> 4); + break; + case 2: + *pValType = (((*pDestByte) & 0x0C) >> 2); + break; + case 3: + *pValType = ((*pDestByte) & 0x03); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0x80) >> 7); + break; + case 1: + *pValType = (((*pDestByte) & 0x40) >> 6); + break; + case 2: + *pValType = (((*pDestByte) & 0x20) >> 5); + break; + case 3: + *pValType = (((*pDestByte) & 0x10) >> 4); + break; + case 4: + *pValType = (((*pDestByte) & 0x08) >> 3); + break; + case 5: + *pValType = (((*pDestByte) & 0x04) >> 2); + break; + case 6: + *pValType = (((*pDestByte) & 0x02) >> 1); + break; + case 7: + *pValType = ((*pDestByte) & 0x01); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); + // set the value and clear other partitions for offset 0 + // *pDestByte |= (valType << 7); + break; + case 1: + *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); + // *pDestByte |= (valType << 6); + break; + case 2: + *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); + // *pDestByte |= (valType << 5); + break; + case 3: + *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); + // *pDestByte |= (valType << 4); + break; + case 4: + *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); + // *pDestByte |= (valType << 3); + break; + case 5: + *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); + // *pDestByte |= (valType << 2); + break; + case 6: + *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); + // *pDestByte |= (valType << 1); + break; + case 7: + *pDestByte = ((*pDestByte) & 0xFE) | valType; + // *pDestByte |= (valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { +#ifdef TD_SUPPORT_BITMAP + TASSERT(colIdx < tdRowGetNCols(pRow) - 1); + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; + } + if (tdValTypeIsNorm(output->valType)) { + if (offset < 0) { + terrno = TSDB_CODE_INVALID_PARA; + output->valType = TD_VTYPE_NONE; + return terrno; + } + output->val = POINTER_SHIFT(pRow, offset); + } +#else + TASSERT(0); + if (offset < 0) { + terrno = TSDB_CODE_INVALID_PARA; + output->valType = TD_VTYPE_NONE; + return terrno; + } + output->val = POINTER_SHIFT(pRow, offset); + output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, + int16_t colIdx) { +#ifdef TD_SUPPORT_BITMAP + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; + } + if (tdValTypeIsNorm(output->valType)) { + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } +#else + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val, + bool isCopyVarData, int32_t offset, col_id_t colIdx) { + STSRow *pRow = pBuilder->pBuf; + if (!val) { +#ifdef TD_SUPPORT_BITMAP + if (tdValTypeIsNorm(valType)) { + terrno = TSDB_CODE_INVALID_PTR; + return terrno; + } +#else + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; +#endif + } + // TS KEY is stored in STSRow.ts and not included in STSRow.data field. + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + TD_ROW_KEY(pRow) = *(TSKEY *)val; + // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. + return TSDB_CODE_SUCCESS; + } + // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. + if (TD_IS_TP_ROW(pRow)) { + tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); + } else { + tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { + if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + offset -= sizeof(SKvRowIdx); + --colIdx; + +#ifdef TD_SUPPORT_BITMAP + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { + return terrno; + } +#endif + + STSRow *row = pBuilder->pBuf; + // No need to store None/Null values. + if (tdValIsNorm(valType, val, colType)) { + // ts key stored in STSRow.ts + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + + if (IS_VAR_DATA_TYPE(colType)) { + if (isCopyVarData) { + memcpy(ptr, val, varDataTLen(val)); + } + TD_ROW_LEN(row) += varDataTLen(val); + } else { + memcpy(ptr, val, TYPE_BYTES[colType]); + TD_ROW_LEN(row) += TYPE_BYTES[colType]; + } + } +#ifdef TD_SUPPORT_BACK2 + // NULL/None value + else { + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + const void *nullVal = getNullValue(colType); + + if (IS_VAR_DATA_TYPE(colType)) { + if (isCopyVarData) { + memcpy(ptr, nullVal, varDataTLen(nullVal)); + } + TD_ROW_LEN(row) += varDataTLen(nullVal); + } else { + memcpy(ptr, nullVal, TYPE_BYTES[colType]); + TD_ROW_LEN(row) += TYPE_BYTES[colType]; + } + } +#endif + + return 0; +} + +int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset) { + if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + offset -= sizeof(TSKEY); + --colIdx; + +#ifdef TD_SUPPORT_BITMAP + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { + return terrno; + } +#endif + + STSRow *row = pBuilder->pBuf; + + // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap + // should be updated simultaneously if Norm val overwrite Null/None cols. + // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. + if (tdValIsNorm(valType, val, colType)) { + // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. + if (IS_VAR_DATA_TYPE(colType)) { + // ts key stored in STSRow.ts + *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); + if (isCopyVarData) { + memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val)); + } + TD_ROW_LEN(row) += varDataTLen(val); + } else { + memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); + } + } +#ifdef TD_SUPPORT_BACK2 + // NULL/None value + else { + // TODO: Null value for new data types imported since 3.0 need to be defined. + const void *nullVal = getNullValue(colType); + if (IS_VAR_DATA_TYPE(colType)) { + // ts key stored in STSRow.ts + *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); + + if (isCopyVarData) { + memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); + } + TD_ROW_LEN(row) += varDataTLen(nullVal); + } else { + memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); + } + } +#endif + + return 0; +} + +int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, + int32_t allNullLen, int32_t boundNullLen) { + if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { + uint32_t tpLen = allNullLen; + uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen; + if (isSelectKVRow(kvLen, tpLen)) { + pBuilder->rowType = TD_ROW_KV; + } else { + pBuilder->rowType = TD_ROW_TP; + } + + } else { + pBuilder->rowType = TD_ROW_TP; + } + + pBuilder->flen = flen; + pBuilder->nCols = nCols; + pBuilder->nBoundCols = nBoundCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + if (nBoundCols > 0) { + pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + } else { + pBuilder->nBoundBitmaps = 0; + } +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TD_ROW_SET_INFO(pBuilder->pBuf, 0); + TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); +#endif + // the primary TS key is stored separatedly + len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; + TD_ROW_SET_LEN(pBuilder->pBuf, len); + TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); +#endif + len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + + pBuilder->nBoundBitmaps; // add + TD_ROW_SET_LEN(pBuilder->pBuf, len); + TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); + TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); +#endif + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); +#endif + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, + int32_t nBoundCols, int32_t flen) { + if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { + return terrno; + } + return tdSRowResetBuf(pBuilder, pBuf); +} + +void tdSRowReset(SRowBuilder *pBuilder) { + pBuilder->rowType = TD_ROW_TP; + pBuilder->pBuf = NULL; + pBuilder->nBoundCols = -1; + pBuilder->nCols = -1; + pBuilder->flen = -1; + pBuilder->pBitmap = NULL; +} + +int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { + pBuilder->flen = flen; + pBuilder->nCols = nCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) { + pBuilder->flen = flen; + pBuilder->nCols = nCols; + pBuilder->nBoundCols = nBoundCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + if (nBoundCols > 0) { + pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + } else { + pBuilder->nBoundBitmaps = 0; + } +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdGetBitmapValTypeII(pBitmap, colIdx, pValType); + break; + case -1: + case 1: + tdGetBitmapValTypeI(pBitmap, colIdx, pValType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { + TDRowValT valType = 0; + tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); + if (tdValTypeIsNorm(valType)) { + return true; + } + return false; +} + +int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS; + int16_t nOffset = colIdx & TD_VTYPE_OPTR; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); + // set the value and clear other partitions for offset 0 + // *pDestByte |= (valType << 6); + break; + case 1: + *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); + // *pDestByte |= (valType << 4); + break; + case 2: + *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); + // *pDestByte |= (valType << 2); + break; + case 3: + *pDestByte = ((*pDestByte) & 0xFC) | valType; + // *pDestByte |= (valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdSetBitmapValTypeII(pBitmap, colIdx, valType); + break; + case -1: + case 1: + tdSetBitmapValTypeI(pBitmap, colIdx, valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { +#ifdef TD_SUPPORT_BITMAP + switch (rowType) { + case TD_ROW_TP: + return tdGetBitmapAddrTp(pRow, flen); + case TD_ROW_KV: + return tdGetBitmapAddrKv(pRow, nKvCols); + default: + break; + } +#endif + return NULL; +} + +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { + pIter->pRow = pRow; + pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); + pIter->offset = 0; + pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; + pIter->kvIdx = 0; +} + +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { + pIter->pSchema = pSchema; + pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; } \ No newline at end of file diff --git a/source/dnode/mgmt/implement/inc/dmImp.h b/source/dnode/mgmt/implement/inc/dmImp.h index 32869aee9e..8a1a116ab3 100644 --- a/source/dnode/mgmt/implement/inc/dmImp.h +++ b/source/dnode/mgmt/implement/inc/dmImp.h @@ -26,8 +26,10 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper); void dmCloseNode(SMgmtWrapper *pWrapper); // dmTransport.c -int32_t dmInitTrans(SDnode *pDnode); -void dmCleanupTrans(SDnode *pDnode); +int32_t dmInitServer(SDnode *pDnode); +void dmCleanupServer(SDnode *pDnode); +int32_t dmInitClient(SDnode *pDnode); +void dmCleanupClient(SDnode *pDnode); SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper); SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper); int32_t dmInitMsgHandle(SDnode *pDnode); diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/implement/src/dmExec.c index 06001028b5..6999cee037 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/implement/src/dmExec.c @@ -213,10 +213,12 @@ static int32_t dmOpenNodes(SDnode *pDnode) { } pWrapper->procType = DND_PROC_CHILD; + if (dmInitClient(pDnode) != 0) { + return -1; + } - SMsgCb msgCb = pDnode->data.msgCb; - msgCb.pWrapper = pWrapper; - tmsgSetDefaultMsgCb(&msgCb); + pDnode->data.msgCb = dmGetMsgcb(pWrapper); + tmsgSetDefaultMsgCb(&pDnode->data.msgCb); if (dmOpenNode(pWrapper) != 0) { dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); @@ -234,6 +236,15 @@ static int32_t dmOpenNodes(SDnode *pDnode) { pWrapper->procType = DND_PROC_SINGLE; } + if (n == DNODE) { + if (dmInitClient(pDnode) != 0) { + return -1; + } + + pDnode->data.msgCb = dmGetMsgcb(pWrapper); + tmsgSetDefaultMsgCb(&pDnode->data.msgCb); + } + if (dmOpenNode(pWrapper) != 0) { dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); return -1; @@ -281,21 +292,21 @@ static void dmProcessProcHandle(void *handle) { } static void dmWatchNodes(SDnode *pDnode) { - taosThreadMutexLock(&pDnode->mutex); - if (pDnode->ptype == DND_PROC_PARENT) { - for (EDndNodeType n = DNODE + 1; n < NODE_END; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - if (!pWrapper->required) continue; - if (pWrapper->procType != DND_PROC_PARENT) continue; - if (pDnode->ntype == NODE_END) continue; + if (pDnode->ptype != DND_PROC_PARENT) return; + if (pDnode->ntype == NODE_END) return; - if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) { - dWarn("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId); - if (pWrapper->procObj) { - taosProcCloseHandles(pWrapper->procObj, dmProcessProcHandle); - } - dmNewNodeProc(pWrapper, n); + taosThreadMutexLock(&pDnode->mutex); + for (EDndNodeType n = DNODE + 1; n < NODE_END; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + if (pWrapper->procType != DND_PROC_PARENT) continue; + + if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) { + dWarn("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId); + if (pWrapper->procObj) { + taosProcCloseHandles(pWrapper->procObj, dmProcessProcHandle); } + dmNewNodeProc(pWrapper, n); } } taosThreadMutexUnlock(&pDnode->mutex); diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 129d41061e..097d18679b 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -242,7 +242,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { return -1; } - if (dmInitTrans(pDnode) != 0) { + if (dmInitServer(pDnode) != 0) { dError("failed to init transport since %s", terrstr()); return -1; } @@ -275,7 +275,8 @@ static void dmCleanupMgmt(SMgmtWrapper *pWrapper) { } taosWUnLockLatch(&pDnode->data.latch); - dmCleanupTrans(pDnode); + dmCleanupClient(pDnode); + dmCleanupServer(pDnode); dInfo("dnode-mgmt is cleaned up"); } diff --git a/source/dnode/mgmt/implement/src/dmObj.c b/source/dnode/mgmt/implement/src/dmObj.c index 66bfb27016..a43439d465 100644 --- a/source/dnode/mgmt/implement/src/dmObj.c +++ b/source/dnode/mgmt/implement/src/dmObj.c @@ -124,9 +124,6 @@ SDnode *dmCreate(const SDnodeOpt *pOption) { goto _OVER; } - pDnode->data.msgCb = dmGetMsgcb(&pDnode->wrappers[DNODE]); - tmsgSetDefaultMsgCb(&pDnode->data.msgCb); - dInfo("dnode is created, data:%p", pDnode); code = 0; diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 114d7b6dfc..a58999bf2d 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "dmImp.h" +#include "qworker.h" + #define INTERNAL_USER "_dnd" #define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" @@ -85,17 +87,14 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER; - if (pWrapper->procType == DND_PROC_SINGLE) { + if (pWrapper->procType != DND_PROC_PARENT) { dTrace("msg:%p, created, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->user); code = (*msgFp)(pWrapper, pMsg); - } else if (pWrapper->procType == DND_PROC_PARENT) { + } else { dTrace("msg:%p, created and put into child queue, type:%s handle:%p code:0x%04x user:%s contLen:%d", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->rpcMsg.code & 0XFFFF, pMsg->user, pRpc->contLen); code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, (isReq && (pMsg->rpcMsg.code == 0)) ? pRpc->handle : NULL, pRpc->refId, PROC_FUNC_REQ); - } else { - dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); - ASSERT(1); } _OVER: @@ -108,7 +107,7 @@ _OVER: } else { dError("msg:%p, type:%s handle:%p failed to process since 0x%04x:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, code & 0XFFFF, terrstr()); - if (msgType & 1U) { + if (isReq) { if (terrno != 0) code = terrno; if (code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_NODE_OFFLINE) { if (msgType > TDMT_MND_MSG && msgType < TDMT_VND_MSG) { @@ -130,22 +129,27 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans * pTrans = &pDnode->trans; + SDnodeTrans *pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; - if (msgType == TDMT_DND_SERVER_STATUS) { - dTrace("server status req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); - dmProcessServerStatusReq(pDnode, pMsg); - return; - } - - if (msgType == TDMT_DND_NET_TEST) { - dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); - dmProcessServerStatusReq(pDnode, pMsg); - return; + switch (msgType) { + case TDMT_DND_SERVER_STATUS: + dTrace("server status req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dmProcessServerStatusReq(pDnode, pMsg); + return; + case TDMT_DND_NET_TEST: + dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dmProcessNetTestReq(pDnode, pMsg); + return; + case TDMT_MND_SYSTABLE_RETRIEVE_RSP: + case TDMT_VND_FETCH_RSP: + dTrace("retrieve rsp is received"); + qWorkerProcessFetchRsp(NULL, NULL, pMsg); + pMsg->pCont = NULL; // already freed in qworker + return; } if (pDnode->status != DND_STAT_RUNNING) { @@ -233,16 +237,6 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { return 0; } -static inline int32_t dmSendRpcReq(SDnode *pDnode, const SEpSet *pEpSet, SRpcMsg *pReq) { - if (pDnode->trans.clientRpc == NULL) { - terrno = TSDB_CODE_NODE_OFFLINE; - return -1; - } - - rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pReq, NULL); - return 0; -} - static void dmSendRpcRedirectRsp(SDnode *pDnode, const SRpcMsg *pReq) { SEpSet epSet = {0}; dmGetMnodeEpSet(pDnode, &epSet); @@ -288,28 +282,20 @@ void dmSendToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { } static inline int32_t dmSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { - if (pWrapper->pDnode->status != DND_STAT_RUNNING) { + SDnode *pDnode = pWrapper->pDnode; + if (pDnode->status != DND_STAT_RUNNING) { terrno = TSDB_CODE_NODE_OFFLINE; dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); return -1; } - if (pWrapper->procType != DND_PROC_CHILD) { - return dmSendRpcReq(pWrapper->pDnode, pEpSet, pReq); - } else { - char *pHead = taosMemoryMalloc(sizeof(SRpcMsg) + sizeof(SEpSet)); - if (pHead == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - memcpy(pHead, pReq, sizeof(SRpcMsg)); - memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); - taosProcPutToParentQ(pWrapper->procObj, pHead, sizeof(SRpcMsg) + sizeof(SEpSet), pReq->pCont, pReq->contLen, - PROC_FUNC_REQ); - taosMemoryFree(pHead); - return 0; + if (pDnode->trans.clientRpc == NULL) { + terrno = TSDB_CODE_NODE_OFFLINE; + return -1; } + + rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pReq, NULL); + return 0; } static inline void dmSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { @@ -396,9 +382,10 @@ static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t pMsg->pCont = pCont; if (ftype == PROC_FUNC_REQ) { + ASSERT(1); dTrace("msg:%p, get from parent queue, send req:%s handle:%p code:0x%04x, app:%p", pMsg, TMSG_INFO(pMsg->msgType), pMsg->handle, code, pMsg->ahandle); - dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); + dmSendReq(pWrapper, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); } else if (ftype == PROC_FUNC_RSP) { dTrace("msg:%p, get from parent queue, rsp handle:%p code:0x%04x, app:%p", pMsg, pMsg->handle, code, pMsg->ahandle); pMsg->refId = taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); @@ -421,23 +408,25 @@ static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t } SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper) { - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dmConsumeChildQueue, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .parentConsumeFp = (ProcConsumeFp)dmConsumeParentQueue, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .shm = pWrapper->procShm, - .parent = pWrapper, - .name = pWrapper->name}; + SProcCfg cfg = { + .childConsumeFp = (ProcConsumeFp)dmConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentConsumeFp = (ProcConsumeFp)dmConsumeParentQueue, + .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .shm = pWrapper->procShm, + .parent = pWrapper, + .name = pWrapper->name, + }; return cfg; } -bool rpcRfp(int32_t code) { +static bool rpcRfp(int32_t code) { if (code == TSDB_CODE_RPC_REDIRECT) { return true; } else { @@ -445,7 +434,7 @@ bool rpcRfp(int32_t code) { } } -static int32_t dmInitClient(SDnode *pDnode) { +int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; @@ -471,11 +460,15 @@ static int32_t dmInitClient(SDnode *pDnode) { return -1; } + pDnode->data.msgCb = dmGetMsgcb(&pDnode->wrappers[DNODE]); + tmsgSetDefaultMsgCb(&pDnode->data.msgCb); + dDebug("dnode rpc client is initialized"); + return 0; } -static void dmCleanupClient(SDnode *pDnode) { +void dmCleanupClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; if (pTrans->clientRpc) { rpcClose(pTrans->clientRpc); @@ -517,7 +510,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -543,7 +536,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s return rpcRsp.code; } -static int32_t dmInitServer(SDnode *pDnode) { +int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; @@ -569,7 +562,7 @@ static int32_t dmInitServer(SDnode *pDnode) { return 0; } -static void dmCleanupServer(SDnode *pDnode) { +void dmCleanupServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; if (pTrans->serverRpc) { rpcClose(pTrans->serverRpc); @@ -578,17 +571,6 @@ static void dmCleanupServer(SDnode *pDnode) { } } -int32_t dmInitTrans(SDnode *pDnode) { - if (dmInitServer(pDnode) != 0) return -1; - if (dmInitClient(pDnode) != 0) return -1; - return 0; -} - -void dmCleanupTrans(SDnode *pDnode) { - dmCleanupServer(pDnode); - dmCleanupClient(pDnode); -} - SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { SMsgCb msgCb = { .sendReqFp = dmSendReq, @@ -598,6 +580,7 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartupByWrapper, .pWrapper = pWrapper, + .clientRpc = pWrapper->pDnode->trans.clientRpc, }; return msgCb; } diff --git a/source/dnode/mgmt/interface/inc/dmInt.h b/source/dnode/mgmt/interface/inc/dmInt.h index 63bfaf5ad2..b56edd2630 100644 --- a/source/dnode/mgmt/interface/inc/dmInt.h +++ b/source/dnode/mgmt/interface/inc/dmInt.h @@ -37,7 +37,7 @@ void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc); void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg); -void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg); +void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg); void dmGetMonitorSysInfo(SMonSysInfo *pInfo); // dmFile.c diff --git a/source/dnode/mgmt/interface/src/dmInt.c b/source/dnode/mgmt/interface/src/dmInt.c index 2d15a7a008..f8e23ad262 100644 --- a/source/dnode/mgmt/interface/src/dmInt.c +++ b/source/dnode/mgmt/interface/src/dmInt.c @@ -171,16 +171,17 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { } } -void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) { +void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) { dDebug("net test req is received"); - SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0}; - rsp.pCont = rpcMallocCont(pRpc->contLen); + SRpcMsg rsp = {.handle = pReq->handle, .refId = pReq->refId, .ahandle = pReq->ahandle, .code = 0}; + rsp.pCont = rpcMallocCont(pReq->contLen); if (rsp.pCont == NULL) { rsp.code = TSDB_CODE_OUT_OF_MEMORY; } else { - rsp.contLen = pRpc->contLen; + rsp.contLen = pReq->contLen; } rpcSendResponse(&rsp); + rpcFreeCont(pReq->pCont); } void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) { @@ -208,6 +209,7 @@ void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) { _OVER: rpcSendResponse(&rspMsg); + rpcFreeCont(pReq->pCont); } void dmGetMonitorSysInfo(SMonSysInfo *pInfo) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index b19602f3ff..c5919e06b6 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -289,7 +289,6 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, vmProcessQueryMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, vmProcessFetchMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 70bdda5855..645f2ff0e7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1550,10 +1550,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)t, false); - // single stable model - int8_t m = 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&m, false); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)b, false); diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 7aa55e2109..36cde396fa 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -20,7 +20,7 @@ int32_t mndProcessQueryMsg(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; - SReadHandle handle = {.mnd = pMnode}; + SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb}; mTrace("msg:%p, in query queue is processing", pReq); switch (pReq->rpcMsg.msgType) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1706820bdc..0b4d49ed58 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -459,7 +459,7 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) { - if (strcmp(alterReq.dbname, "*") != 0) { + if (strcmp(alterReq.dbname, "1.*") != 0) { int32_t len = strlen(alterReq.dbname) + 1; SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); if (pDb == NULL) { @@ -483,7 +483,7 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) { - if (strcmp(alterReq.dbname, "*") != 0) { + if (strcmp(alterReq.dbname, "1.*") != 0) { int32_t len = strlen(alterReq.dbname) + 1; SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname); if (pDb == NULL) { diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 1e03d8ff4a..3c15bc3492 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -367,8 +367,8 @@ TEST_F(MndTestUser, 03_Alter_User) { EXPECT_EQ(numOfReadDbs, 1); EXPECT_EQ(numOfWriteDbs, 0); - char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 5); - EXPECT_STREQ(dbname, "1.d2"); + char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 4); + EXPECT_TRUE(dbname != NULL); taosHashCleanup(authRsp.readDbs); taosHashCleanup(authRsp.writeDbs); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index b21141001a..54b29f546c 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -51,7 +51,7 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { qTrace("message in qnode query queue is processing"); - SReadHandle handle = {0}; + SReadHandle handle = {.pMsgCb = &pQnode->msgCb}; switch (pMsg->msgType) { case TDMT_VND_QUERY: { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index a7617291b0..986b2740f3 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -34,6 +34,7 @@ #include "tlockfree.h" #include "tlosertree.h" #include "tmallocator.h" +#include "tmsgcb.h" #include "tskiplist.h" #include "tstream.h" #include "ttime.h" @@ -121,7 +122,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); // sma -int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq); +int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb); int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 48a0f6ad61..6ca60945cd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -380,6 +380,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); @@ -859,6 +860,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { SReadHandle handle = { .reader = pExec->pExecReader[i], .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, }; pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle); ASSERT(pExec->task[i]); @@ -916,6 +918,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { SReadHandle handle = { .reader = pStreamReader, .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, }; pTask->exec.runners[i].inputHandle = pStreamReader; pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 6c0df33d05..1315963090 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -211,7 +211,7 @@ int tsdbCommit(STsdb *pRepo) { void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); - TSKEY minKey, midKey, maxKey, now; + TSKEY minKey, midKey, maxKey, now; now = taosGetTimestamp(pCfg->precision); minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; @@ -1386,34 +1386,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt tSkipListIterNext(pCommitIter->pIter); } else { -#if 0 - if (update != TD_ROW_OVERWRITE_UPDATE) { - // copy disk data - for (int i = 0; i < pDataCols->numOfCols; ++i) { - // TODO: dataColAppendVal may fail - SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode); - } - - if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; - } - if (update != TD_ROW_DISCARD_UPDATE) { - // copy mem data - if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row)); - ASSERT(pSchema != NULL); - } - - tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); - } - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); -#endif - - if(lastKey != key1) { + if (lastKey != key1) { lastKey = key1; ++pTarget->numOfRows; } @@ -1484,29 +1457,4 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p } return false; -} - -// int tsdbApplyRtn(STsdbRepo *pRepo) { -// SRtn rtn; -// SFSIter fsiter; -// STsdbFS * pfs = REPO_FS(pRepo); -// SDFileSet *pSet; - -// // Get retention snapshot -// tsdbGetRtnSnap(pRepo, &rtn); - -// tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); -// while ((pSet = tsdbFSIterNext(&fsiter))) { -// if (pSet->fid < rtn.minFid) { -// tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, -// TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); -// continue; -// } - -// if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { -// return -1; -// } -// } - -// return 0; -// } +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index eaa0893f29..384f3fd489 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -248,11 +248,13 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey pMergeInfo->nOperations++; pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - lastKey = rowKey; if (pCols) { - ++pCols->numOfRows; + if (lastKey != TSKEY_INITIAL_VAL) { + ++pCols->numOfRows; + } tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); } + lastKey = rowKey; } else { tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true); } @@ -326,6 +328,11 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext); +#ifdef TD_DEBUG_PRINT_ROW + printf("!!! %s:%d table %" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, pTbData->uid, + SL_SIZE(pTbData->pData)); +#endif + // Set statistics keyMax = TD_ROW_KEY(blkIter.row); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 952ccfda9c..3168ff53f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -21,16 +21,19 @@ typedef struct SMemSkipList SMemSkipList; typedef struct SMemSkipListNode SMemSkipListNode; typedef struct SMemSkipListCurosr SMemSkipListCurosr; +#define SL_MAX_LEVEL 5 + struct SMemTable { - STsdb *pTsdb; - TSKEY minKey; - TSKEY maxKey; - int64_t minVer; - int64_t maxVer; - int64_t nRows; - int32_t nHash; - int32_t nBucket; - SMemData **pBuckets; + STsdb *pTsdb; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + int32_t nHash; + int32_t nBucket; + SMemData **pBuckets; + SMemSkipListCurosr *pSlc; }; struct SMemSkipListNode { @@ -60,9 +63,15 @@ struct SMemData { struct SMemSkipListCurosr { SMemSkipList *pSl; - SMemSkipListNode *pNodeC; + SMemSkipListNode *pNodes[SL_MAX_LEVEL]; }; +typedef struct { + int64_t version; + uint32_t szRow; + const STSRow *pRow; +} STsdbRow; + #define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET)) #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) @@ -76,7 +85,19 @@ struct SMemSkipListCurosr { #define SL_HEAD_NODE_FORWARD(n, l) SL_NODE_FORWARD(n, l) #define SL_TAIL_NODE_BACKWARD(n, l) SL_NODE_FORWARD(n, l) -static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); +static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); +static int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow); +static int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow); +static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc); +static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc); +static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl); +static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode); +static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags); +static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc); +static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc); +static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc); +static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); +static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { @@ -102,6 +123,11 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { taosMemoryFree(pMemTb); return -1; } + if (tsdbMemSkipListCursorCreate(pTsdb->pVnode->config.tsdbCfg.slLevel, &pMemTb->pSlc) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pMemTb->pBuckets); + taosMemoryFree(pMemTb); + } *ppMemTb = pMemTb; return 0; @@ -110,6 +136,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { if (pMemTb) { // loop to destroy the contents (todo) + tsdbMemSkipListCursorDestroy(pMemTb->pSlc); taosMemoryFree(pMemTb->pBuckets); taosMemoryFree(pMemTb); } @@ -177,52 +204,42 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p } // do insert data to SMemData - SMemSkipListCurosr slc = {0}; - const STSRow *pRow; - uint32_t szRow; - SDecoder decoder = {0}; + SMemSkipListNode *forwards[SL_MAX_LEVEL]; + SMemSkipListNode *pNode; + int32_t iRow; + STsdbRow tRow = {.version = version}; + SEncoder ec = {0}; + SDecoder dc = {0}; - tDecoderInit(&decoder, pSubmitBlk->pData, pSubmitBlk->nData); - for (;;) { - if (tDecodeIsEnd(&decoder)) break; + tDecoderInit(&dc, pSubmitBlk->pData, pSubmitBlk->nData); + tsdbMemSkipListCursorInit(pMemTb->pSlc, &pMemData->sl); + for (iRow = 0;; iRow++) { + if (tDecodeIsEnd(&dc)) break; - if (tDecodeBinary(&decoder, (const uint8_t **)&pRow, &szRow) < 0) { + // decode row + if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - // check the row (todo) - - // // move the cursor to position to write (todo) - // int32_t c; - // tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c); - // ASSERT(c); + // move cursor + tsdbMemSkipListCursorMoveTo(pMemTb->pSlc, version, tRow.pRow->ts, 0); // encode row - int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); - int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (0 /*todo*/); - SMemSkipListNode *pNode = vnodeBufPoolMalloc(pPool, tsize); + pNode = tsdbMemSkipListNodeCreate(pPool, &pMemData->sl, &tRow); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pNode->level = level; - - // uint8_t *pData = SL_NODE_DATA(pSlNode); - // *(int64_t *)pData = version; - // pData += sizeof(version); - // memcpy(pData, pt, p - pt); - - // // insert row - // tsdbMemSkipListCursorPut(&slc, pSlNode); + // put the node + tsdbMemSkipListCursorPut(pMemTb->pSlc, pNode); // update status - if (pRow->ts < pMemData->minKey) pMemData->minKey = pRow->ts; - if (pRow->ts > pMemData->maxKey) pMemData->maxKey = pRow->ts; + if (tRow.pRow->ts < pMemData->minKey) pMemData->minKey = tRow.pRow->ts; + if (tRow.pRow->ts > pMemData->maxKey) pMemData->maxKey = tRow.pRow->ts; } - tDecoderClear(&decoder); - // tsdbMemSkipListCursorClose(&slc); + tDecoderClear(&dc); // update status if (pMemData->minVer == -1) pMemData->minVer = version; @@ -236,17 +253,128 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p return 0; } -static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { +static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { int8_t level = 1; - int8_t tlevel; + int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); const uint32_t factor = 4; - if (pSl->size) { - tlevel = TMIN(pSl->maxLevel, pSl->level + 1); - while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { - level++; - } + while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { + level++; } return level; +} + +static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) { + if (tEncodeI64(pEncoder, pRow->version) < 0) return -1; + if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1; + return 0; +} + +static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) { + if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1; + if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; + return 0; +} + +static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc) { + *ppSlc = (SMemSkipListCurosr *)taosMemoryCalloc(1, sizeof(**ppSlc) + sizeof(SMemSkipListNode *) * maxLevel); + if (*ppSlc == NULL) { + return -1; + } + return 0; +} + +static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc) { taosMemoryFree(pSlc); } + +static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl) { + SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); + pSlc->pSl = pSl; + // for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + // pSlc->forwards[iLevel] = pHead; + // } +} + +static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode) { + SMemSkipList *pSl = pSlc->pSl; + SMemSkipListNode *pNodeNext; + + for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) { + // todo + + ASSERT(0); + } + + if (pSl->level < pNode->level) { + pSl->level = pNode->level; + } + + pSl->size += 1; +} + +static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags) { + SMemSkipListNode **pForwards = NULL; + SMemSkipList *pSl = pSlc->pSl; + int8_t maxLevel = pSl->maxLevel; + SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); + SMemSkipListNode *pTail = SL_TAIL_NODE(pSl); + + if (pSl->size == 0) { + for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + pForwards[iLevel] = pHead; + } + } + + return 0; +} + +static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc) { + SMemSkipList *pSl = pSlc->pSl; + SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); + + for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + pSlc->pNodes[iLevel] = pHead; + } + + tsdbMemSkipListCursorMoveToNext(pSlc); +} + +static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc) { + SMemSkipList *pSl = pSlc->pSl; + SMemSkipListNode *pTail = SL_TAIL_NODE(pSl); + + for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { + pSlc->pNodes[iLevel] = pTail; + } + + tsdbMemSkipListCursorMoveToPrev(pSlc); +} + +static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) { + // TODO + return 0; +} + +static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc) { + // TODO + return 0; +} + +static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow) { + int32_t tsize; + int32_t ret; + int8_t level = tsdbMemSkipListRandLevel(pSl); + SMemSkipListNode *pNode = NULL; + SEncoder ec = {0}; + + tEncodeSize(tsdbEncodeRow, pTRow, tsize, ret); + pNode = vnodeBufPoolMalloc(pPool, tsize + SL_NODE_SIZE(level)); + if (pNode) { + pNode->level = level; + tEncoderInit(&ec, (uint8_t *)SL_NODE_DATA(pNode), tsize); + tsdbEncodeRow(&ec, pTRow); + tEncoderClear(&ec); + } + + return pNode; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9294718550..60f2f74f5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -351,35 +351,26 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } -#if 1 -int nQUERY = 0; -#endif + static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { - int level = 0; -#if 0 + int level = 0; int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { - SRetention* pRetention = retentions + i; - if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { + SRetention* pRetention = retentions + level; + if (pRetention->keep <= 0) { + if (level > 0) { + --level; + } + break; + } + if ((now - pRetention->keep) <= winSKey) { break; } ++level; } -#endif -#if 1 - switch ((nQUERY++) % 3) { - case 0: - level = 0; - break; - case 1: - level = 1; - break; - default: - level = 2; - break; - } -#endif + if (level == TSDB_RETENTION_L0) { tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA0(pVnode); @@ -391,7 +382,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, return VND_RSMA2(pVnode); } } - return pVnode->pTsdb; + return VND_TSDB(pVnode); } static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) { @@ -3879,7 +3870,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch if (metaGetTableEntryByUid(&mr, uid) < 0) { tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _error; } else { tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); @@ -3949,7 +3940,7 @@ int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGr metaReaderInit(&mr, (SMeta*)pMeta, 0); if (metaGetTableEntryByUid(&mr, uid) < 0) { - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _error; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 615018a9ea..32051c2de4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1698,7 +1698,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { * @param pReq * @return int32_t */ -int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq) { +int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) { if (!pReq->rollup) { tsdbDebug("vgId:%d return directly since no rollup for stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; @@ -1742,6 +1742,7 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateStbReq *pReq) { SReadHandle handle = { .reader = pReadHandle, .meta = pMeta, + .pMsgCb = pMsgCb, }; if (param->qmsg1) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ee6bdc1a59..0082ca0802 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -142,9 +142,9 @@ _err: int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); #if 0 - SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode}; + SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; #endif - SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode}; + SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { case TDMT_VND_QUERY: return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); @@ -305,7 +305,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, goto _err; } - tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req); + tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req, &pVnode->msgCb); tDecoderClear(&coder); return 0; @@ -497,6 +497,47 @@ _exit: return 0; } +static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char* tags) { + ASSERT(pMsg != NULL); + SSubmitMsgIter msgIter = {0}; + SMeta *pMeta = pVnode->pMeta; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + STSchema *pSchema = NULL; + tb_uid_t suid = 0; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + if (blkIter.row == NULL) continue; + if (!pSchema || (suid != msgIter.suid)) { + if (pSchema) { + taosMemoryFreeClear(pSchema); + } + pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema + if(pSchema) { + suid = msgIter.suid; + } + } + if(!pSchema) { + printf("%s:%d no valid schema\n", tags, __LINE__); + continue; + } + char __tags[128] = {0}; + snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter.uid); + while ((row = tGetSubmitBlkNext(&blkIter))) { + tdSRowPrint(row, pSchema, __tags); + } + } + + taosMemoryFreeClear(pSchema); + + return 0; +} + static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitMsgIter msgIter = {0}; @@ -508,6 +549,10 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in pRsp->code = 0; +#ifdef TD_DEBUG_PRINT_ROW + vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); +#endif + // handle the request if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; @@ -550,7 +595,8 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in goto _exit; } - rsp.numOfRows += nRows; + rsp.affectedRows += nRows; + } _exit: diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 3e8528e3d9..1bf21ad7d1 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -36,7 +36,7 @@ extern "C" { #define CTG_DEFAULT_INVALID_VERSION (-1) -#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_TDB_INVALID_TABLE_ID +#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_PAR_TABLE_NOT_EXIST enum { CTG_READ = 1, diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5cfe796a29..93e81aa70e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -62,11 +62,6 @@ enum { * 2. when all data within queried time window, it is also denoted as query_completed */ TASK_COMPLETED = 0x2u, - - /* when the result is not completed return to client, this status will be - * usually used in case of interval query with interpolation option - */ - TASK_OVER = 0x4u, }; typedef struct SResultRowCell { @@ -288,12 +283,6 @@ typedef struct SOperatorInfo { SOperatorFpSet fpSet; } SOperatorInfo; -typedef struct { - int32_t numOfTags; - int32_t numOfCols; - SColumnInfo* colList; -} SQueriedTableInfo; - typedef enum { EX_SOURCE_DATA_NOT_READY = 0x1, EX_SOURCE_DATA_READY = 0x2, @@ -392,10 +381,7 @@ typedef struct SStreamBlockScanInfo { } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { - union { - void* pTransporter; - SReadHandle readHandle; - }; + SReadHandle readHandle; SRetrieveMetaTableRsp* pRsp; SRetrieveTableReq req; @@ -632,8 +618,7 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf); -void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, - SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); +void finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, @@ -663,8 +648,6 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); -SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); - SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, int32_t dataLoadFlag, const uint8_t* scanInfo, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo); @@ -716,8 +699,6 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pE #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, - SExprInfo* pExpr, int32_t numOfOutput); #endif int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, @@ -742,7 +723,6 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); int32_t getMaximumIdleDurationSec(); -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx* pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, EOPTR_EXEC_MODEL model); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index ba77950912..3cc75a815d 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -202,7 +202,7 @@ int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER); + return isTaskKilled(pTaskInfo); } void qDestroyTask(qTaskInfo_t qTaskHandle) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2166de9fb2..33f0c440ec 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -107,7 +107,6 @@ static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo); static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); -static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo* pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); @@ -204,9 +203,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = nodesListGetNode(pNode->pSlots, i); -// if (!pDescNode->output) { // todo disable it temporarily -// continue; -// } + // if (!pDescNode->output) { // todo disable it temporarily + // continue; + // } idata.info.type = pDescNode->dataType.type; idata.info.bytes = pDescNode->dataType.bytes; @@ -620,7 +619,7 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* continue; } - if (functionNeedToExecute(&pCtx[k])) { + if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { pCtx[k].fpSet.process(&pCtx[k]); } @@ -803,9 +802,12 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx) { for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { if (functionNeedToExecute(&pCtx[k])) { - pCtx[k].startTs = startTs; // this can be set during create the struct - if (pCtx[k].fpSet.process != NULL) + pCtx[k].startTs = startTs; + // this can be set during create the struct + // todo add a dummy funtion to avoid process check + if (pCtx[k].fpSet.process != NULL) { pCtx[k].fpSet.process(&pCtx[k]); + } } } } @@ -835,7 +837,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); if (pResult->info.rows > 0 && !createNewColModel) { - colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], + pfCtx->input.numOfRows); } else { colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); } @@ -921,6 +924,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { if (IS_VAR_DATA_TYPE(type)) { // todo disable this + // if (pResultRow->key == NULL) { // pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); // varDataCopy(pResultRow->key, pData); @@ -1074,7 +1078,7 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* } // set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { int32_t num = 0; SqlFunctionCtx* p = NULL; @@ -1086,7 +1090,7 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { if (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_select_value") == 0) { pValCtx[num++] = &pCtx[i]; - } else { + } else if (fmIsAggFunc(pCtx[i].functionId)) { p = &pCtx[i]; } // if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { @@ -1214,7 +1218,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize); } - setCtxTagColumnInfo(pFuncCtx, numOfOutput); + setSelectValueColumnInfo(pFuncCtx, numOfOutput); return pFuncCtx; } @@ -1450,8 +1454,6 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in } } -#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) - // static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, // SqlFunctionCtx *pCtx, int32_t numOfRows) { // STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1876,7 +1878,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) } void initResultRow(SResultRow* pResultRow) { -// pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); + // pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); } /* @@ -1888,7 +1890,8 @@ void initResultRow(SResultRow* pResultRow) { * offset[0] offset[1] offset[2] */ // TODO refactor: some function move away -void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, int32_t numOfExprs, SExecTaskInfo* pTaskInfo) { +void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, int32_t numOfExprs, + SExecTaskInfo* pTaskInfo) { SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; @@ -2007,8 +2010,8 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } // todo merged with the build group result. -void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, - SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { +void finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, + int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < pResultRowInfo->size; ++i) { SResultRowPosition* pPos = &pResultRowInfo->pPosition[i]; @@ -2021,17 +2024,11 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD // } for (int32_t j = 0; j < numOfOutput; ++j) { - pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); - - struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; + struct SResultRowEntryInfo* pResInfo = getResultCell(pRow, j, rowCellInfoOffset); if (!isRowEntryInitialized(pResInfo)) { continue; } - if (pCtx[j].fpSet.process) { // TODO set the dummy function, to avoid the check for null ptr. - // pCtx[j].fpSet.finalize(&pCtx[j]); - } - if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; } @@ -2185,17 +2182,15 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased int32_t numOfResult = pBlock->info.rows; // there are already exists result rows int32_t start = 0; - int32_t step = -1; + int32_t step = 1; // qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv)); assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); if (orderType == TSDB_ORDER_ASC) { start = pGroupResInfo->index; - step = 1; } else { // desc order copy all data start = numOfRows - pGroupResInfo->index - 1; - step = -1; } for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { @@ -2225,10 +2220,13 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor } else { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - - char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); - colDataAppend(pColInfoData, pBlock->info.rows, in, pCtx[j].resultInfo->isNullRes); + // expand the result into multiple rows. E.g., _wstartts, top(k, 20) + // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for(int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } } } @@ -2245,12 +2243,13 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased return 0; } -void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf) { +void doBuildResultDatablock(SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, + SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; - SSDataBlock* pBlock = pbInfo->pRes; - SqlFunctionCtx* pCtx = pbInfo->pCtx; + int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; + SSDataBlock* pBlock = pbInfo->pRes; + SqlFunctionCtx* pCtx = pbInfo->pCtx; blockDataCleanup(pBlock); if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { @@ -2424,7 +2423,8 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { - qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 " us, total blocks:%d, " + qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 + " us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows); @@ -2959,7 +2959,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI relocateColumnData(pRes, pColList, pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock); taosMemoryFree(pBlock); -// blockDataDestroy(pBlock); + // blockDataDestroy(pBlock); } pRes->info.rows = numOfRows; @@ -3264,7 +3264,8 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { return TSDB_CODE_SUCCESS; } -SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createExchangeOperatorInfo(void *pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, + SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -3307,29 +3308,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL, NULL, NULL); - -#if 1 - { // todo refactor - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "EX"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = qProcessFetchRsp; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char*)"root"; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char*)"dcc5bed04851fec854c035b2e40263b6"; - - pInfo->pTransporter = rpcOpen(&rpcInit); - if (pInfo->pTransporter == NULL) { - return NULL; // todo - } - } -#endif + pInfo->pTransporter = pTransporter; return pOperator; @@ -3699,10 +3678,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SOptrBasicInfo* pInfo = &pAggInfo->binfo; - int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; - bool newgroup = true; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -3715,6 +3692,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { // setTagValue(pOperator, pAggInfo->current->pTable, pInfo->pCtx, pOperator->numOfExprs); // } + int32_t order = getTableScanOrder(pOperator); + // there is an scalar expression that needs to be calculated before apply the group aggregation. if (pAggInfo->pScalarExprInfo != NULL) { int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, @@ -3747,8 +3726,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } closeAllResultRows(&pAggInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf, - &pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset); + finalizeMultiTupleQueryResult(pOperator->numOfExprs, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo, + pAggInfo->binfo.rowCellInfoOffset); initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false); OPTR_SET_OPENED(pOperator); @@ -4031,7 +4010,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } } - // todo dynamic set tags + // todo set tags + // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // if (pTableQueryInfo != NULL) { // setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfExprs); @@ -4045,7 +4025,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { pTaskInfo->code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, pProjectInfo->pPseudoColInfo); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, pTaskInfo->code); } @@ -4109,8 +4088,8 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { } // todo handle different group data interpolation - bool n = false; - bool *newgroup = &n; + bool n = false; + bool* newgroup = &n; doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); if (pResBlock->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pResBlock->info.rows > 0)) { return pResBlock; @@ -4216,7 +4195,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { } } - taosMemoryFree(pOperator->pExpr); + taosMemoryFreeClear(pOperator->pExpr); taosMemoryFreeClear(pOperator->info); taosMemoryFreeClear(pOperator); } @@ -4234,7 +4213,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n } uint32_t defaultPgsz = 4096; - while(defaultPgsz < pAggSup->resultRowSize*4) { + while (defaultPgsz < pAggSup->resultRowSize * 4) { defaultPgsz <<= 1u; } @@ -4296,11 +4275,8 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i); for (int32_t j = 0; j < taosArrayGetSize(pa); ++j) { STableKeyInfo* pk = taosArrayGet(pa, j); - STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++]; - // pTQueryInfo->uid = pk->uid; pTQueryInfo->lastKey = pk->lastKey; - // pTQueryInfo->groupIndex = i; } } @@ -4319,7 +4295,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - int32_t numOfRows = 1; + int32_t numOfRows = 10; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, numOfRows); @@ -4330,9 +4306,6 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - pOperator->resultInfo.capacity = 4096; - pOperator->resultInfo.threshold = 4096 * 0.75; - int32_t numOfGroup = 10; // todo replaced with true value pInfo->groupId = INT32_MIN; initResultRowInfo(&pInfo->binfo.resultRowInfo, numOfGroup); @@ -4385,7 +4358,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { } void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { - SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*) param; + SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; } void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { @@ -4401,6 +4374,9 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { } static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { + if (NULL == param) { + return; + } SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); cleanupAggSup(&pInfo->aggSup); @@ -4497,8 +4473,9 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* pValueNode, - bool multigroupResult, SExecTaskInfo* pTaskInfo) { + SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, + int32_t fillType, SNodeListNode* pValueNode, bool multigroupResult, + SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -4532,8 +4509,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SResultInfo* pResultInfo = &pOperator->resultInfo; initResultSizeInfo(pOperator, 4096); - int32_t code = initFillInfo(pInfo, pExpr, numOfCols, pValueNode, *pWindow, pResultInfo->capacity, - pTaskInfo->id.str, pInterval, type); + int32_t code = initFillInfo(pInfo, pExpr, numOfCols, pValueNode, *pWindow, pResultInfo->capacity, pTaskInfo->id.str, + pInterval, type); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4558,42 +4535,6 @@ _error: return NULL; } -static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { - int32_t j = 0; - - if (TSDB_COL_IS_TAG(pExpr->pParam[0].pCol->type)) { - if (pExpr->pParam[0].pCol->colId == TSDB_TBNAME_COLUMN_INDEX) { - return TSDB_TBNAME_COLUMN_INDEX; - } - - while (j < pTableInfo->numOfTags) { - if (pExpr->pParam[0].pCol->colId == pTagCols[j].colId) { - return j; - } - - j += 1; - } - - } /*else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { // user specified column data - return TSDB_UD_COLUMN_INDEX; - } else { - while (j < pTableInfo->numOfCols) { - if (pExpr->colInfo.colId == pTableInfo->colList[j].colId) { - return j; - } - - j += 1; - } - }*/ - - return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value -} - -bool validateExprColumnInfo(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { - int32_t j = getColumnIndexInSource(pTableInfo, pExpr, pTagCols); - return j != INT32_MIN; -} - static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision, const char* name) { SResSchema s = {0}; @@ -4746,12 +4687,13 @@ static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t uint64_t queryId, uint64_t taskId); static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractColumnInfo(SNodeList* pNodeList); -static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type); +static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, + int32_t type); static SArray* createSortInfo(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); -static void setJoinColumnInfo(SColumnInfo* pInfo, const SColumnNode* pLeftNode); +static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) { SInterval interval = { @@ -4786,33 +4728,34 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pDescNode); SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); + int32_t code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { return NULL; } - SInterval interval = extractIntervalInfo(pTableScanNode); - SOperatorInfo* pOperator = createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, - pTableScanNode->scanSeq, pColList, pResBlock, pScanPhyNode->node.pConditions, - &interval, pTableScanNode->ratio, pTaskInfo); + SInterval interval = extractIntervalInfo(pTableScanNode); + SOperatorInfo* pOperator = createTableScanOperatorInfo( + pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList, pResBlock, + pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); - return createExchangeOperatorInfo(pExchange->pSrcEndPoints, pResBlock, pTaskInfo); + return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, + queryId, taskId); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - SSDataBlock* pResBlock = createResDataBlock(pDescNode); + SSDataBlock* pResBlock = createResDataBlock(pDescNode); - int32_t numOfCols = 0; - SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); + int32_t numOfCols = 0; + SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, pScanPhyNode->node.pConditions); taosArrayDestroy(tableIdList); @@ -4832,25 +4775,27 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { - STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode; + STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; SSDataBlock* pResBlock = createResDataBlock(pDescNode); - int32_t code = - doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, + queryId, taskId); if (code != TSDB_CODE_SUCCESS) { return NULL; } - int32_t num = 0; + int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); int32_t numOfOutputCols = 0; - SArray* colList = extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, COL_MATCH_FROM_COL_ID); + SArray* colList = + extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, COL_MATCH_FROM_COL_ID); - SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); + SOperatorInfo* pOperator = + createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); return pOperator; } else { ASSERT(0); @@ -4897,7 +4842,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr, pTaskInfo, pTableGroupInfo); } - } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type) { + } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); @@ -4935,13 +4880,15 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType}; + STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, + .calTrigger = pSessionNode->window.triggerType}; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; - pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); + pOptr = + createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode; SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); @@ -4967,11 +4914,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createMergeJoinOperatorInfo(ops, size, pExprInfo, num, pResBlock, pJoinNode->pOnConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { SFillPhysiNode* pFillNode = (SFillPhysiNode*)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SExprInfo* pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num); SInterval* pInterval = &((SIntervalAggOperatorInfo*)ops[0]->info)->interval; - pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, (SNodeListNode*)pFillNode->pValues, false, pTaskInfo); + pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, + (SNodeListNode*)pFillNode->pValues, false, pTaskInfo); } else { ASSERT(0); } @@ -5111,7 +5059,8 @@ SArray* createSortInfo(SNodeList* pNodeList) { return pList; } -SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type) { +SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, + int32_t type) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColMatchInfo)); if (pList == NULL) { @@ -5124,10 +5073,10 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; SColMatchInfo c = {0}; - c.output = true; - c.colId = pColNode->colId; - c.srcSlotId = pColNode->slotId; - c.matchType = type; + c.output = true; + c.colId = pColNode->colId; + c.srcSlotId = pColNode->slotId; + c.matchType = type; c.targetSlotId = pNode->slotId; taosArrayPush(pList, &c); } @@ -5245,28 +5194,6 @@ _complete: return code; } -static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, - int32_t numOfOutput, int32_t tagLen, bool superTable) { - for (int32_t i = 0; i < numOfOutput; ++i) { - int16_t functId = getExprFunctionId(&pExprs[i]); - - if (functId == FUNCTION_TOP || functId == FUNCTION_BOTTOM) { - int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); - if (j < 0 || j >= pTableInfo->numOfCols) { - return TSDB_CODE_QRY_INVALID_MSG; - } else { - SColumnInfo* pCol = &pTableInfo->colList[j]; - // int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i, - // &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, - // &pExprs[i].base.interBytes, tagLen, superTable, NULL); - // assert(ret == TSDB_CODE_SUCCESS); - } - } - } - - return TSDB_CODE_SUCCESS; -} - void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); @@ -5537,8 +5464,8 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { } SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, - SExecTaskInfo* pTaskInfo) { + int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, + SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pOperator == NULL || pInfo == NULL) { @@ -5547,14 +5474,14 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t initResultSizeInfo(pOperator, 4096); - pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; pOperator->numOfExprs = numOfCols; - pOperator->info = pInfo; + pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; SOperatorNode* pNode = (SOperatorNode*)pOnCondition; @@ -5578,9 +5505,9 @@ _error: } void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { - pColumn->slotId = pColumnNode->slotId; - pColumn->type = pColumnNode->node.resType.type; - pColumn->bytes = pColumnNode->node.resType.bytes; + pColumn->slotId = pColumnNode->slotId; + pColumn->type = pColumnNode->node.resType.type; + pColumn->bytes = pColumnNode->node.resType.bytes; pColumn->precision = pColumnNode->node.resType.precision; - pColumn->scale = pColumnNode->node.resType.scale; + pColumn->scale = pColumnNode->node.resType.scale; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 2ba3e257b2..e3a507bf7c 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -304,8 +304,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, + pInfo->binfo.rowCellInfoOffset); // if (!stableQuery) { // finalize include the update of result rows // finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs); // } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c1aee5ba35..eaacb561d5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1057,7 +1057,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { pMsgSendInfo->fp = loadSysTableCallback; int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); + int32_t code = + asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); tsem_wait(&pInfo->ready); if (pTaskInfo->code) { @@ -1182,29 +1183,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe } else { tsem_init(&pInfo->ready, 0, 0); pInfo->epSet = epset; - -#if 1 - { // todo refactor - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "DB-META"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = qProcessFetchRsp; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char*)"root"; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char*)"dcc5bed04851fec854c035b2e40263b6"; - - pInfo->pTransporter = rpcOpen(&rpcInit); - if (pInfo->pTransporter == NULL) { - return NULL; // todo - } - } -#endif + pInfo->readHandle = *(SReadHandle*)readHandle; } pOperator->name = "SysTableScanOperator"; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ebde1c7997..738f4821bd 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -798,8 +798,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { } closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, + pInfo->binfo.rowCellInfoOffset); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); OPTR_SET_OPENED(pOperator); @@ -916,7 +916,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); @@ -1293,7 +1293,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + finalizeMultiTupleQueryResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index ce2f0b0651..a45fdab030 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -25,6 +25,7 @@ extern "C" { bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx)); int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult); EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); @@ -55,6 +56,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevInvertFunction(SqlFunctionCtx* pCtx); +bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool leastSQRFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t leastSQRFunction(SqlFunctionCtx* pCtx); +int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx); + bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index 3869a5d7b2..4d45eb91ce 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -40,6 +40,7 @@ extern "C" { #define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) #define FUNC_MGT_SCAN_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(12) #define FUNC_MGT_SELECT_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(13) +#define FUNC_MGT_REPEAT_SCAN_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(14) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 2be2682bc9..80bfb626d8 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -226,6 +226,23 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (3 != numOfParams) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + for (int32_t i = 0; i < numOfParams; ++i) { + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + } + + pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY }; + return TSDB_CODE_SUCCESS; +} + static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (4 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -242,7 +259,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = 512, .type = TSDB_DATA_TYPE_BINARY }; + pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; } @@ -274,7 +291,8 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } SExprNode* p1 = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); - if (!IS_NUMERIC_TYPE(p1->resType.type)) { + if (!IS_SIGNED_NUMERIC_TYPE(p1->resType.type) && !IS_FLOAT_TYPE(p1->resType.type) && + TSDB_DATA_TYPE_BOOL != p1->resType.type) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } pFunc->node.resType = p1->resType; @@ -536,6 +554,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = stddevFinalize, .invertFunc = stddevInvertFunction }, + { + .name = "leastsquares", + .type = FUNCTION_TYPE_LEASTSQUARES, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateLeastSQR, + .getEnvFunc = getLeastSQRFuncEnv, + .initFunc = leastSQRFunctionSetup, + .processFunc = leastSQRFunction, + .finalizeFunc = leastSQRFinalize, + .invertFunc = leastSQRInvertFunction + }, { .name = "avg", .type = FUNCTION_TYPE_AVG, @@ -550,7 +579,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "percentile", .type = FUNCTION_TYPE_PERCENTILE, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC, .translateFunc = translatePercentile, .getEnvFunc = getPercentileFuncEnv, .initFunc = percentileFunctionSetup, @@ -958,6 +987,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = NULL, .finalizeFunc = NULL }, + { + .name = "_c0", + .type = FUNCTION_TYPE_ROWTS, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + .translateFunc = translateTimePseudoColumn, + .getEnvFunc = getTimePseudoFuncEnv, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL + }, { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, @@ -1035,7 +1074,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateSelectValue, .getEnvFunc = getSelectivityFuncEnv, // todo remove this function later. .initFunc = functionSetup, - .sprocessFunc = NULL, + .processFunc = NULL, .finalizeFunc = NULL } }; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1cb47a0bf1..effc4888a8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -65,6 +65,13 @@ typedef struct SStddevRes { }; } SStddevRes; +typedef struct SLeastSQRInfo { + double matrix[2][3]; + double startVal; + double stepVal; + int64_t num; +} SLeastSQRInfo; + typedef struct SPercentileInfo { double result; tMemBucket* pMemBucket; @@ -105,6 +112,7 @@ typedef struct SHistoFuncBin { typedef struct SHistoFuncInfo { int32_t numOfBins; + int32_t totalCount; bool normalized; SHistoFuncBin bins[]; } SHistoFuncInfo; @@ -176,7 +184,6 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; - /*cleanupResultRowEntry(pResInfo);*/ char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -184,6 +191,10 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx)) { + return 0; +} + int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -1404,6 +1415,181 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +bool getLeastSQRFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SLeastSQRInfo); + return true; +} + +bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + + pInfo->startVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[1].param.d : + (double)pCtx->param[1].param.i; + pInfo->stepVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[2].param.d : + (double)pCtx->param[1].param.i; + return true; +} + +#define LEASTSQR_CAL(p, x, y, index, step) \ + do { \ + (p)[0][0] += (double)(x) * (x); \ + (p)[0][1] += (double)(x); \ + (p)[0][2] += (double)(x) * (y)[index]; \ + (p)[1][2] += (y)[index]; \ + (x) += step; \ + } while (0) + +int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { + int32_t numOfElem = 0; + + SInputColumnInfoData* pInput = &pCtx->input; + int32_t type = pInput->pData[0]->info.type; + + SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + SColumnInfoData* pCol = pInput->pData[0]; + + double(*param)[3] = pInfo->matrix; + double x = pInfo->startVal; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* plist = (int8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + + break; + } + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* plist = (int16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t* plist = (int64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + float* plist = (float*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* plist = (double*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + default: + break; + } + + pInfo->startVal = x; + pInfo->num += numOfElem; + + SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + int32_t currentRow = pBlock->info.rows; + + if (0 == pInfo->num) { + return 0; + } + + double(*param)[3] = pInfo->matrix; + + param[1][1] = (double)pInfo->num; + param[1][0] = param[0][1]; + + param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); + param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); + param[0][1] = 0; + param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); + param[1][0] = 0; + param[0][2] /= param[0][0]; + + param[1][2] /= param[1][1]; + + char buf[64] = {0}; + size_t len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%.6lf, intercept:%.6lf}", param[0][2], param[1][2]); + varDataSetLen(buf, len); + + colDataAppend(pCol, currentRow, buf, false); + + return pResInfo->numOfRes; +} + +int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx) { + //TODO + return TSDB_CODE_SUCCESS; +} + bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SPercentileInfo); return true; @@ -2398,6 +2584,9 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultIn } SHistoFuncInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + pInfo->numOfBins = 0; + pInfo->totalCount = 0; + pInfo->normalized = 0; int8_t binType = getHistogramBinType(varDataVal(pCtx->param[1].param.pz)); if (binType == UNKNOWN_BIN) { @@ -2427,7 +2616,6 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { int32_t numOfRows = pInput->numOfRows; int32_t numOfElems = 0; - int32_t totalElems = 0; for (int32_t i = start; i < numOfRows + start; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -2442,23 +2630,13 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { for (int32_t k = 0; k < pInfo->numOfBins; ++k) { if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) { pInfo->bins[k].count++; - totalElems++; + pInfo->totalCount++; break; } } } - if (pInfo->normalized) { - for (int32_t k = 0; k < pInfo->numOfBins; ++k) { - if(totalElems != 0) { - pInfo->bins[k].percentage = pInfo->bins[k].count / (double)totalElems; - } else { - pInfo->bins[k].percentage = 0; - } - } - } - SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); return TSDB_CODE_SUCCESS; } @@ -2471,14 +2649,24 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; + if (pInfo->normalized) { + for (int32_t k = 0; k < pResInfo->numOfRes; ++k) { + if(pInfo->totalCount != 0) { + pInfo->bins[k].percentage = pInfo->bins[k].count / (double)pInfo->totalCount; + } else { + pInfo->bins[k].percentage = 0; + } + } + } + for (int32_t i = 0; i < pResInfo->numOfRes; ++i) { int32_t len; char buf[512] = {0}; if (!pInfo->normalized) { - len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}", + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}", pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count); } else { - len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].percentage); } varDataSetLen(buf, len); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 46bbb33aa7..73ec7f510b 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -169,6 +169,8 @@ bool fmIsDynamicScanOptimizedFunc(int32_t funcId) { bool fmIsMultiResFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_RES_FUNC); } +bool fmIsRepeatScanFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_REPEAT_SCAN_FUNC); } + bool fmIsUserDefinedFunc(int32_t funcId) { return funcId > FUNC_UDF_ID_START; } void fmFuncMgtDestroy() { @@ -197,15 +199,14 @@ int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet) { bool fmIsInvertible(int32_t funcId) { bool res = false; switch (funcMgtBuiltins[funcId].type) { - case FUNCTION_TYPE_COUNT: - case FUNCTION_TYPE_SUM: - case FUNCTION_TYPE_STDDEV: - case FUNCTION_TYPE_AVG: - res = true; - break; - default: - break; + case FUNCTION_TYPE_COUNT: + case FUNCTION_TYPE_SUM: + case FUNCTION_TYPE_STDDEV: + case FUNCTION_TYPE_AVG: + res = true; + break; + default: + break; } return res; } - diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 11502b4c47..a577ea200f 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -18,6 +18,7 @@ #include "tudf.h" #include "tudfInt.h" #include "tarray.h" +#include "tglobal.h" #include "tdatablock.h" #include "querynodes.h" #include "builtinsimpl.h" @@ -138,6 +139,10 @@ static void udfWatchUdfd(void *args) { } int32_t udfStartUdfd(int32_t startDnodeId) { + if (!tsStartUdfd) { + fnInfo("start udfd is disabled.") + return 0; + } SUdfdData *pData = &udfdGlobal; if (pData->startCalled) { fnInfo("dnode-mgmt start udfd already called"); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 7695598fb8..0ad4674cfa 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -409,7 +409,7 @@ void udfdPipeCloseCb(uv_handle_t *pipe) { void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); } void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - fnDebug("udf read %zu bytes from client", nread); + fnDebug("udf read %zd bytes from client", nread); if (nread == 0) return; SUdfdUvConn *conn = client->data; diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 3b07429089..4cab71f92c 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -20,11 +20,23 @@ extern "C" { #endif +#include "indexInt.h" +#include "tcompare.h" + extern char JSON_COLUMN[]; extern char JSON_VALUE_DELIM; char* indexPackJsonData(SIndexTerm* itm); char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); + +typedef enum { MATCH, CONTINUE, BREAK } TExeCond; + +typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); + +TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b); + +_cache_range_compare indexGetCompare(RangeType ty); + #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 929f33909e..0653c1d1fa 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -60,50 +60,6 @@ static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, RangeType type); -typedef enum { MATCH, CONTINUE, BREAK } TExeCond; -typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); - -static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { - // optime later - int32_t ret = func(a, b); - switch (comType) { - case QUERY_LESS_THAN: { - if (ret < 0) return MATCH; - } break; - case QUERY_LESS_EQUAL: { - if (ret <= 0) return MATCH; - break; - } - case QUERY_GREATER_THAN: { - if (ret > 0) return MATCH; - break; - } - case QUERY_GREATER_EQUAL: { - if (ret >= 0) return MATCH; - } - } - return CONTINUE; -} -static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_LESS_THAN, a, b); -} -static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); -} -static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_GREATER_THAN, a, b); -} -static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { - __compar_fn_t func = getComparFunc(type, 0); - return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); -} - -static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, - tCompareGreaterThan, tCompareGreaterEqual}; - static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = { {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}, @@ -169,7 +125,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes return 0; } - _cache_range_compare cmpFn = rangeCompare[type]; + _cache_range_compare cmpFn = indexGetCompare(type); MemTable* mem = cache; IndexCache* pCache = mem->pCache; @@ -295,7 +251,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe if (cache == NULL) { return 0; } - _cache_range_compare cmpFn = rangeCompare[type]; + _cache_range_compare cmpFn = indexGetCompare(type); MemTable* mem = cache; IndexCache* pCache = mem->pCache; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index cdd7b35675..9e85a6680a 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -13,12 +13,58 @@ * along with this program. If not, see . */ +#include "indexComm.h" #include "index.h" #include "indexInt.h" +#include "tcompare.h" char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_THAN, a, b); +} +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); +} +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_THAN, a, b); +} +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); +} + +TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { + // optime later + int32_t ret = func(a, b); + switch (comType) { + case QUERY_LESS_THAN: { + if (ret < 0) return MATCH; + } break; + case QUERY_LESS_EQUAL: { + if (ret <= 0) return MATCH; + break; + } + case QUERY_GREATER_THAN: { + if (ret > 0) return MATCH; + break; + } + case QUERY_GREATER_EQUAL: { + if (ret >= 0) return MATCH; + } + } + return CONTINUE; +} + +static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, + tCompareGreaterThan, tCompareGreaterEqual}; + +_cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; } + char* indexPackJsonData(SIndexTerm* itm) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| @@ -46,6 +92,7 @@ char* indexPackJsonData(SIndexTerm* itm) { return buf; } + char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index b5551e825f..c56d65fc6a 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -72,9 +72,23 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype); -static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { - tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, - tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}; +static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr); + +static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype); + +static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { + {tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual, + tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}, + {tfSearchTerm_JSON, tfSearchPrefix_JSON, tfSearchSuffix_JSON, tfSearchRegex_JSON, tfSearchLessThan_JSON, + tfSearchLessEqual_JSON, tfSearchGreaterThan_JSON, tfSearchGreaterEqual_JSON, tfSearchRange_JSON}}; TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); @@ -202,14 +216,10 @@ void tfileReaderDestroy(TFileReader* reader) { taosMemoryFree(reader); } static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); int ret = 0; char* p = tem->colVal; uint64_t sz = tem->nColVal; - if (hasJson) { - p = indexPackJsonData(tem); - sz = strlen(p); - } + int64_t st = taosGetTimestampUs(); FstSlice key = fstSliceCreate(p, sz); uint64_t offset; @@ -224,9 +234,6 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, tem->colName, tem->colVal, cost); } - if (hasJson) { - taosMemoryFree(p); - } fstSliceDestroy(&key); return 0; } @@ -308,14 +315,11 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) } static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); - int ret = 0; - char* p = tem->colVal; - int skip = 0; + int ret = 0; + char* p = tem->colVal; + int skip = 0; + _cache_range_compare cmpFn = indexGetCompare(type); - if (hasJson) { - p = indexPackJsonDataPrefix(tem, &skip); - } SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); @@ -328,7 +332,16 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { - taosArrayPush(offsets, &(rt->out.out)); + FstSlice* s = &rt->data; + char* ch = (char*)fstSliceData(s, NULL); + TExeCond cond = cmpFn(ch, p, tem->colType); + if (MATCH == cond) { + tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); + } else if (CONTINUE == cond) { + } else if (BREAK == cond) { + swsResultDestroy(rt); + break; + } swsResultDestroy(rt); } streamWithStateDestroy(st); @@ -376,17 +389,105 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) fstSliceDestroy(&key); return 0; } +static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + int ret = 0; + char* p = indexPackJsonData(tem); + int sz = strlen(p); + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + uint64_t offset; + if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + int64_t et = taosGetTimestampUs(); + int64_t cost = et - st; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + tem->suid, tem->colName, tem->colVal, cost); + + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + cost = taosGetTimestampUs() - et; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + tem->colName, tem->colVal, cost); + } + fstSliceDestroy(&key); + return 0; + // deprecate api + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} +static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, LT); +} +static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, LE); +} +static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, GT); +} +static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + return tfSearchCompareFunc_JSON(reader, tem, tr, GE); +} +static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return TSDB_CODE_SUCCESS; +} + +static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype) { + int ret = 0; + int skip = 0; + + char* p = indexPackJsonDataPrefix(tem, &skip); + + _cache_range_compare cmpFn = indexGetCompare(ctype); + + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + + FstSlice h = fstSliceCreate((uint8_t*)p, skip); + fstStreamBuilderSetRange(sb, &h, ctype); + fstSliceDestroy(&h); + + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + FstSlice* s = &rt->data; + char* ch = (char*)fstSliceData(s, NULL); + TExeCond cond = cmpFn(ch, p, tem->colType); + if (MATCH == cond) { + tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); + } else if (CONTINUE == cond) { + } else if (BREAK == cond) { + swsResultDestroy(rt); + break; + } + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + return TSDB_CODE_SUCCESS; +} int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - if (qtype >= sizeof(tfSearch) / sizeof(tfSearch[0])) { - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, - term->colVal); - return -1; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + return tfSearch[1][qtype](reader, term, tr); } else { - return tfSearch[qtype](reader, term, tr); + return tfSearch[0][qtype](reader, term, tr); } + tfileReaderUnRef(reader); return 0; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 507cd79411..a625fc0d0c 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -222,6 +222,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + return "PhysiStreamInterval"; case QUERY_NODE_PHYSICAL_PLAN_FILL: return "PhysiFill"; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -2893,6 +2895,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_FILL: return physiFillNodeToJson(pObj, pJson); @@ -2983,6 +2986,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_FILL: return jsonToPhysiFillNode(pJson, pObj); @@ -3099,6 +3103,7 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode) { return TSDB_CODE_FAILED; } int32_t code = makeNodeByJson(pJson, pNode); + tjsonDelete(pJson); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode(*pNode); *pNode = NULL; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index f2c043b9ea..e8274c3c8e 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -513,6 +513,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk break; } case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index eeb069383f..af7ebf5d10 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -252,6 +252,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + return makeNode(type, sizeof(SStreamIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_FILL: return makeNode(type, sizeof(SFillPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -644,6 +646,7 @@ void nodesDestroyNode(SNodeptr pNode) { break; } case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index e8ac562072..a9481593e0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -907,6 +907,13 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S return (SNode*)pStmt; } +static SNode* createAlterTableStmtFinalize(SNode* pRealTable, SAlterTableStmt* pStmt) { + strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName); + strcpy(pStmt->tableName, ((SRealTableNode*)pRealTable)->table.tableName); + nodesDestroyNode(pRealTable); + return (SNode*)pStmt; +} + SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions) { if (NULL == pRealTable) { return NULL; @@ -915,7 +922,7 @@ SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, CHECK_OUT_OF_MEM(pStmt); pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_OPTIONS; pStmt->pOptions = (STableOptions*)pOptions; - return (SNode*)pStmt; + return createAlterTableStmtFinalize(pRealTable, pStmt); } SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, @@ -928,7 +935,7 @@ SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, pStmt->alterType = alterType; strncpy(pStmt->colName, pColName->z, pColName->n); pStmt->dataType = dataType; - return (SNode*)pStmt; + return createAlterTableStmtFinalize(pRealTable, pStmt); } SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, const SToken* pColName) { @@ -939,7 +946,7 @@ SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_ CHECK_OUT_OF_MEM(pStmt); pStmt->alterType = alterType; strncpy(pStmt->colName, pColName->z, pColName->n); - return (SNode*)pStmt; + return createAlterTableStmtFinalize(pRealTable, pStmt); } SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, @@ -952,7 +959,7 @@ SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int pStmt->alterType = alterType; strncpy(pStmt->colName, pOldColName->z, pOldColName->n); strncpy(pStmt->newColName, pNewColName->z, pNewColName->n); - return (SNode*)pStmt; + return createAlterTableStmtFinalize(pRealTable, pStmt); } SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const SToken* pTagName, SNode* pVal) { @@ -964,7 +971,7 @@ SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, const pStmt->alterType = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; strncpy(pStmt->colName, pTagName->z, pTagName->n); pStmt->pVal = (SValueNode*)pVal; - return (SNode*)pStmt; + return createAlterTableStmtFinalize(pRealTable, pStmt); } SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index ee1a92d8b3..ebc8281f56 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -64,8 +64,8 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { goto abort_parse; } default: - Parse(pParser, t0.type, t0, &cxt); // ParseTrace(stdout, ""); + Parse(pParser, t0.type, t0, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { goto abort_parse; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 11dfe60015..64d2934282 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -241,6 +241,15 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool SParseContext* pBasicCtx = pCxt->pComCxt; SName name = {0}; createSName(&name, pTname, pBasicCtx->acctId, pBasicCtx->db, &pCxt->msg); + + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + + bool pass = false; + CHECK_CODE(catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, dbFname, AUTH_TYPE_WRITE, &pass)); + if (!pass) { + return TSDB_CODE_PAR_PERMISSION_DENIED; + } if (isStb) { CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); @@ -1151,6 +1160,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { (*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); pCxt->pVgroupsHashObj = NULL; pCxt->pTableBlockHashObj = NULL; + pCxt->pTableMeta = NULL; return TSDB_CODE_SUCCESS; } @@ -1276,7 +1286,7 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tN return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = pDataBlock->pTableMeta->schema; SKvParam param = {.builder = &tagBuilder}; for (int c = 0; c < tags->numOfBound; ++c) { diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index c5c4463a95..c074334722 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -469,7 +469,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p // the maximum expanded size in byte when a row-wise data is converted to SDataRow format int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen; if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); @@ -601,6 +601,7 @@ int32_t qResetStmtDataBlock(void* block, bool keepBuf) { pBlock->numOfTables = 1; pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; pBlock->headerSize = pBlock->size; + pBlock->createTbReqLen = 0; memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 54a0fce3e7..abc7ddb17f 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -214,6 +214,7 @@ static SKeyword keywordTable[] = { {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, {"WITH", TK_WITH}, {"WRITE", TK_WRITE}, + {"_C0", TK_ROWTS}, {"_QENDTS", TK_QENDTS}, {"_QSTARTTS", TK_QSTARTTS}, {"_ROWTS", TK_ROWTS}, @@ -590,6 +591,8 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { if (seg == 4) { // ip address *tokenId = TK_NK_IPTOKEN; return i; + } else if (seg > 2) { + break; } if ((z[i] == 'e' || z[i] == 'E') && diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1bd7e28c74..cdcb2592a7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -699,6 +699,10 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) if (isCountStar(pFunc)) { pCxt->errCode = rewriteCountStar(pCxt, pFunc); } + + if (fmIsRepeatScanFunc(pFunc->funcId)) { + pCxt->pCurrStmt->hasRepeatScanFuncs = true; + } } return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } @@ -778,6 +782,7 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSel return DEAL_RES_ERROR; } strcpy(pFunc->functionName, "_select_value"); + strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); if (TSDB_CODE_SUCCESS == pCxt->errCode) { translateFunction(pCxt, pFunc); @@ -2255,8 +2260,8 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SN code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); } if (TSDB_CODE_SUCCESS == code) { - if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_BINARY_LEN) || - (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_NCHAR_LEN)) { + if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN)) { code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } } @@ -2536,10 +2541,18 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm pReq->igExists = pStmt->ignoreExists; pReq->xFilesFactor = pStmt->pOptions->filesFactor; pReq->delay = pStmt->pOptions->delay; + pReq->ttl = pStmt->pOptions->ttl; columnDefNodeToField(pStmt->pCols, &pReq->pColumns); columnDefNodeToField(pStmt->pTags, &pReq->pTags); pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); pReq->numOfTags = LIST_LENGTH(pStmt->pTags); + if ('\0' != pStmt->pOptions->comment[0]) { + pReq->comment = strdup(pStmt->pOptions->comment); + if (NULL == pReq->comment) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->commentLen = strlen(pStmt->pOptions->comment) + 1; + } SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); @@ -2575,7 +2588,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt SName tableName; int32_t code = getTableMetaImpl( pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); - if ((TSDB_CODE_TDB_INVALID_TABLE_ID == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) { + if ((TSDB_CODE_PAR_TABLE_NOT_EXIST == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) { return TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code) { @@ -2598,6 +2611,18 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS } static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { + if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) { + pAlterReq->ttl = pStmt->pOptions->ttl; + if ('\0' != pStmt->pOptions->comment[0]) { + pAlterReq->comment = strdup(pStmt->pOptions->comment); + if (NULL == pAlterReq->comment) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pAlterReq->commentLen = strlen(pStmt->pOptions->comment) + 1; + } + return TSDB_CODE_SUCCESS; + } + pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); if (NULL == pAlterReq->pFields) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2610,7 +2635,7 @@ static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAlterStbReq* pAlterR case TSDB_ALTER_TABLE_DROP_COLUMN: case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: { - TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = pStmt->dataType.bytes}; + TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; strcpy(field.name, pStmt->colName); taosArrayPush(pAlterReq->pFields, &field); break; @@ -2621,7 +2646,7 @@ static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAlterStbReq* pAlterR strcpy(oldField.name, pStmt->colName); taosArrayPush(pAlterReq->pFields, &oldField); TAOS_FIELD newField = {0}; - strcpy(oldField.name, pStmt->newColName); + strcpy(newField.name, pStmt->newColName); taosArrayPush(pAlterReq->pFields, &newField); break; } @@ -2638,7 +2663,7 @@ static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pSt SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), alterReq.name); alterReq.alterType = pStmt->alterType; - if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType || TSDB_ALTER_TABLE_UPDATE_TAG_VAL == pStmt->alterType) { + if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == pStmt->alterType) { return TSDB_CODE_FAILED; } else { if (TSDB_CODE_SUCCESS != setAlterTableField(pStmt, &alterReq)) { @@ -2925,7 +2950,6 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS SName name; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); tNameGetFullDbName(&name, pReq->name); - /*tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name);*/ pReq->igExists = pStmt->ignoreExists; pReq->withTbName = pStmt->pOptions->withTable; pReq->withSchema = pStmt->pOptions->withSchema; @@ -2989,7 +3013,8 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt SMDropTopicReq dropReq = {0}; SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), dropReq.name); + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); + tNameGetFullDbName(&name, dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); @@ -3029,28 +3054,48 @@ static int32_t translateKillTransaction(STranslateContext* pCxt, SKillStmt* pStm return buildCmdMsg(pCxt, TDMT_MND_KILL_TRANS, (FSerializeFunc)tSerializeSKillTransReq, &killReq); } -static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { - SCMCreateStreamReq createReq = {0}; +static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { + if (NULL == pStmt->pQuery) { + return TSDB_CODE_SUCCESS; + } - createReq.igExists = pStmt->ignoreExists; + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable)) { + return TSDB_CODE_SUCCESS; + } + } + + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY); +} + +static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { + SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; + strcpy(name.dbname, ((SRealTableNode*)(((SSelectStmt*)pStmt)->pFromTable))->table.dbName); + tNameGetFullDbName(&name, pDbFName); +} + +static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { + pReq->igExists = pStmt->ignoreExists; SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->streamName, &name), createReq.name); + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->streamName, &name), pReq->name); if ('\0' != pStmt->targetTabName[0]) { strcpy(name.dbname, pStmt->targetDbName); strcpy(name.tname, pStmt->targetTabName); - tNameExtractFullName(&name, createReq.targetStbFullName); + tNameExtractFullName(&name, pReq->targetStbFullName); } int32_t code = translateQuery(pCxt, pStmt->pQuery); if (TSDB_CODE_SUCCESS == code) { - code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL); + getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); + code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } if (TSDB_CODE_SUCCESS == code) { - createReq.sql = strdup(pCxt->pParseCxt->pSql); - if (NULL == createReq.sql) { + pReq->sql = strdup(pCxt->pParseCxt->pSql); + if (NULL == pReq->sql) { code = TSDB_CODE_OUT_OF_MEMORY; } } @@ -3060,11 +3105,20 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* : TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code) { - createReq.triggerType = pStmt->pOptions->triggerType; - createReq.watermark = - (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); + pReq->triggerType = pStmt->pOptions->triggerType; + pReq->watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); } + return code; +} + +static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { + SCMCreateStreamReq createReq = {0}; + + int32_t code = checkCreateStream(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = buildCreateStreamReq(pCxt, pStmt, &createReq); + } if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_STREAM, (FSerializeFunc)tSerializeSCMCreateStreamReq, &createReq); } @@ -3146,7 +3200,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB; } strcpy(req.user, pStmt->userName); - strcpy(req.dbname, pStmt->dbName); + sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); } @@ -3162,7 +3216,7 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB; } strcpy(req.user, pStmt->userName); - strcpy(req.dbname, pStmt->dbName); + sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); } diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index 5bd9eb5c43..4ddb7736b2 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -19,7 +19,7 @@ using namespace std; namespace ParserTest { -class ParserInitialATest : public ParserTestBase {}; +class ParserInitialATest : public ParserDdlTest {}; TEST_F(ParserInitialATest, alterAccount) { useDb("root", "test"); @@ -72,16 +72,103 @@ TEST_F(ParserInitialATest, alterDatabase) { TEST_F(ParserInitialATest, alterTable) { useDb("root", "test"); - // run("ALTER TABLE t1 TTL 10"); - // run("ALTER TABLE t1 COMMENT 'test'"); + SMAlterStbReq expect = {0}; + + auto setAlterStbReqFunc = [&](const char* pTbname, int8_t alterType, int32_t numOfFields = 0, + const char* pField1Name = nullptr, int8_t field1Type = 0, int32_t field1Bytes = 0, + const char* pField2Name = nullptr, const char* pComment = nullptr, + int32_t ttl = TSDB_DEFAULT_TABLE_TTL) { + int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname); + expect.name[len] = '\0'; + expect.alterType = alterType; + expect.ttl = ttl; + if (nullptr != pComment) { + expect.comment = strdup(pComment); + expect.commentLen = strlen(pComment) + 1; + } + + expect.numOfFields = numOfFields; + if (NULL == expect.pFields) { + expect.pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); + TAOS_FIELD field = {0}; + taosArrayPush(expect.pFields, &field); + taosArrayPush(expect.pFields, &field); + } + + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 0); + if (NULL != pField1Name) { + strcpy(pField->name, pField1Name); + pField->name[strlen(pField1Name)] = '\0'; + } else { + memset(pField, 0, sizeof(TAOS_FIELD)); + } + pField->type = field1Type; + pField->bytes = field1Bytes > 0 ? field1Bytes : (field1Type > 0 ? tDataTypes[field1Type].bytes : 0); + + pField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 1); + if (NULL != pField2Name) { + strcpy(pField->name, pField2Name); + pField->name[strlen(pField2Name)] = '\0'; + } else { + memset(pField, 0, sizeof(TAOS_FIELD)); + } + pField->type = 0; + pField->bytes = 0; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_TABLE_STMT); + SMAlterStbReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(req.alterType, expect.alterType); + ASSERT_EQ(req.numOfFields, expect.numOfFields); + if (expect.numOfFields > 0) { + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(req.pFields, 0); + TAOS_FIELD* pExpectField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 0); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + } + if (expect.numOfFields > 1) { + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(req.pFields, 1); + TAOS_FIELD* pExpectField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 1); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + } + }); + + setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10); + run("ALTER TABLE t1 TTL 10"); + + setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test"); + run("ALTER TABLE t1 COMMENT 'test'"); + + setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, 1, "cc1", TSDB_DATA_TYPE_BIGINT); run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT"); + + setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_DROP_COLUMN, 1, "c1"); run("ALTER TABLE t1 DROP COLUMN c1"); + + setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c1", TSDB_DATA_TYPE_VARCHAR, + 20 + VARSTR_HEADER_SIZE); run("ALTER TABLE t1 MODIFY COLUMN c1 VARCHAR(20)"); + + setAlterStbReqFunc("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1"); run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_ADD_TAG, 1, "tag11", TSDB_DATA_TYPE_BIGINT); run("ALTER TABLE st1 ADD TAG tag11 BIGINT"); + + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_DROP_TAG, 1, "tag1"); run("ALTER TABLE st1 DROP TAG tag1"); + + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, 1, "tag1", TSDB_DATA_TYPE_VARCHAR, + 20 + VARSTR_HEADER_SIZE); run("ALTER TABLE st1 MODIFY TAG tag1 VARCHAR(20)"); + + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_NAME, 2, "tag1", 0, 0, "tag11"); run("ALTER TABLE st1 RENAME TAG tag1 tag11"); // run("ALTER TABLE st1s1 SET TAG tag1=10"); diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 540560fcd7..cf364aba5c 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -19,7 +19,7 @@ using namespace std; namespace ParserTest { -class ParserInitialCTest : public ParserTestBase {}; +class ParserInitialCTest : public ParserDdlTest {}; // todo compact @@ -97,17 +97,143 @@ TEST_F(ParserInitialCTest, createSnode) { TEST_F(ParserInitialCTest, createStable) { useDb("root", "test"); + SMCreateStbReq expect = {0}; + + auto setCreateStbReqFunc = [&](const char* pTbname, int8_t igExists = 0, + float xFilesFactor = TSDB_DEFAULT_ROLLUP_FILE_FACTOR, + int32_t delay = TSDB_DEFAULT_ROLLUP_DELAY, int32_t ttl = TSDB_DEFAULT_TABLE_TTL, + const char* pComment = nullptr) { + memset(&expect, 0, sizeof(SMCreateStbReq)); + int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname); + expect.name[len] = '\0'; + expect.igExists = igExists; + expect.xFilesFactor = xFilesFactor; + expect.delay = delay; + expect.ttl = ttl; + if (nullptr != pComment) { + expect.comment = strdup(pComment); + expect.commentLen = strlen(pComment) + 1; + } + }; + + auto addFieldToCreateStbReqFunc = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, + int8_t flags = SCHEMA_SMA_ON) { + SField field = {0}; + strcpy(field.name, pFieldName); + field.type = type; + field.bytes = bytes > 0 ? bytes : tDataTypes[type].bytes; + field.flags = flags; + + if (col) { + if (NULL == expect.pColumns) { + expect.pColumns = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SField)); + } + taosArrayPush(expect.pColumns, &field); + expect.numOfColumns += 1; + } else { + if (NULL == expect.pTags) { + expect.pTags = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SField)); + } + taosArrayPush(expect.pTags, &field); + expect.numOfTags += 1; + } + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_TABLE_STMT); + SMCreateStbReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMCreateStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(req.igExists, expect.igExists); + ASSERT_EQ(req.xFilesFactor, expect.xFilesFactor); + ASSERT_EQ(req.delay, expect.delay); + ASSERT_EQ(req.ttl, expect.ttl); + ASSERT_EQ(req.numOfColumns, expect.numOfColumns); + ASSERT_EQ(req.numOfTags, expect.numOfTags); + ASSERT_EQ(req.commentLen, expect.commentLen); + ASSERT_EQ(req.ast1Len, expect.ast1Len); + ASSERT_EQ(req.ast2Len, expect.ast2Len); + + if (expect.numOfColumns > 0) { + ASSERT_EQ(taosArrayGetSize(req.pColumns), expect.numOfColumns); + ASSERT_EQ(taosArrayGetSize(req.pColumns), taosArrayGetSize(expect.pColumns)); + for (int32_t i = 0; i < expect.numOfColumns; ++i) { + SField* pField = (SField*)taosArrayGet(req.pColumns, i); + SField* pExpectField = (SField*)taosArrayGet(expect.pColumns, i); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + ASSERT_EQ(pField->flags, pExpectField->flags); + } + } + if (expect.numOfTags > 0) { + ASSERT_EQ(taosArrayGetSize(req.pTags), expect.numOfTags); + ASSERT_EQ(taosArrayGetSize(req.pTags), taosArrayGetSize(expect.pTags)); + for (int32_t i = 0; i < expect.numOfTags; ++i) { + SField* pField = (SField*)taosArrayGet(req.pTags, i); + SField* pExpectField = (SField*)taosArrayGet(expect.pTags, i); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + ASSERT_EQ(pField->flags, pExpectField->flags); + } + } + if (expect.commentLen > 0) { + ASSERT_EQ(std::string(req.comment), std::string(expect.comment)); + } + if (expect.ast1Len > 0) { + ASSERT_EQ(std::string(req.pAst1), std::string(expect.pAst1)); + } + if (expect.ast2Len > 0) { + ASSERT_EQ(std::string(req.pAst2), std::string(expect.pAst2)); + } + }); + + setCreateStbReqFunc("t1"); + addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP); + addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReqFunc(false, "id", TSDB_DATA_TYPE_INT); run("create stable t1(ts timestamp, c1 int) TAGS(id int)"); + setCreateStbReqFunc("t1", 1, 0.1, 2, 100, "test create table"); + addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0); + addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReqFunc(true, "c2", TSDB_DATA_TYPE_UINT); + addFieldToCreateStbReqFunc(true, "c3", TSDB_DATA_TYPE_BIGINT); + addFieldToCreateStbReqFunc(true, "c4", TSDB_DATA_TYPE_UBIGINT, 0, 0); + addFieldToCreateStbReqFunc(true, "c5", TSDB_DATA_TYPE_FLOAT, 0, 0); + addFieldToCreateStbReqFunc(true, "c6", TSDB_DATA_TYPE_DOUBLE, 0, 0); + addFieldToCreateStbReqFunc(true, "c7", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReqFunc(true, "c8", TSDB_DATA_TYPE_SMALLINT, 0, 0); + addFieldToCreateStbReqFunc(true, "c9", TSDB_DATA_TYPE_USMALLINT, 0, 0); + addFieldToCreateStbReqFunc(true, "c10", TSDB_DATA_TYPE_TINYINT, 0, 0); + addFieldToCreateStbReqFunc(true, "c11", TSDB_DATA_TYPE_UTINYINT, 0, 0); + addFieldToCreateStbReqFunc(true, "c12", TSDB_DATA_TYPE_BOOL, 0, 0); + addFieldToCreateStbReqFunc(true, "c13", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReqFunc(true, "c14", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReqFunc(false, "a1", TSDB_DATA_TYPE_TIMESTAMP); + addFieldToCreateStbReqFunc(false, "a2", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReqFunc(false, "a3", TSDB_DATA_TYPE_UINT); + addFieldToCreateStbReqFunc(false, "a4", TSDB_DATA_TYPE_BIGINT); + addFieldToCreateStbReqFunc(false, "a5", TSDB_DATA_TYPE_UBIGINT); + addFieldToCreateStbReqFunc(false, "a6", TSDB_DATA_TYPE_FLOAT); + addFieldToCreateStbReqFunc(false, "a7", TSDB_DATA_TYPE_DOUBLE); + addFieldToCreateStbReqFunc(false, "a8", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE); + addFieldToCreateStbReqFunc(false, "a9", TSDB_DATA_TYPE_SMALLINT); + addFieldToCreateStbReqFunc(false, "a10", TSDB_DATA_TYPE_USMALLINT); + addFieldToCreateStbReqFunc(false, "a11", TSDB_DATA_TYPE_TINYINT); + addFieldToCreateStbReqFunc(false, "a12", TSDB_DATA_TYPE_UTINYINT); + addFieldToCreateStbReqFunc(false, "a13", TSDB_DATA_TYPE_BOOL); + addFieldToCreateStbReqFunc(false, "a14", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + addFieldToCreateStbReqFunc(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); run("create stable if not exists test.t1(" - "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), c8 " - "SMALLINT, " - "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), " - "c15 VARCHAR(50)) " - "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 " - "BINARY(20), a8 SMALLINT, " - "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), " - "a15 VARCHAR(50)) " + "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " + "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c13 NCHAR(30), c14 VARCHAR(50)) " + "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " + "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, " + "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"); } diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index a9ff756a95..250ac1c528 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -49,6 +49,8 @@ struct TerminateFlag : public exception { class ParserTestBaseImpl { public: + ParserTestBaseImpl(ParserTestBase* pBase) : pBase_(pBase) {} + void useDb(const string& acctId, const string& db) { caseEnv_.acctId_ = acctId; caseEnv_.db_ = db; @@ -156,11 +158,13 @@ class ParserTestBaseImpl { void doParse(SParseContext* pCxt, SQuery** pQuery) { DO_WITH_THROW(parse, pCxt, pQuery); + ASSERT_NE(*pQuery, nullptr); res_.parsedAst_ = toString((*pQuery)->pRoot); } void doTranslate(SParseContext* pCxt, SQuery* pQuery) { DO_WITH_THROW(translate, pCxt, pQuery); + checkQuery(pQuery, PARSER_STAGE_TRANSLATE); res_.translatedAst_ = toString(pQuery->pRoot); } @@ -178,12 +182,15 @@ class ParserTestBaseImpl { return str; } - caseEnv caseEnv_; - stmtEnv stmtEnv_; - stmtRes res_; + void checkQuery(const SQuery* pQuery, ParserStage stage) { pBase_->checkDdl(pQuery, stage); } + + caseEnv caseEnv_; + stmtEnv stmtEnv_; + stmtRes res_; + ParserTestBase* pBase_; }; -ParserTestBase::ParserTestBase() : impl_(new ParserTestBaseImpl()) {} +ParserTestBase::ParserTestBase() : impl_(new ParserTestBaseImpl(this)) {} ParserTestBase::~ParserTestBase() {} @@ -193,4 +200,6 @@ void ParserTestBase::run(const std::string& sql, int32_t expect, ParserStage che return impl_->run(sql, expect, checkStage); } +void ParserTestBase::checkDdl(const SQuery* pQuery, ParserStage stage) { return; } + } // namespace ParserTest diff --git a/source/libs/parser/test/parTestUtil.h b/source/libs/parser/test/parTestUtil.h index 0e8703b2c8..c7d7ead8db 100644 --- a/source/libs/parser/test/parTestUtil.h +++ b/source/libs/parser/test/parTestUtil.h @@ -18,6 +18,9 @@ #include +#define ALLOW_FORBID_FUNC + +#include "querynodes.h" #include "taoserror.h" namespace ParserTest { @@ -34,10 +37,32 @@ class ParserTestBase : public testing::Test { void useDb(const std::string& acctId, const std::string& db); void run(const std::string& sql, int32_t expect = TSDB_CODE_SUCCESS, ParserStage checkStage = PARSER_STAGE_ALL); + virtual void checkDdl(const SQuery* pQuery, ParserStage stage); + private: std::unique_ptr impl_; }; +class ParserDdlTest : public ParserTestBase { + public: + void setCheckDdlFunc(const std::function& func) { checkDdl_ = func; } + + virtual void checkDdl(const SQuery* pQuery, ParserStage stage) { + ASSERT_NE(pQuery, nullptr); + ASSERT_EQ(pQuery->haveResultSet, false); + ASSERT_NE(pQuery->pRoot, nullptr); + ASSERT_EQ(pQuery->numOfResCols, 0); + ASSERT_EQ(pQuery->pResSchema, nullptr); + ASSERT_EQ(pQuery->precision, 0); + if (nullptr != checkDdl_) { + checkDdl_(pQuery, stage); + } + } + + private: + std::function checkDdl_; +}; + extern bool g_isDump; } // namespace ParserTest diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 3e19ccbd82..d4b9f5b292 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -54,6 +54,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); + if (QUERY_NODE_FUNCTION == nodeType(pExpr) && FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pExpr)->funcType) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; return DEAL_RES_IGNORE_CHILD; @@ -253,7 +256,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect TSWAP(pScan->pMeta, pRealTable->pMeta); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); - pScan->scanSeq[0] = 1; + pScan->scanSeq[0] = pSelect->hasRepeatScanFuncs ? 2 : 1; pScan->scanSeq[1] = 0; pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index dbce9abf36..30f6f03a6c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -754,10 +754,7 @@ static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNo EOrder order = opkGetPrimaryKeyOrder(pSort); if (ORDER_DESC == order) { SNode* pScan = NULL; - FOREACH(pScan, pScanNodes) { - ((SScanLogicNode*)pScan)->scanSeq[0] = 0; - ((SScanLogicNode*)pScan)->scanSeq[1] = 1; - } + FOREACH(pScan, pScanNodes) { TSWAP(((SScanLogicNode*)pScan)->scanSeq[0], ((SScanLogicNode*)pScan)->scanSeq[1]); } } if (NULL == pSort->node.pParent) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 18fa5b93dd..835d607099 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -869,7 +869,8 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_INTERVAL); + pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, + (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL : QUERY_NODE_PHYSICAL_PLAN_INTERVAL)); if (NULL == pInterval) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 639af2875e..a17d8cd850 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -48,4 +48,6 @@ TEST_F(PlanBasicTest, func) { useDb("root", "test"); run("SELECT DIFF(c1) FROM t1"); + + run("SELECT PERCENTILE(c1, 60) FROM t1"); } diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 4938618db6..6c7b1d0a0e 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -37,4 +37,6 @@ TEST_F(PlanOptimizeTest, orderByPrimaryKey) { run("SELECT * FROM t1 ORDER BY ts DESC"); run("SELECT c1 FROM t1 ORDER BY ts"); run("SELECT c1 FROM t1 ORDER BY ts DESC"); + + run("SELECT COUNT(*) FROM t1 INTERVAL(10S) ORDER BY _WSTARTTS DESC"); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 288d2e5f76..3e3e393f5f 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -136,7 +136,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) return 0; } -int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *rpcCtx) { +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, + bool persistHandle, void* rpcCtx) { char* pMsg = rpcMallocCont(pInfo->msgInfo.len); if (NULL == pMsg) { qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 80e5669cc2..eaab8e1f53 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3577,6 +3577,22 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } if (QUERY_NODE_NODE_LIST == nodeType(*pNode)) { + SNodeListNode *listNode = (SNodeListNode *)*pNode; + if (QUERY_NODE_VALUE != nodeType(listNode->pNodeList->pHead->pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + SValueNode *valueNode = (SValueNode *)listNode->pNodeList->pHead->pNode; + uint8_t type = valueNode->node.resType.type; + SNode *node = NULL; + FOREACH(node, listNode->pNodeList) { + if (type != ((SValueNode *)node)->node.resType.type) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } + return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 1e37533f2c..b83147bfee 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -265,13 +265,53 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r } static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { - int64_t value = strtoll(buf, NULL, 10); - colDataAppendInt64(pOut->columnData, rowIndex, &value); + switch (pOut->columnData->info.type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t value = (int8_t)strtoll(buf, NULL, 10); + colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t value = (int16_t)strtoll(buf, NULL, 10); + colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t value = (int32_t)strtoll(buf, NULL, 10); + colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t value = (int64_t)strtoll(buf, NULL, 10); + colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); + break; + } + } } static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { - uint64_t value = strtoull(buf, NULL, 10); - colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*) &value); + switch (pOut->columnData->info.type) { + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t value = (uint8_t)strtoull(buf, NULL, 10); + colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t value = (uint16_t)strtoull(buf, NULL, 10); + colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t value = (uint32_t)strtoull(buf, NULL, 10); + colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t value = (uint64_t)strtoull(buf, NULL, 10); + colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); + break; + } + } } static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { @@ -453,6 +493,71 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t } } +int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int16_t inType, int16_t outType) { + SColumnInfoData* pInputCol = pIn->columnData; + SColumnInfoData* pOutputCol = pOut->columnData; + char tmp[128] = {0}; + + if (IS_SIGNED_NUMERIC_TYPE(inType) || inType == TSDB_DATA_TYPE_BOOL || inType == TSDB_DATA_TYPE_TIMESTAMP) { + for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutputCol, i); + continue; + } + + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i)); + int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value); + varDataLen(tmp) = len; + if (outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pOut, i); + } else { + colDataAppend(pOutputCol, i, (char *)&value, false); + } + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(inType)) { + for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutputCol, i); + continue; + } + + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i)); + int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value); + varDataLen(tmp) = len; + if (outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pOut, i); + } else { + colDataAppend(pOutputCol, i, (char *)&value, false); + } + } + } else if (IS_FLOAT_TYPE(inType)) { + for (int32_t i = 0; i < pIn->numOfRows; ++i) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + colDataAppendNULL(pOutputCol, i); + continue; + } + + double value = 0; + GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i)); + int32_t len = sprintf(varDataVal(tmp), "%lf", value); + varDataLen(tmp) = len; + if (outType == TSDB_DATA_TYPE_NCHAR) { + varToNchar(tmp, pOut, i); + } else { + colDataAppend(pOutputCol, i, (char *)&value, false); + } + } + } else { + sclError("not supported input type:%d", inType); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + // TODO opt performance int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { SColumnInfoData* pInputCol = pIn->columnData; @@ -610,6 +715,10 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) { } break; } + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + return vectorConvertToVarData(pIn, pOut, inType, outType); + } default: sclError("invalid convert output type:%d", outType); return TSDB_CODE_QRY_APP_ERROR; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 031722ab3c..66c85b9cbb 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -60,7 +60,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr); + sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(code == 0); @@ -79,7 +79,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char *errStr = tstrerror(err); - sError("walReadWithHandle error, err:%d, msg:%s", err, errStr); + sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); ASSERT(0); } //assert(walReadWithHandle(pWalHandle, index) == 0); @@ -108,7 +108,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(walRollback(pWal, fromIndex) == 0); + //assert(walRollback(pWal, fromIndex) == 0); + int32_t code = walRollback(pWal, fromIndex); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + ASSERT(0); + } return 0; // to avoid compiler error } @@ -132,7 +139,14 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(walCommit(pWal, index) == 0); + //assert(walCommit(pWal, index) == 0); + int32_t code = walCommit(pWal, index); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno)); + ASSERT(0); + } return 0; // to avoid compiler error } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 43822cd311..cf7dd50103 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -113,7 +113,9 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SB } int tdbBtreeClose(SBTree *pBt) { - // TODO + if (pBt) { + tdbOsFree(pBt); + } return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index ceaac6dff1..383807cc35 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -66,7 +66,10 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn } int tdbDbClose(TDB *pDb) { - // TODO + if (pDb) { + tdbBtreeClose(pDb->pBt); + tdbOsFree(pDb); + } return 0; } diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index d64b31234e..c0c1343a4f 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -56,7 +56,7 @@ int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) { pEnv->nPgrHash = 8; tsize = sizeof(SPager *) * pEnv->nPgrHash; - pEnv->pgrHash = tdbRealloc(pEnv->pgrHash, tsize); + pEnv->pgrHash = tdbOsMalloc(tsize); if (pEnv->pgrHash == NULL) { return -1; } @@ -69,7 +69,19 @@ int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) { } int tdbEnvClose(TENV *pEnv) { - // TODO + SPager *pPager; + + if (pEnv) { + for (pPager = pEnv->pgrList; pPager; pPager = pEnv->pgrList) { + pEnv->pgrList = pPager->pNext; + tdbPagerClose(pPager); + } + + tdbPCacheClose(pEnv->pCache); + tdbOsFree(pEnv->pgrHash); + tdbOsFree(pEnv); + } + return 0; } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 646b83298e..aa05687426 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -292,6 +292,10 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pPage->pFreeNext = pCache->pFree; pCache->pFree = pPage; pCache->nFree++; + + // add to local list + pPage->pCacheNext = pCache->pList; + pCache->pList = pPage; } // Open the hash table @@ -317,9 +321,10 @@ static int tdbPCacheCloseImpl(SPCache *pCache) { for (pPage = pCache->pList; pPage; pPage = pCache->pList) { pCache->pList = pPage->pCacheNext; - tdbPageDestroy(pPage, NULL, NULL); + tdbPageDestroy(pPage, tdbDefaultFree, NULL); } + tdbOsFree(pCache->pgHash); tdbPCacheDestroyLock(pCache); return 0; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 1941592602..fbd5cb3aac 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -88,7 +88,13 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { } int tdbPagerClose(SPager *pPager) { - // TODO + if (pPager) { + if (pPager->inTran) { + tdbOsClose(pPager->jfd); + } + tdbOsClose(pPager->fd); + tdbOsFree(pPager); + } return 0; } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 323ef43e25..af66d39904 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -1108,7 +1108,7 @@ void transSendResponse(const STransMsg* msg) { SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->msg = tmsg; srvMsg->type = Normal; - tTrace("server conn %p start to send resp (1/2)", exh->handle); + tDebug("server conn %p start to send resp (1/2)", exh->handle); transSendAsync(pThrd->asyncPool, &srvMsg->q); uvReleaseExHandle(refId); return; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index cbf20f02cd..575d5bc187 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -125,7 +125,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { strcpy(outTimezoneStr, tz); } -#elif defined(_TD_DARWIN_64) +#else char buf[4096] = {0}; char *tz = NULL; { @@ -170,64 +170,5 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); - -#else - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = taosGetTimestampSec(); - struct tm tm1; - taosLocalTime(&tx1, &tm1); - - /* load time zone string from /etc/timezone */ - // FILE *f = fopen("/etc/timezone", "r"); - errno = 0; - TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); - char buf[68] = {0}; - if (pFile != NULL) { - int len = taosReadFile(pFile, buf, 64); - if (len < 64 && taosGetErrorFile(pFile)) { - taosCloseFile(&pFile); - printf("read /etc/timezone error, reason:%s", strerror(errno)); - return; - } - - taosCloseFile(&pFile); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); - } - } - // get and set default timezone - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - *tsTimezone = tz; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - #endif } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 14503e7068..9dcbafca7a 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -911,7 +911,7 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1) void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; - taosThreadJoin(cacheRefreshWorker, NULL); + if(cacheThreadInit != PTHREAD_ONCE_INIT) taosThreadJoin(cacheRefreshWorker, NULL); taosArrayDestroy(pCacheArrayList); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 81d31dbbde..e32ecfc695 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -94,6 +94,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY, "Cannot add duplicate keys to hash") +TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c deleted file mode 100644 index 77f5ed5051..0000000000 --- a/source/util/src/tskiplist2.c +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tskiplist2.h" - -struct SSLNode { - int8_t level; - SSLNode *forwards[]; -}; - -struct SSkipList2 { - int8_t level; - uint32_t seed; - int32_t size; - const SSLCfg *pCfg; - SSLNode *pHead[]; -}; - -static void *slMalloc(void *pPool, int32_t size); -static void slFree(void *pPool, void *p); -static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData); - -const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, - .nKey = -1, - .nData = -1, - .cmprFn = slCmprFn, - .pPool = NULL, - .xMalloc = slMalloc, - .xFree = slFree}; - -int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { - SSkipList2 *pSl = NULL; - int32_t size; - - *ppSl = NULL; - if (pCfg == NULL) pCfg = &slDefaultCfg; - - // check config (TODO) - - // malloc handle - size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2; - pSl = pCfg->xMalloc(pCfg->pPool, size); - if (pSl == NULL) { - return -1; - } - - pSl->level = 0; - pSl->seed = taosRand(); - pSl->size = 0; - pSl->pCfg = pCfg; - - // init an empty skiplist - for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) { - pSl->pHead[i] = NULL; - } - - *ppSl = pSl; - return 0; -} - -int32_t slClose(SSkipList2 *pSl) { - if (pSl) { - slClear(pSl); - if (pSl->pCfg->xFree) { - pSl->pCfg->xFree(pSl->pCfg->pPool, pSl); - } - } - - return 0; -} - -int32_t slClear(SSkipList2 *pSl) { - // loop to clear sl - for (;;) { - // (TODO) - } - - // init sl (TODO) - - return 0; -} - -int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { - pSlc->pSl = pSl; - - for (int i = 0; i < SL_MAX_LEVEL; i++) { - if (i < pSl->pCfg->maxLevel) { - } else { - pSlc->forwards[i] = NULL; - } - } - - // TODO - return 0; -} - -int32_t slcClose(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) { - // TODO - return 0; -} - -int32_t slcMoveToNext(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveToPrev(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveToFirst(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcMoveToLast(SSLCursor *pSlc) { - // TODO - return 0; -} - -int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) { - // TODO - return 0; -} - -int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) { - // TODO - return 0; -} - -int32_t slcDrop(SSLCursor *pSlc) { - // TODO - return 0; -} - -static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); } - -static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); } - -static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) { - ASSERT(nKey1 >= 0 && nKey2 >= 0); - - int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1; - int32_t c; - - c = memcmp(pKey1, pKey2, nKey); - if (c == 0) { - if (nKey1 > nKey2) { - c = 1; - } else if (nKey1 < nKey2) { - c = -1; - } - } - - return c; -} \ No newline at end of file diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 00e12a4fe8..974677d26c 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -440,6 +440,7 @@ TEST(td_encode_test, compound_struct_encode_test) { tCoderClear(&decoder); } #endif + #pragma GCC diagnostic pop #endif diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 8a29a0a7a6..9dcd485194 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -460,8 +460,7 @@ class TDDnodes: processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" + binPath = self.dnodes[0].getPath() + "/../../../" tdLog.debug("binPath %s" % (binPath)) binPath = os.path.realpath(binPath) tdLog.debug("binPath real path %s" % (binPath)) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 99e546fcd0..8169e6c503 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,7 +11,7 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_NCHAR}; int32_t optrIdxList[] = {0, 1, 2}; typedef struct { @@ -20,6 +20,11 @@ typedef struct { bool enclose; } OperInfo; +typedef enum { + BP_BIND_TAG = 1, + BP_BIND_COL, +} BP_BIND_TYPE; + OperInfo operInfo[] = { {">", 2, false}, {">=", 2, false}, @@ -41,6 +46,8 @@ int32_t varoperatorList[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; char *bpStbPrefix = "st"; char *bpTbPrefix = "t"; +int32_t bpDefaultStbId = 1; + //char *operatorList[] = {">", ">=", "<", "<=", "=", "<>", "in", "not in"}; @@ -65,6 +72,7 @@ typedef struct { char* isNull; int32_t* binaryLen; TAOS_MULTI_BIND* pBind; + TAOS_MULTI_BIND* pTags; char* sql; int32_t* colTypes; int32_t colNum; @@ -83,6 +91,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos); int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos); int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos); int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos); +int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos); int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos); enum { @@ -95,52 +104,55 @@ typedef struct { int32_t colNum; int32_t *colList; // full table column list int32_t testType; - bool prepareStb; + bool autoCreateTbl; bool fullCol; int32_t (*runFn)(TAOS_STMT*, TAOS*); int32_t tblNum; int32_t rowNum; int32_t bindRowNum; int32_t bindColNum; // equal colNum in full column case + int32_t bindTagNum; // equal colNum in full column case int32_t bindNullNum; int32_t runTimes; int32_t preCaseIdx; } CaseCfg; CaseCfg gCase[] = { - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, // 11 - {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1, -1}, - {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1, -1}, + {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 1, -1}, - {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1, -1}, + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, // 22 - {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 1, 2}, + {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, + + {"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 0, 1, 2}, }; @@ -149,17 +161,20 @@ CaseCfg *gCurCase = NULL; typedef struct { char caseCatalog[255]; int32_t bindNullNum; - bool prepareStb; bool checkParamNum; bool printRes; bool printCreateTblSql; bool printQuerySql; bool printStmtSql; + bool autoCreateTbl; int32_t rowNum; //row num for one table int32_t bindColNum; + int32_t bindTagNum; int32_t bindRowNum; //row num for once bind int32_t bindColTypeNum; int32_t* bindColTypeList; + int32_t bindTagTypeNum; + int32_t* bindTagTypeList; int32_t optrIdxListNum; int32_t* optrIdxList; int32_t runTimes; @@ -172,15 +187,18 @@ typedef struct { #if 0 CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, - .prepareStb = false, .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, + .autoCreateTbl = false, .rowNum = 0, .bindColNum = 0, + .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, .bindColTypeList = NULL, + .bindTagTypeNum = 0, + .bindTagTypeList = NULL, .optrIdxListNum = 0, .optrIdxList = NULL, .checkParamNum = false, @@ -193,10 +211,37 @@ CaseCtrl gCaseCtrl = { // default }; #endif + +#if 1 +CaseCtrl gCaseCtrl = { // default + .bindNullNum = 0, + .printCreateTblSql = true, + .printQuerySql = true, + .printStmtSql = true, + .autoCreateTbl = false, + .rowNum = 0, + .bindColNum = 0, + .bindTagNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .bindTagTypeNum = 0, + .bindTagTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .checkParamNum = false, + .printRes = true, + .runTimes = 0, + .caseIdx = -1, + .caseNum = 1, + .caseRunIdx = 11, + .caseRunNum = 1, +}; +#endif + #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, - .prepareStb = false, .printCreateTblSql = false, .printQuerySql = true, .printStmtSql = true, @@ -221,28 +266,29 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper }; #endif -#if 1 +#if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, - .prepareStb = false, - .printCreateTblSql = false, + .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, + .autoCreateTbl = true, .rowNum = 0, .bindColNum = 0, + .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, .bindColTypeList = NULL, .optrIdxListNum = 0, .optrIdxList = NULL, - .checkParamNum = true, + .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, - .optrIdxListNum = tListLen(optrIdxList), - .optrIdxList = optrIdxList, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, + //.optrIdxListNum = tListLen(optrIdxList), + //.optrIdxList = optrIdxList, + //.bindColTypeNum = tListLen(bindColTypeList), + //.bindColTypeList = bindColTypeList, .caseIdx = 22, .caseNum = 1, .caseRunNum = 1, @@ -299,6 +345,78 @@ void generateInsertSQL(BindData *data) { } else { len = sprintf(data->sql, "insert into %s0 ", bpTbPrefix); } + + if (gCurCase->bindTagNum > 0) { + len += sprintf(data->sql + len, "using %s%d ", bpStbPrefix, bpDefaultStbId); + + if (!gCurCase->fullCol) { + len += sprintf(data->sql + len, "("); + for (int c = 0; c < gCurCase->bindTagNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); + } + switch (data->pTags[c].buffer_type) { + case TSDB_DATA_TYPE_BOOL: + len += sprintf(data->sql + len, "tbooldata"); + break; + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(data->sql + len, "ttinydata"); + break; + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(data->sql + len, "tsmalldata"); + break; + case TSDB_DATA_TYPE_INT: + len += sprintf(data->sql + len, "tintdata"); + break; + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(data->sql + len, "tbigdata"); + break; + case TSDB_DATA_TYPE_FLOAT: + len += sprintf(data->sql + len, "tfloatdata"); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += sprintf(data->sql + len, "tdoubledata"); + break; + case TSDB_DATA_TYPE_VARCHAR: + len += sprintf(data->sql + len, "tbinarydata"); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(data->sql + len, "tts"); + break; + case TSDB_DATA_TYPE_NCHAR: + len += sprintf(data->sql + len, "tnchardata"); + break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(data->sql + len, "tutinydata"); + break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(data->sql + len, "tusmalldata"); + break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(data->sql + len, "tuintdata"); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(data->sql + len, "tubigdata"); + break; + default: + printf("!!!invalid tag type:%d", data->pTags[c].buffer_type); + exit(1); + } + } + + len += sprintf(data->sql + len, ") "); + } + + len += sprintf(data->sql + len, "tags ("); + for (int c = 0; c < gCurCase->bindTagNum; ++c) { + if (c) { + len += sprintf(data->sql + len, ","); + } + len += sprintf(data->sql + len, "?"); + } + len += sprintf(data->sql + len, ") "); + } + if (!gCurCase->fullCol) { len += sprintf(data->sql + len, "("); for (int c = 0; c < gCurCase->bindColNum; ++c) { @@ -528,6 +646,7 @@ void generateQueryMiscSQL(BindData *data, int32_t tblIdx) { } + void generateErrorSQL(BindData *data, int32_t tblIdx) { int32_t len = 0; data->sql = taosMemoryCalloc(1, 1024); @@ -552,7 +671,7 @@ void generateErrorSQL(BindData *data, int32_t tblIdx) { } } -void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { +void generateColDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { if (gCaseCtrl.bindColTypeNum) { *dataType = gCaseCtrl.bindColTypeList[colIdx]; @@ -584,104 +703,146 @@ void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t * } } -int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) { - int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP; +void generateTagDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { + if (bindIdx < gCurCase->bindTagNum) { + if (gCaseCtrl.bindTagTypeNum) { + *dataType = gCaseCtrl.bindTagTypeList[colIdx]; + return; + } else if (gCurCase->fullCol) { + *dataType = gCurCase->colList[bindIdx]; + return; + } else { + while (true) { + *dataType = rand() % (TSDB_DATA_TYPE_MAX - 1) + 1; + if (*dataType == TSDB_DATA_TYPE_JSON || *dataType == TSDB_DATA_TYPE_DECIMAL + || *dataType == TSDB_DATA_TYPE_BLOB || *dataType == TSDB_DATA_TYPE_MEDIUMBLOB + || *dataType == TSDB_DATA_TYPE_VARBINARY) { + continue; + } + + if (colExists(data->pTags, *dataType)) { + continue; + } + + break; + } + } + } else { + *dataType = data->pTags[bindIdx%gCurCase->bindTagNum].buffer_type; + } +} + + +int32_t prepareColData(BP_BIND_TYPE bType, BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) { + int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP; + TAOS_MULTI_BIND *pBase = NULL; + + if (bType == BP_BIND_TAG) { + pBase = data->pTags; + generateTagDataType(data, bindIdx, colIdx, &dataType); + } else { + pBase = data->pBind; + generateColDataType(data, bindIdx, colIdx, &dataType); + } - generateDataType(data, bindIdx, colIdx, &dataType); switch (dataType) { case TSDB_DATA_TYPE_BOOL: - data->pBind[bindIdx].buffer_length = sizeof(bool); - data->pBind[bindIdx].buffer = data->boolData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(bool); + pBase[bindIdx].buffer = data->boolData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_TINYINT: - data->pBind[bindIdx].buffer_length = sizeof(int8_t); - data->pBind[bindIdx].buffer = data->tinyData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int8_t); + pBase[bindIdx].buffer = data->tinyData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_SMALLINT: - data->pBind[bindIdx].buffer_length = sizeof(int16_t); - data->pBind[bindIdx].buffer = data->smallData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int16_t); + pBase[bindIdx].buffer = data->smallData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_INT: - data->pBind[bindIdx].buffer_length = sizeof(int32_t); - data->pBind[bindIdx].buffer = data->intData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int32_t); + pBase[bindIdx].buffer = data->intData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_BIGINT: - data->pBind[bindIdx].buffer_length = sizeof(int64_t); - data->pBind[bindIdx].buffer = data->bigData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(int64_t); + pBase[bindIdx].buffer = data->bigData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_FLOAT: - data->pBind[bindIdx].buffer_length = sizeof(float); - data->pBind[bindIdx].buffer = data->floatData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(float); + pBase[bindIdx].buffer = data->floatData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_DOUBLE: - data->pBind[bindIdx].buffer_length = sizeof(double); - data->pBind[bindIdx].buffer = data->doubleData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(double); + pBase[bindIdx].buffer = data->doubleData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_VARCHAR: - data->pBind[bindIdx].buffer_length = gVarCharSize; - data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; - data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = gVarCharSize; + pBase[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; + pBase[bindIdx].length = data->binaryLen; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_TIMESTAMP: - data->pBind[bindIdx].buffer_length = sizeof(int64_t); - data->pBind[bindIdx].buffer = data->tsData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = NULL; + pBase[bindIdx].buffer_length = sizeof(int64_t); + pBase[bindIdx].buffer = data->tsData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = NULL; break; case TSDB_DATA_TYPE_NCHAR: - data->pBind[bindIdx].buffer_length = gVarCharSize; - data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; - data->pBind[bindIdx].length = data->binaryLen; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = gVarCharSize; + pBase[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize; + pBase[bindIdx].length = data->binaryLen; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UTINYINT: - data->pBind[bindIdx].buffer_length = sizeof(uint8_t); - data->pBind[bindIdx].buffer = data->utinyData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint8_t); + pBase[bindIdx].buffer = data->utinyData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_USMALLINT: - data->pBind[bindIdx].buffer_length = sizeof(uint16_t); - data->pBind[bindIdx].buffer = data->usmallData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint16_t); + pBase[bindIdx].buffer = data->usmallData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UINT: - data->pBind[bindIdx].buffer_length = sizeof(uint32_t); - data->pBind[bindIdx].buffer = data->uintData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint32_t); + pBase[bindIdx].buffer = data->uintData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; case TSDB_DATA_TYPE_UBIGINT: - data->pBind[bindIdx].buffer_length = sizeof(uint64_t); - data->pBind[bindIdx].buffer = data->ubigData + rowIdx; - data->pBind[bindIdx].length = NULL; - data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; + pBase[bindIdx].buffer_length = sizeof(uint64_t); + pBase[bindIdx].buffer = data->ubigData + rowIdx; + pBase[bindIdx].length = NULL; + pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL; break; default: printf("!!!invalid col type:%d", dataType); exit(1); } - data->pBind[bindIdx].buffer_type = dataType; - data->pBind[bindIdx].num = gCurCase->bindRowNum; + pBase[bindIdx].buffer_type = dataType; + pBase[bindIdx].num = gCurCase->bindRowNum; + if (bType == BP_BIND_TAG) { + pBase[bindIdx].num = 1; + } + return 0; } @@ -694,6 +855,7 @@ int32_t prepareInsertData(BindData *data) { data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->sql = taosMemoryCalloc(1, 1024); data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); + data->pTags = taosMemoryCalloc(gCurCase->tblNum*gCurCase->bindTagNum, sizeof(TAOS_MULTI_BIND)); data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool)); data->tinyData = taosMemoryMalloc(allRowNum * sizeof(int8_t)); @@ -734,10 +896,17 @@ int32_t prepareInsertData(BindData *data) { for (int b = 0; b < (allRowNum/gCurCase->bindRowNum); b++) { for (int c = 0; c < gCurCase->bindColNum; ++c) { - prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); } } + for (int b = 0; b < gCurCase->tblNum; b++) { + for (int c = 0; c < gCurCase->bindTagNum; ++c) { + prepareColData(BP_BIND_TAG, data, b*gCurCase->bindTagNum+c, b, c); + } + } + + generateInsertSQL(data); return 0; @@ -791,11 +960,11 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { for (int b = 0; b < bindNum; b++) { for (int c = 0; c < gCurCase->bindColNum; ++c) { - prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); } } - generateQuerySQL(data, tblIdx); + generateQueryCondSQL(data, tblIdx); return 0; } @@ -849,7 +1018,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { for (int b = 0; b < bindNum; b++) { for (int c = 0; c < gCurCase->bindColNum; ++c) { - prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); } } @@ -1099,6 +1268,15 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B printf("***sql res num match stmt res num %d\n", stmtResNum); } +int32_t bpSetTableNameTags(BindData *data, int32_t tblIdx, char *tblName, TAOS_STMT *stmt) { + if (gCurCase->bindTagNum > 0) { + return taos_stmt_set_tbname_tags(stmt, tblName, data->pTags + tblIdx * gCurCase->bindTagNum); + } else { + return taos_stmt_set_tbname(stmt, tblName); + } +} + + /* prepare [settbname [bind add]] exec */ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -1117,7 +1295,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1174,7 +1352,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1227,7 +1405,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1281,7 +1459,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1335,7 +1513,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1350,7 +1528,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1402,7 +1580,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1458,7 +1636,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { if (gCurCase->tblNum > 1) { char buf[32]; sprintf(buf, "t%d", t); - code = taos_stmt_set_tbname(stmt, buf); + code = bpSetTableNameTags(&data, t, buf, stmt); if (code != 0){ printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); exit(1); @@ -1502,6 +1680,74 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } + +/* [prepare [settbnametag [bind add] exec]] */ +int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { + int32_t loop = 0; + + while (gCurCase->bindTagNum > 0 && gCurCase->bindColNum > 0) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + if (gCurCase->tblNum > 1) { + char buf[32]; + sprintf(buf, "t%d", t); + code = taos_stmt_set_tbname_tags(stmt, buf, data.pTags + t * gCurCase->bindTagNum); + if (code != 0){ + printf("!!!taos_stmt_set_tbname_tags error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + + bpCheckIsInsert(stmt, 1); + + destroyData(&data); + + gCurCase->bindColNum -= 2; + gCurCase->bindTagNum -= 2; + gCurCase->fullCol = false; + loop++; + } + + bpCheckAffectedRows(stmt, loop); + + gExecLoopTimes = loop; + + return 0; +} + + /* select * from table */ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -2911,63 +3157,6 @@ int sql_s_perf1(TAOS *taos) { void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, bool stable) { int32_t blen = 0; blen = sprintf(buf, "create table %s%d ", (stable ? bpStbPrefix : bpTbPrefix), tblIdx); - if (stable) { - blen += sprintf(buf + blen, "tags ("); - for (int c = 0; c < colNum; ++c) { - if (c > 0) { - blen += sprintf(buf + blen, ","); - } - switch (colList[c]) { - case TSDB_DATA_TYPE_BOOL: - blen += sprintf(buf + blen, "tbooldata bool"); - break; - case TSDB_DATA_TYPE_TINYINT: - blen += sprintf(buf + blen, "ttinydata tinyint"); - break; - case TSDB_DATA_TYPE_SMALLINT: - blen += sprintf(buf + blen, "tsmalldata smallint"); - break; - case TSDB_DATA_TYPE_INT: - blen += sprintf(buf + blen, "tintdata int"); - break; - case TSDB_DATA_TYPE_BIGINT: - blen += sprintf(buf + blen, "tbigdata bigint"); - break; - case TSDB_DATA_TYPE_FLOAT: - blen += sprintf(buf + blen, "tfloatdata float"); - break; - case TSDB_DATA_TYPE_DOUBLE: - blen += sprintf(buf + blen, "tdoubledata double"); - break; - case TSDB_DATA_TYPE_VARCHAR: - blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - blen += sprintf(buf + blen, "tts ts"); - break; - case TSDB_DATA_TYPE_NCHAR: - blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize); - break; - case TSDB_DATA_TYPE_UTINYINT: - blen += sprintf(buf + blen, "tutinydata tinyint unsigned"); - break; - case TSDB_DATA_TYPE_USMALLINT: - blen += sprintf(buf + blen, "tusmalldata smallint unsigned"); - break; - case TSDB_DATA_TYPE_UINT: - blen += sprintf(buf + blen, "tuintdata int unsigned"); - break; - case TSDB_DATA_TYPE_UBIGINT: - blen += sprintf(buf + blen, "tubigdata bigint unsigned"); - break; - default: - printf("invalid col type:%d", colList[c]); - exit(1); - } - } - - blen += sprintf(buf + blen, ")"); - } blen += sprintf(buf + blen, " ("); @@ -3027,6 +3216,64 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t * blen += sprintf(buf + blen, ")"); + if (stable) { + blen += sprintf(buf + blen, "tags ("); + for (int c = 0; c < colNum; ++c) { + if (c > 0) { + blen += sprintf(buf + blen, ","); + } + switch (colList[c]) { + case TSDB_DATA_TYPE_BOOL: + blen += sprintf(buf + blen, "tbooldata bool"); + break; + case TSDB_DATA_TYPE_TINYINT: + blen += sprintf(buf + blen, "ttinydata tinyint"); + break; + case TSDB_DATA_TYPE_SMALLINT: + blen += sprintf(buf + blen, "tsmalldata smallint"); + break; + case TSDB_DATA_TYPE_INT: + blen += sprintf(buf + blen, "tintdata int"); + break; + case TSDB_DATA_TYPE_BIGINT: + blen += sprintf(buf + blen, "tbigdata bigint"); + break; + case TSDB_DATA_TYPE_FLOAT: + blen += sprintf(buf + blen, "tfloatdata float"); + break; + case TSDB_DATA_TYPE_DOUBLE: + blen += sprintf(buf + blen, "tdoubledata double"); + break; + case TSDB_DATA_TYPE_VARCHAR: + blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + blen += sprintf(buf + blen, "tts timestamp"); + break; + case TSDB_DATA_TYPE_NCHAR: + blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize); + break; + case TSDB_DATA_TYPE_UTINYINT: + blen += sprintf(buf + blen, "tutinydata tinyint unsigned"); + break; + case TSDB_DATA_TYPE_USMALLINT: + blen += sprintf(buf + blen, "tusmalldata smallint unsigned"); + break; + case TSDB_DATA_TYPE_UINT: + blen += sprintf(buf + blen, "tuintdata int unsigned"); + break; + case TSDB_DATA_TYPE_UBIGINT: + blen += sprintf(buf + blen, "tubigdata bigint unsigned"); + break; + default: + printf("invalid col type:%d", colList[c]); + exit(1); + } + } + + blen += sprintf(buf + blen, ")"); + } + if (gCaseCtrl.printCreateTblSql) { printf("\tCreate Table SQL:%s\n", buf); } @@ -3067,7 +3314,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) { } } else { char buf[1024]; - generateCreateTableSQL(buf, 1, colNum, colList, true); + generateCreateTableSQL(buf, bpDefaultStbId, colNum, colList, true); result = taos_query(taos, buf); code = taos_errno(result); @@ -3110,17 +3357,34 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { if (gCaseCtrl.rowNum) { gCurCase->rowNum = gCaseCtrl.rowNum; } + + if (gCaseCtrl.autoCreateTbl) { + if (gCurCase->testType == TTYPE_INSERT && gCurCase->tblNum > 1) { + gCurCase->autoCreateTbl = true; + if (gCurCase->bindTagNum <= 0) { + gCurCase->bindTagNum = gCurCase->colNum; + } + } else { + return 1; + } + } if (gCurCase->fullCol) { gCurCase->bindColNum = gCurCase->colNum; + if (gCurCase->autoCreateTbl) { + gCurCase->bindTagNum = gCurCase->colNum; + } } gCurCase->bindNullNum = gCaseCtrl.bindNullNum; - gCurCase->prepareStb = gCaseCtrl.prepareStb; if (gCaseCtrl.bindColNum) { gCurCase->bindColNum = gCaseCtrl.bindColNum; gCurCase->fullCol = false; } + if (gCaseCtrl.bindTagNum) { + gCurCase->bindTagNum = gCaseCtrl.bindTagNum; + gCurCase->fullCol = false; + } if (gCaseCtrl.bindRowNum) { gCurCase->bindRowNum = gCaseCtrl.bindRowNum; } @@ -3128,6 +3392,10 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { gCurCase->bindColNum = gCaseCtrl.bindColTypeNum; gCurCase->fullCol = false; } + if (gCaseCtrl.bindTagTypeNum) { + gCurCase->bindTagNum = gCaseCtrl.bindTagTypeNum; + gCurCase->fullCol = false; + } if (!silent) { printf("* Case %d - [%s]%s Begin *\n", caseRunIdx, gCaseCtrl.caseCatalog, gCurCase->caseDesc); @@ -3136,7 +3404,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) { totalUs = 0; for (int32_t n = 0; n < gCurCase->runTimes; ++n) { if (gCurCase->preCaseIdx < 0) { - prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->prepareStb); + prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreateTbl); } beginUs = taosGetTimestampUs(); @@ -3197,10 +3465,18 @@ void* runCaseList(TAOS *taos) { } void runAll(TAOS *taos) { +/* strcpy(gCaseCtrl.caseCatalog, "Normal Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); runCaseList(taos); +*/ + strcpy(gCaseCtrl.caseCatalog, "Auto Create Table Test"); + gCaseCtrl.autoCreateTbl = true; + printf("%s Begin\n", gCaseCtrl.caseCatalog); + runCaseList(taos); + gCaseCtrl.autoCreateTbl = false; +/* strcpy(gCaseCtrl.caseCatalog, "Null Test"); printf("%s Begin\n", gCaseCtrl.caseCatalog); gCaseCtrl.bindNullNum = 1; @@ -3244,7 +3520,8 @@ void runAll(TAOS *taos) { gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList); gCaseCtrl.bindColTypeList = bindColTypeList; runCaseList(taos); - +*/ + printf("All Test End\n"); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index f2b9e0caab..9f197b16c8 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -70,6 +70,11 @@ ./test.sh -f tsim/tmq/basic2.sim ./test.sh -f tsim/tmq/basic3.sim ./test.sh -f tsim/tmq/basic4.sim +./test.sh -f tsim/tmq/basic1Of2Cons.sim +./test.sh -f tsim/tmq/basic2Of2Cons.sim +./test.sh -f tsim/tmq/basic3Of2Cons.sim +./test.sh -f tsim/tmq/basic4Of2Cons.sim +./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim # --- stable ./test.sh -f tsim/stable/disk.sim diff --git a/tests/script/sh/copy_udf.sh b/tests/script/sh/copy_udf.sh index 7b5b5f4720..c3c300fb7b 100755 --- a/tests/script/sh/copy_udf.sh +++ b/tests/script/sh/copy_udf.sh @@ -4,9 +4,10 @@ set +e #set -x echo "Executing copy_udf.sh" - -SCRIPT_DIR=`pwd` +SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` +echo "SCRIPT_DIR: ${SCRIPT_DIR}" IN_TDINTERNAL="community" if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then @@ -16,6 +17,7 @@ else fi TAOS_DIR=`pwd` +echo "find udf library in $TAOS_DIR" UDF1_DIR=`find $TAOS_DIR -name "libudf1.so"|grep lib|head -n1` UDF2_DIR=`find $TAOS_DIR -name "libudf2.so"|grep lib|head -n1` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 1310cf2656..74015eebd6 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -101,8 +101,8 @@ if [ "$EXEC_OPTON" = "start" ]; then if [ "$VALGRIND_OPTION" = "true" ]; then TT=`date +%s` #mkdir ${LOG_DIR}/${TT} - echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" - nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" + nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else echo "nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & diff --git a/tests/script/test.sh b/tests/script/test.sh index e4191da0a9..1cfe8dd6f5 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -131,8 +131,8 @@ if [ -n "$FILE_NAME" ]; then FLAG="-v" fi - echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG - valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG else if [[ $MULTIPROCESS -eq 1 ]];then echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index b3cbda5090..6290403214 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -3,6 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode1 -c startUdfd -v 1 print ========= start dnode1 as LEADER system sh/exec.sh -n dnode1 -s start @@ -64,35 +65,35 @@ if $data00 != 1.414213562 then return -1 endi -#sql insert into t2 values(now+2s, 1, null)(now+3s, null, 2); -#sql select udf1(f1, f2) from t2; -#print $rows , $data00 , $data10 , $data20 , $data30 -#if $rows != 4 then -# return -1 -#endi -#if $data00 != 88 then -# return -1 -#endi -#if $data10 != 88 then -# return -1 -#endi -# -#if $data20 != NULL then -# return -1 -#endi -# -#if $data30 != NULL then -# return -1 -#endi -# -#sql select udf2(f1, f2) from t2; -#print $rows, $data00 -#if $rows != 1 then -# return -1 -#endi -#if $data00 != 2.645751311 then -# return -1 -#endi +sql insert into t2 values(now+2s, 1, null)(now+3s, null, 2); +sql select udf1(f1, f2) from t2; +print $rows , $data00 , $data10 , $data20 , $data30 +if $rows != 4 then + return -1 +endi +if $data00 != 88 then + return -1 +endi +if $data10 != 88 then + return -1 +endi + +if $data20 != NULL then + return -1 +endi + +if $data30 != NULL then + return -1 +endi + +sql select udf2(f1, f2) from t2; +print $rows, $data00 +if $rows != 1 then + return -1 +endi +if $data00 != 2.645751311 then + return -1 +endi sql drop function udf1; sql show functions; if $rows != 1 then diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index b583ee93e8..d8050c53c5 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -199,7 +199,8 @@ class TDTestCase: os.system("%s -f %s -y " %(taosBenchbin,jsonFile)) return - def taosBenchCreate(self,dbname,stbname,vgroups,threadNumbers,count): + def taosBenchCreate(self,host,dropdb,dbname,stbname,vgroups,threadNumbers,count): + # count=50000 buildPath = self.getBuildPath() if (buildPath == ""): @@ -207,20 +208,28 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) taosBenchbin = buildPath+ "/build/bin/taosBenchmark" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + tsql=self.newcur(host,config) # insert: create one or mutiple tables per sql and insert multiple rows per sql - tdSql.execute("drop database if exists %s"%dbname) + tsql.execute("drop database if exists %s"%dbname) - tdSql.execute("create database %s vgroups %d"%(dbname,vgroups)) - tdSql.execute("use %s" %dbname) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + print("db has been created") + # tsql.getResult("show databases") + # print(tdSql.queryResult) + tsql.execute("use %s" %dbname) threads = [] # threadNumbers=2 for i in range(threadNumbers): jsonfile="1-insert/Vgroups%d%d.json"%(vgroups,i) os.system("cp -f 1-insert/manyVgroups.json %s"%(jsonfile)) - os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s%d\",/g' %s"%(dbname,i,jsonfile)) - os.system("sed -i 's/\"childtable_count\": 300000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) + os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s\",/g' %s"%(dbname,jsonfile)) + os.system("sed -i 's/\"drop\": \"no\",/\"drop\": \"%s\",/g' %s"%(dropdb,jsonfile)) + os.system("sed -i 's/\"host\": \"127.0.0.1\",/\"host\": \"%s\",/g' %s"%(host,jsonfile)) + os.system("sed -i 's/\"childtable_count\": 10000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile)) os.system("sed -i 's/\"name\": \"stb1\",/\"name\": \"%s%d\",/g' %s "%(stbname,i,jsonfile)) os.system("sed -i 's/\"childtable_prefix\": \"stb1_\",/\"childtable_prefix\": \"%s%d_\",/g' %s "%(stbname,i,jsonfile)) threads.append(mp.Process(target=self.taosBench, args=("%s"%jsonfile,))) @@ -337,8 +346,11 @@ class TDTestCase: return def test_case3(self): + # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000) + self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*1000) - self.taosBenchCreate("db1", "stb1", 1, 2, 1*50000) + # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000) + # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000) return diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index df6f1163e8..5487dff708 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -1,7 +1,7 @@ { "filetype": "insert", "cfgdir": "/etc/taos/", - "host": "test216", + "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", @@ -16,14 +16,14 @@ { "dbinfo": { "name": "db", - "drop": "yes", + "drop": "no", "vgroups": 1 }, "super_tables": [ { "name": "stb1", "child_table_exists": "no", - "childtable_count": 300000, + "childtable_count": 10000, "childtable_prefix": "stb1_", "auto_create_table": "no", "batch_create_tbl_num": 50000, diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 82c450771f..03b3899dc6 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -1,3 +1,4 @@ +from wsgiref.headers import tspecials from util.log import * from util.cases import * from util.sql import * @@ -27,6 +28,36 @@ class TDTestCase: def run(self): tdSql.prepare() + tdSql.execute("create table ntb(ts timestamp,c1 int,c2 double,c3 float)") + tdSql.execute("insert into ntb values(now,1,1.0,10.5)(now+1s,10,-100.0,5.1)(now+10s,-1,15.1,5.0)") + + tdSql.query("select diff(c1,0) from ntb") + tdSql.checkRows(2) + tdSql.checkData(0,0,9) + tdSql.checkData(1,0,-11) + tdSql.query("select diff(c1,1) from ntb") + tdSql.checkRows(2) + tdSql.checkData(0,0,9) + tdSql.checkData(1,0,None) + + tdSql.query("select diff(c2,0) from ntb") + tdSql.checkRows(2) + tdSql.checkData(0,0,-101) + tdSql.checkData(1,0,115.1) + tdSql.query("select diff(c2,1) from ntb") + tdSql.checkRows(2) + tdSql.checkData(0,0,None) + tdSql.checkData(1,0,115.1) + + tdSql.query("select diff(c3,0) from ntb") + tdSql.checkRows(2) + tdSql.checkData(0,0,-5.4) + tdSql.checkData(1,0,-0.1) + tdSql.query("select diff(c3,1) from ntb") + tdSql.checkRows(2) + tdSql.checkData(0,0,None) + tdSql.checkData(1,0,None) + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') @@ -52,68 +83,42 @@ class TDTestCase: tdSql.query("select diff(col6) from stb_1") tdSql.checkRows(0) + tdSql.query("select diff(col7) from stb_1") + tdSql.checkRows(0) + for i in range(self.rowNum): tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.error("select diff(ts) from stb") + # tdSql.error("select diff(ts) from stb") tdSql.error("select diff(ts) from stb_1") - tdSql.error("select diff(col1) from stb") - tdSql.error("select diff(col2) from stb") - tdSql.error("select diff(col3) from stb") - tdSql.error("select diff(col4) from stb") - tdSql.error("select diff(col5) from stb") - tdSql.error("select diff(col6) from stb") - tdSql.error("select diff(col7) from stb") - tdSql.error("select diff(col7) from stb_1") - tdSql.error("select diff(col8) from stb") + # tdSql.error("select diff(col7) from stb") + + # tdSql.error("select diff(col8) from stb") tdSql.error("select diff(col8) from stb_1") - tdSql.error("select diff(col9) from stb") + # tdSql.error("select diff(col9) from stb") tdSql.error("select diff(col9) from stb_1") tdSql.error("select diff(col11) from stb_1") tdSql.error("select diff(col12) from stb_1") tdSql.error("select diff(col13) from stb_1") tdSql.error("select diff(col14) from stb_1") - tdSql.error("select diff(col11) from stb") - tdSql.error("select diff(col12) from stb") - tdSql.error("select diff(col13) from stb") - tdSql.error("select diff(col14) from stb") tdSql.query("select ts,diff(col1),ts from stb_1") - tdSql.checkRows(10) + tdSql.checkRows(11) tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 0, "2018-09-17 09:00:00.000") + tdSql.checkData(1, 2, "2018-09-17 09:00:00.000") tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + tdSql.checkData(9, 2, "2018-09-17 09:00:00.009") - tdSql.query("select ts,diff(col1),ts from stb group by tbname") - tdSql.checkRows(10) - tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") - tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") - - tdSql.query("select ts,diff(col1),ts from stb_1") - tdSql.checkRows(10) - tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") - tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") - - tdSql.query("select ts,diff(col1),ts from stb group by tbname") - tdSql.checkRows(10) - tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") - tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") - tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") - tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") + # tdSql.query("select ts,diff(col1),ts from stb group by tbname") + # tdSql.checkRows(10) + # tdSql.checkData(0, 0, "2018-09-17 09:00:00.000") + # tdSql.checkData(0, 1, "2018-09-17 09:00:00.000") + # tdSql.checkData(0, 3, "2018-09-17 09:00:00.000") + # tdSql.checkData(9, 0, "2018-09-17 09:00:00.009") + # tdSql.checkData(9, 1, "2018-09-17 09:00:00.009") + # tdSql.checkData(9, 3, "2018-09-17 09:00:00.009") tdSql.query("select diff(col1) from stb_1") tdSql.checkRows(10) @@ -132,15 +137,6 @@ class TDTestCase: tdSql.query("select diff(col6) from stb_1") tdSql.checkRows(10) - - self.insertData() - - tdSql.query("select diff(col) from st group by tbname") - tdSql.checkRows(185) - - tdSql.error("select diff(col) from st group by dev") - - tdSql.error("select diff(col) from st group by col") def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index 55881db149..a62960cf43 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -1242,11 +1242,11 @@ class TDTestCase: # and or query_sql = f'select {select_elm} from {tb_name} where c7 > "binary" and c7 >= "binary8" or c7 < "binary9" and c7 <= "binary" and c7 != 2 and c7 <> 2 and c7 = 4 or c7 is not null and c7 between 2 and 4 and c7 not between 1 and 2 and c7 in (2,4) and c7 not in (1,2) or c7 match "binary[28]" or c7 nmatch "binary"' tdSql.query(query_sql) - tdSql.checkRows(11) - tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) if select_elm == "*" else False query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c7 > "binary" and c7 >= "binary8" or c7 < "binary9" and c7 <= "binary" and c7 != 2 and c7 <> 2 and c7 = 4 or c7 is not null and c7 between 2 and 4 and c7 not between 1 and 2 and c7 in (2,4) and c7 not in (1,2) or c7 match "binary[28]" or c7 nmatch "binary"' tdSql.query(query_sql) - tdSql.checkRows(11) + tdSql.checkRows(1) def queryNcharCol(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f713f707cb..a6b4408cdc 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -18,6 +18,7 @@ python3 ./test.py -f 2-query/char_length.py python3 ./test.py -f 2-query/upper.py python3 ./test.py -f 2-query/lower.py python3 ./test.py -f 2-query/join.py +python3 ./test.py -f 2-query/cast.py # python3 ./test.py -f 2-query/concat.py # after wal ,crash occured # python3 ./test.py -f 2-query/concat_ws.py @@ -31,8 +32,8 @@ python3 ./test.py -f 2-query/last.py #python3 ./test.py -f 2-query/To_iso8601.py python3 ./test.py -f 2-query/To_unixtimestamp.py python3 ./test.py -f 2-query/timetruncate.py - -# python3 ./test.py -f 2-query/Timediff.py +# python3 ./test.py -f 2-query/diff.py +python3 ./test.py -f 2-query/Timediff.py #python3 ./test.py -f 2-query/cast.py @@ -48,4 +49,5 @@ python3 ./test.py -f 2-query/cos.py python3 ./test.py -f 2-query/tan.py python3 ./test.py -f 2-query/arcsin.py python3 ./test.py -f 2-query/arccos.py -python3 ./test.py -f 2-query/arctan.py \ No newline at end of file +python3 ./test.py -f 2-query/arctan.py +# python3 ./test.py -f 2-query/query_cols_tags_and_or.py diff --git a/tests/system-test/insert.json b/tests/system-test/insert.json new file mode 100644 index 0000000000..5dea9eabfe --- /dev/null +++ b/tests/system-test/insert.json @@ -0,0 +1,76 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 16, + "create_table_thread_count": 1, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 10000, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups":4, + "replica": 1, + "precision": "ms" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "stb_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 100000, + "interlace_rows": 0, + "insert_interval": 0, + "disorder_ratio": 0, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 4, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "TINYINT", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["beijing", "shanghai"] + } + ] + } + ] + } + ] +}