xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/TD-14481-3.0

This commit is contained in:
Cary Xu 2022-05-10 13:29:12 +08:00
parent a82b728b38 1df5468f79
commit 22cb29e9b8
45 changed files with 2493 additions and 2758 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
include/common/tdata.h Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDATA_H_
#define _TD_TDATA_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
typedef struct STaosData TDATA, tdata_t;
typedef enum {
TAOS_META_STABLE_DATA = 0, // super table meta
TAOS_META_TABLE_DATA, // non-super table meta
TAOS_TS_ROW_DATA, // row time-series data
TAOS_TS_COL_DATA, // col time-series data
TAOS_DATA_MAX
} ETaosDataT;
struct STaosData {
ETaosDataT type;
uint32_t nPayload;
uint8_t *pPayload;
};
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDATA_H_*/

410
include/common/tdataformat.h
View File

@ -150,29 +150,6 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags,
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
// ----------------- Semantic timestamp key definition
#ifdef TD_2_0
typedef uint64_t TKEY;
#define TKEY_INVALID UINT64_MAX
#define TKEY_NULL TKEY_INVALID
#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff)
#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
#else
// typedef uint64_t TKEY;
#define TKEY TSKEY
@ -192,8 +169,6 @@ typedef uint64_t TKEY;
#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
#endif
static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
TSKEY lkey = key;
if (key > MAX_TS_KEY) {
@ -218,157 +193,6 @@ static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) {
}
}
#if 0
// ----------------- Data row structure
/* A data row, the format is like below:
* |<------------------------------------------------ len ---------------------------------->|
* |<-- Head -->|<--------- flen -------------->| |
* +---------------------+---------------------------------+---------------------------------+
* | uint16_t | int16_t | | |
* +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+
*
* NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/
typedef void *SDataRow;
#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535
#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r))
#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
#define dataRowKey(r) tdGetKey(dataRowTKey(r))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row);
void tdInitDataRow(SDataRow row, STSchema *pSchema);
SDataRow tdDataRowDup(SDataRow row);
// offset here not include dataRow header length
static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
int32_t offset) {
assert(value != NULL);
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
if (IS_VAR_DATA_TYPE(type)) {
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
if (isCopyVarData) {
memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
}
dataRowLen(row) += varDataTLen(value);
} else {
if (offset == 0) {
assert(type == TSDB_DATA_TYPE_TIMESTAMP);
TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]);
} else {
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
}
}
return 0;
}
// offset here not include dataRow header length
static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
return tdAppendDataColVal(row, value, true, type, offset);
}
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
if (IS_VAR_DATA_TYPE(type)) {
return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
} else {
return POINTER_SHIFT(row, offset);
}
}
static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) {
return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset);
}
static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
int8_t type = pSchema->columns[idx].type;
return tdGetRowDataOfCol(row, type, offset);
}
static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
int8_t type = pSchema->columns[idx].type;
return isNull(tdGetRowDataOfCol(row, type, offset), type);
}
static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
int8_t type = pSchema->columns[idx].type;
int16_t bytes = pSchema->columns[idx].bytes;
setNull(tdGetRowDataOfCol(row, type, offset), type, bytes);
}
static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src,
STSchema *pSrcSchema, int32_t srcIdx) {
int8_t type = pDstSchema->columns[dstIdx].type;
assert(type == pSrcSchema->columns[srcIdx].type);
void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx);
void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx);
switch (type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
*(VarDataOffsetT *)pData = *(VarDataOffsetT *)value;
pData = POINTER_SHIFT(dst, *(VarDataOffsetT *)pData);
value = POINTER_SHIFT(src, *(VarDataOffsetT *)value);
memcpy(pData, value, varDataTLen(value));
break;
case TSDB_DATA_TYPE_NULL:
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT:
*(uint8_t *)pData = *(uint8_t *)value;
break;
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT:
*(uint16_t *)pData = *(uint16_t *)value;
break;
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT:
*(uint32_t *)pData = *(uint32_t *)value;
break;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT:
*(uint64_t *)pData = *(uint64_t *)value;
break;
case TSDB_DATA_TYPE_FLOAT:
SET_FLOAT_PTR(pData, value);
break;
case TSDB_DATA_TYPE_DOUBLE:
SET_DOUBLE_PTR(pData, value);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
if (pSrcSchema->columns[srcIdx].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
*(TSKEY *)pData = tdGetKey(*(TKEY *)value);
} else {
*(TSKEY *)pData = *(TSKEY *)value;
}
break;
default:
memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes);
}
}
#endif
// ----------------- Data column structure
// SDataCol arrangement: data => bitmap => dataOffset
typedef struct SDataCol {
@ -398,29 +222,6 @@ void *dataColSetOffset(SDataCol *pCol, int32_t nEle);
bool isNEleNull(SDataCol *pCol, int32_t nEle);
#if 0
// Get the data pointer from a column-wised data
static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) {
if (isAllRowsNull(pCol)) {
return getNullValue(pCol->type);
}
if (IS_VAR_DATA_TYPE(pCol->type)) {
return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
} else {
return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
}
}
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) {
assert(rows > 0);
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
} else {
return TYPE_BYTES[pDataCol->type] * rows;
}
}
#endif
typedef struct {
col_id_t maxCols; // max number of columns
col_id_t numOfCols; // Total number of cols
@ -479,7 +280,8 @@ void tdResetDataCols(SDataCols *pCols);
int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
SDataCols *tdFreeDataCols(SDataCols *pCols);
int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer);
int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update,
TDRowVerT maxVer);
// ----------------- K-V data row structure
/* |<-------------------------------------- len -------------------------------------------->|
@ -542,54 +344,6 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
}
#if 0
// offset here not include kvRow header length
static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
int32_t offset) {
assert(value != NULL);
int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE;
SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
char * ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));
pColIdx->colId = colId;
pColIdx->offset = kvRowLen(row); // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
if (IS_VAR_DATA_TYPE(type)) {
if (isCopyValData) {
memcpy(ptr, value, varDataTLen(value));
}
kvRowLen(row) += varDataTLen(value);
} else {
if (offset == 0) {
assert(type == TSDB_DATA_TYPE_TIMESTAMP);
TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
} else {
memcpy(ptr, value, TYPE_BYTES[type]);
}
kvRowLen(row) += TYPE_BYTES[type];
}
return 0;
}
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); }
static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_t *nIdx) {
while (*nIdx < kvRowNCols(row)) {
SColIdx *pColIdx = kvRowColIdxAt(row, *nIdx);
if (pColIdx->colId == colId) {
++(*nIdx);
return tdGetKvRowDataOfCol(row, pColIdx->offset);
} else if (pColIdx->colId > colId) {
return NULL;
} else {
++(*nIdx);
}
}
return NULL;
}
#endif
// ----------------- K-V data row builder
typedef struct {
int16_t tCols;
@ -632,166 +386,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
return 0;
}
#if 0
// ----------------- SMemRow appended with tuple row structure
/*
* |---------|------------------------------------------------- len ---------------------------------->|
* |<-------- Head ------>|<--------- flen -------------->| |
* |---------+---------------------+---------------------------------+---------------------------------+
* | uint8_t | uint16_t | int16_t | | |
* |---------+----------+----------+---------------------------------+---------------------------------+
* | flag | len | sversion | First part | Second part |
* +---------+----------+----------+---------------------------------+---------------------------------+
*
* NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/
// ----------------- SMemRow appended with extended K-V data row structure
/* |--------------------|------------------------------------------------ len ---------------------------------->|
* |<------------- Head ------------>|<--------- flen -------------->| |
* |--------------------+----------+--------------------------------------------+---------------------------------+
* | uint8_t | int16_t | uint16_t | int16_t | | |
* |---------+----------+----------+----------+---------------------------------+---------------------------------+
* | flag | sversion | len | ncols | cols index | data part |
* |---------+----------+----------+----------+---------------------------------+---------------------------------+
*/
typedef void *SMemRow;
#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t)
#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t)
#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE)
#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
#define SMEM_ROW_DATA 0x0U // SDataRow
#define SMEM_ROW_KV 0x01U // SKVRow
#define KVRatioConvert (0.9f)
#define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory
#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r))
#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
#define isKvRow(r) (SMEM_ROW_KV == memRowType(r))
#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag
#define memRowKvBody(r) \
POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse SKVRow
#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535
#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535
#define memRowDataTLen(r) \
((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE)) // using uint32_t/int32_t to store the TLen
#define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE))
#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen
static FORCE_INLINE char *memRowEnd(SMemRow row) {
if (isDataRow(row)) {
return (char *)dataRowEnd(memRowDataBody(row));
} else {
return (char *)kvRowEnd(memRowKvBody(row));
}
}
#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r))
#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version
#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v))
#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r)))
#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r)))
#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r)))
#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r)))
#define memRowSetTKey(r, k) \
do { \
if (isDataRow(r)) { \
dataRowTKey(memRowDataBody(r)) = (k); \
} else { \
kvRowTKey(memRowKvBody(r)) = (k); \
} \
} while (0)
#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v))
#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r))
#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE)
#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r))
SMemRow tdMemRowDup(SMemRow row);
void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull);
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) {
if (isDataRow(row)) {
return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
} else {
return tdGetKVRowValOfCol(memRowKvBody(row), colId);
}
}
/**
* NOTE:
* 1. Applicable to scan columns one by one
* 2. offset here including the header size
*/
static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_t colType, int32_t offset,
int32_t *kvNIdx) {
if (isDataRow(row)) {
return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
} else {
return tdGetKVRowValOfColEx(memRowKvBody(row), colId, kvNIdx);
}
}
static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
int8_t type, int32_t offset) {
if (isDataRow(row)) {
tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);
} else {
tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset);
}
return 0;
}
// make sure schema->flen appended for SDataRow
static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value, int8_t colType) {
int32_t len = 0;
if (IS_VAR_DATA_TYPE(colType)) {
len += varDataTLen(value);
if (rowType == SMEM_ROW_KV) {
len += sizeof(SColIdx);
}
} else {
if (rowType == SMEM_ROW_KV) {
len += TYPE_BYTES[colType];
len += sizeof(SColIdx);
}
}
return len;
}
typedef struct {
int16_t colId;
uint8_t colType;
char * colVal;
} SColInfo;
static FORCE_INLINE void setSColInfo(SColInfo *colInfo, int16_t colId, uint8_t colType, char *colVal) {
colInfo->colId = colId;
colInfo->colType = colType;
colInfo->colVal = colVal;
}
SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
#endif
#ifdef __cplusplus
}

1276
include/common/trow.h
View File

File diff suppressed because it is too large Load Diff

2
include/libs/function/functionMgt.h
View File

@ -31,13 +31,13 @@ typedef enum EFunctionType {
FUNCTION_TYPE_ELAPSED,
FUNCTION_TYPE_IRATE,
FUNCTION_TYPE_LAST_ROW,
FUNCTION_TYPE_LEASTSQUARES,
FUNCTION_TYPE_MAX,
FUNCTION_TYPE_MIN,
FUNCTION_TYPE_MODE,
FUNCTION_TYPE_PERCENTILE,
FUNCTION_TYPE_SPREAD,
FUNCTION_TYPE_STDDEV,
FUNCTION_TYPE_LEASTSQUARES,
FUNCTION_TYPE_SUM,
FUNCTION_TYPE_TWA,
FUNCTION_TYPE_HISTOGRAM,

2
include/libs/qcom/query.h
View File

@ -184,7 +184,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))

1
include/util/taoserror.h
View File

@ -88,6 +88,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0114)
#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0115)
#define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0116)
#define TSDB_CODE_NEED_RETRY TAOS_DEF_ERROR_CODE(0, 0x0117)
#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0140)
#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0141)

70
include/util/tskiplist2.h
View File

@ -1,70 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_SKIPLIST2_H_
#define _TD_UTIL_SKIPLIST2_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
#define SL_MAX_LEVEL 15
typedef struct SSkipList2 SSkipList2;
typedef struct SSLCursor SSLCursor;
typedef struct SSLCfg SSLCfg;
typedef struct SSLNode SSLNode;
typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2);
// SSkipList2
int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl);
int32_t slClose(SSkipList2 *pSl);
int32_t slClear(SSkipList2 *pSl);
// SSLCursor
int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc);
int32_t slcClose(SSLCursor *pSlc);
int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey);
int32_t slcMoveToNext(SSLCursor *pSlc);
int32_t slcMoveToPrev(SSLCursor *pSlc);
int32_t slcMoveToFirst(SSLCursor *pSlc);
int32_t slcMoveToLast(SSLCursor *pSlc);
int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData);
int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData);
int32_t slcDrop(SSLCursor *pSlc);
// struct
struct SSLCfg {
int8_t maxLevel;
int32_t nKey;
int32_t nData;
tslCmprFn cmprFn;
void *pPool;
void *(*xMalloc)(void *, int32_t size);
void (*xFree)(void *, void *);
};
struct SSLCursor {
SSkipList2 *pSl;
SSLNode **forwards[SL_MAX_LEVEL];
};
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_SKIPLIST2_H_*/

1
source/client/inc/clientInt.h
View File

@ -310,6 +310,7 @@ void hbMgrInitMqHbRspHandle();
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList);
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
#ifdef __cplusplus
}

1
source/client/inc/clientStmt.h
View File

@ -60,6 +60,7 @@ typedef struct SStmtBindInfo {
int32_t sBindRowNum;
int32_t sBindLastIdx;
int8_t tbType;
bool tagsCached;
void* boundTags;
char* tbName;
SName sname;

2
source/client/src/clientSml.c
View File

@ -405,7 +405,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta);
if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_MND_INVALID_STB) {
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_INVALID_STB) {
SSchemaAction schemaAction = {0};
schemaAction.action = SCHEMA_ACTION_CREATE_STABLE;
memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen);

53
source/client/src/clientStmt.c
View File

@ -29,6 +29,11 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
/*
if ((pStmt->sql.type == STMT_TYPE_MULTI_INSERT) && ()) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
*/
break;
case STMT_BIND_COL:
if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) {
@ -123,6 +128,7 @@ int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
pStmt->bInfo.tbSuid = pTableMeta->suid;
pStmt->bInfo.tbType = pTableMeta->tableType;
pStmt->bInfo.boundTags = tags;
pStmt->bInfo.tagsCached = false;
return TSDB_CODE_SUCCESS;
}
@ -207,8 +213,6 @@ int32_t stmtParseSql(STscStmt* pStmt) {
STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR);
}
STMT_ERR_RET(stmtCacheBlock(pStmt));
return TSDB_CODE_SUCCESS;
}
@ -219,8 +223,10 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
pStmt->bInfo.needParse = true;
taosMemoryFreeClear(pStmt->bInfo.tbName);
destroyBoundColumnInfo(pStmt->bInfo.boundTags);
taosMemoryFreeClear(pStmt->bInfo.boundTags);
if (!pStmt->bInfo.tagsCached) {
destroyBoundColumnInfo(pStmt->bInfo.boundTags);
taosMemoryFreeClear(pStmt->bInfo.boundTags);
}
return TSDB_CODE_SUCCESS;
}
@ -275,6 +281,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
qDestroyStmtDataBlock(pCache->pDataBlock);
destroyBoundColumnInfo(pCache->boundTags);
taosMemoryFreeClear(pCache->boundTags);
pIter = taosHashIterate(pStmt->sql.pTableCache, pIter);
}
@ -302,7 +309,15 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
STableMeta *pTableMeta = NULL;
SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
int32_t code = catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta);
if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
STMT_ERR_RET(stmtCleanBindInfo(pStmt));
return TSDB_CODE_SUCCESS;
}
STMT_ERR_RET(code);
uint64_t uid = pTableMeta->uid;
uint64_t suid = pTableMeta->suid;
int8_t tableType = pTableMeta->tableType;
@ -328,6 +343,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
pStmt->bInfo.tbSuid = suid;
pStmt->bInfo.tbType = tableType;
pStmt->bInfo.boundTags = pCache->boundTags;
pStmt->bInfo.tagsCached = true;
return TSDB_CODE_SUCCESS;
}
@ -340,6 +356,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
pStmt->bInfo.tbSuid = suid;
pStmt->bInfo.tbType = tableType;
pStmt->bInfo.boundTags = pCache->boundTags;
pStmt->bInfo.tagsCached = true;
STableDataBlocks* pNewBlock = NULL;
STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock));
@ -448,10 +465,12 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) {
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
if (!pStmt->bInfo.needParse) {
return TSDB_CODE_SUCCESS;
}
STMT_ERR_RET(stmtParseSql(pStmt));
STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
if (NULL == pDataBlock) {
tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
@ -501,8 +520,6 @@ int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fiel
int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
STscStmt* pStmt = (STscStmt*)stmt;
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
pStmt->bInfo.needParse = false;
}
@ -520,6 +537,8 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
STMT_ERR_RET(stmtParseSql(pStmt));
}
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
if (STMT_TYPE_QUERY == pStmt->sql.type) {
if (NULL == pStmt->sql.pQueryPlan) {
STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
@ -587,6 +606,16 @@ int stmtExec(TAOS_STMT *stmt) {
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
}
if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) {
code = refreshMeta(pStmt->exec.pRequest->pTscObj, pStmt->exec.pRequest);
if (code) {
pStmt->exec.pRequest->code = code;
} else {
STMT_ERR_RET(stmtResetStmt(pStmt));
STMT_ERR_RET(TSDB_CODE_NEED_RETRY);
}
}
STMT_ERR_JRET(pStmt->exec.pRequest->code);
pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest);
@ -613,13 +642,11 @@ int stmtClose(TAOS_STMT *stmt) {
const char *stmtErrstr(TAOS_STMT *stmt) {
STscStmt* pStmt = (STscStmt*)stmt;
if (stmt == NULL) {
if (stmt == NULL || NULL == pStmt->exec.pRequest) {
return (char*) tstrerror(terrno);
}
if (pStmt->exec.pRequest) {
pStmt->exec.pRequest->code = terrno;
}
pStmt->exec.pRequest->code = terrno;
return taos_errstr(pStmt->exec.pRequest);
}

14
source/common/src/tdata.c Normal file
View File

@ -0,0 +1,14 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

450
source/common/src/tdataformat.c
View File

@ -129,50 +129,6 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
return buf;
}
#if 0
int32_t tEncodeSTColumn(SCoder *pEncoder, const STColumn *pCol) {
if (tEncodeI16(pEncoder, pCol->colId) < 0) return -1;
if (tEncodeI8(pEncoder, pCol->type) < 0) return -1;
if (tEncodeI8(pEncoder, pCol->sma) < 0) return -1;
if (tEncodeI32(pEncoder, pCol->bytes) < 0) return -1;
if (tEncodeI32(pEncoder, pCol->offset) < 0) return -1;
return pEncoder->pos;
}
int32_t tDecodeSTColumn(SCoder *pDecoder, STColumn *pCol) {
if (tDecodeI16(pDecoder, &pCol->colId) < 0) return -1;
if (tDecodeI8(pDecoder, &pCol->type) < 0) return -1;
if (tDecodeI8(pDecoder, &pCol->sma) < 0) return -1;
if (tDecodeI32(pDecoder, &pCol->bytes) < 0) return -1;
if (tDecodeI32(pDecoder, &pCol->offset) < 0) return -1;
return 0;
}
int32_t tEncodeSchema(SCoder *pEncoder, const STSchema *pSchema) {
if (tEncodeI32(pEncoder, pSchema->numOfCols) < 0) return -1;
if (tEncodeI16(pEncoder, pSchema->version) < 0) return -1;
if (tEncodeU16(pEncoder, pSchema->flen) < 0) return -1;
if (tEncodeI32(pEncoder, pSchema->vlen) < 0) return -1;
if (tEncodeI32(pEncoder, pSchema->tlen) < 0) return -1;
for (int32_t i = 0; i < schemaNCols(pSchema); i++) {
const STColumn *pCol = schemaColAt(pSchema, i);
if (tEncodeSTColumn(pEncoder, pCol) < 0) return -1;
}
return 0;
}
int32_t tDecodeSchema(SCoder *pDecoder, STSchema *pSchema) {
if (tDecodeI32(pDecoder, &pSchema->numOfCols) < 0) return -1;
if (tDecodeI16(pDecoder, &pSchema->version) < 0) return -1;
if (tDecodeU16(pDecoder, &pSchema->flen) < 0) return -1;
if (tDecodeI32(pDecoder, &pSchema->vlen) < 0) return -1;
if (tDecodeI32(pDecoder, &pSchema->tlen) < 0) return -1;
return 0;
}
#endif
int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
if (pBuilder == NULL) return -1;
@ -260,49 +216,6 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
return pSchema;
}
#if 0
/**
* Initialize a data row
*/
void tdInitDataRow(SDataRow row, STSchema *pSchema) {
dataRowSetLen(row, TD_DATA_ROW_HEAD_SIZE + schemaFLen(pSchema));
dataRowSetVersion(row, schemaVersion(pSchema));
}
SDataRow tdNewDataRowFromSchema(STSchema *pSchema) {
int32_t size = dataRowMaxBytesFromSchema(pSchema);
SDataRow row = taosMemoryMalloc(size);
if (row == NULL) return NULL;
tdInitDataRow(row, pSchema);
return row;
}
/**
* Free the SDataRow object
*/
void tdFreeDataRow(SDataRow row) {
if (row) taosMemoryFree(row);
}
SDataRow tdDataRowDup(SDataRow row) {
SDataRow trow = taosMemoryMalloc(dataRowLen(row));
if (trow == NULL) return NULL;
dataRowCpy(trow, row);
return trow;
}
SMemRow tdMemRowDup(SMemRow row) {
SMemRow trow = taosMemoryMalloc(memRowTLen(row));
if (trow == NULL) return NULL;
memRowCpy(trow, row);
return trow;
}
#endif
void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
pDataCol->type = colType(pCol);
pDataCol->colId = colColId(pCol);
@ -312,39 +225,6 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
pDataCol->len = 0;
}
#if 0
// value from timestamp should be TKEY here instead of TSKEY
int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
ASSERT(pCol != NULL && value != NULL);
if (isAllRowsNull(pCol)) {
if (isNull(value, pCol->type)) {
// all null value yet, just return
return 0;
}
if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1;
if (numOfRows > 0) {
// Find the first not null value, fill all previouse values as NULL
dataColSetNEleNull(pCol, numOfRows);
}
}
if (IS_VAR_DATA_TYPE(pCol->type)) {
// set offset
pCol->dataOff[numOfRows] = pCol->len;
// Copy data
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
// Update the length
pCol->len += varDataTLen(value);
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
pCol->len += pCol->bytes;
}
return 0;
}
#endif
static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
@ -361,31 +241,6 @@ bool isNEleNull(SDataCol *pCol, int nEle) {
return true;
}
#if 0
static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->dataOff[index] = pCol->len;
char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
setVardataNull(ptr, pCol->type);
pCol->len += varDataTLen(ptr);
} else {
setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);
pCol->len += TYPE_BYTES[pCol->type];
}
}
static void dataColSetNEleNull(SDataCol *pCol, int nEle, int8_t bitmapMode) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->len = 0;
for (int i = 0; i < nEle; ++i) {
dataColSetNullAt(pCol, i);
}
} else {
setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
pCol->len = TYPE_BYTES[pCol->type] * nEle;
}
}
#endif
void *dataColSetOffset(SDataCol *pCol, int nEle) {
ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR)));
@ -483,42 +338,6 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
return NULL;
}
#if 0
SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
if (pRet == NULL) return NULL;
pRet->numOfCols = pDataCols->numOfCols;
pRet->sversion = pDataCols->sversion;
if (keepData) pRet->numOfRows = pDataCols->numOfRows;
for (int i = 0; i < pDataCols->numOfCols; i++) {
pRet->cols[i].type = pDataCols->cols[i].type;
pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
pRet->cols[i].colId = pDataCols->cols[i].colId;
pRet->cols[i].bytes = pDataCols->cols[i].bytes;
pRet->cols[i].offset = pDataCols->cols[i].offset;
if (keepData) {
if (pDataCols->cols[i].len > 0) {
if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) {
tdFreeDataCols(pRet);
return NULL;
}
pRet->cols[i].len = pDataCols->cols[i].len;
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints;
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
}
}
}
}
return pRet;
}
#endif
void tdResetDataCols(SDataCols *pCols) {
if (pCols != NULL) {
pCols->numOfRows = 0;
@ -528,180 +347,6 @@ void tdResetDataCols(SDataCols *pCols) {
}
}
}
#if 0
static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
int rcol = 0;
int dcol = 0;
while (dcol < pCols->numOfCols) {
bool setCol = 0;
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (rcol >= schemaNCols(pSchema)) {
dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
dcol++;
continue;
}
STColumn *pRowCol = schemaColAt(pSchema, rcol);
if (pRowCol->colId == pDataCol->colId) {
void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE);
if(!isNull(value, pDataCol->type)) setCol = 1;
dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
dcol++;
rcol++;
} else if (pRowCol->colId < pDataCol->colId) {
rcol++;
} else {
if(forceSetNull || setCol) {
dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
}
dcol++;
}
}
pCols->numOfRows++;
}
static void tdAppendKVRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row));
int rcol = 0;
int dcol = 0;
int nRowCols = kvRowNCols(row);
while (dcol < pCols->numOfCols) {
bool setCol = 0;
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) {
dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
++dcol;
continue;
}
SColIdx *colIdx = kvRowColIdxAt(row, rcol);
if (colIdx->colId == pDataCol->colId) {
void *value = tdGetKvRowDataOfCol(row, colIdx->offset);
if(!isNull(value, pDataCol->type)) setCol = 1;
dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
++dcol;
++rcol;
} else if (colIdx->colId < pDataCol->colId) {
++rcol;
} else {
if(forceSetNull || setCol) {
dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
}
++dcol;
}
}
pCols->numOfRows++;
}
void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
if (isDataRow(row)) {
tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull);
} else if (isKvRow(row)) {
tdAppendKVRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull);
} else {
ASSERT(0);
}
}
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
ASSERT(target->numOfCols == source->numOfCols);
int offset = 0;
if (pOffset == NULL) {
pOffset = &offset;
}
SDataCols *pTarget = NULL;
if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
for (int i = 0; i < rowsToMerge; i++) {
for (int j = 0; j < source->numOfCols; j++) {
if (source->cols[j].len > 0 || target->cols[j].len > 0) {
dataColAppendVal(target->cols + j, tdGetColDataOfRow(source->cols + j, i + (*pOffset)), target->numOfRows,
target->maxPoints);
}
}
target->numOfRows++;
}
(*pOffset) += rowsToMerge;
} else {
pTarget = tdDupDataCols(target, true);
if (pTarget == NULL) goto _err;
int iter1 = 0;
tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows,
pTarget->numOfRows + rowsToMerge, forceSetNull);
}
tdFreeDataCols(pTarget);
return 0;
_err:
tdFreeDataCols(pTarget);
return -1;
}
// src2 data has more priority than src1
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows, bool forceSetNull) {
tdResetDataCols(target);
ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
while (target->numOfRows < tRows) {
if (*iter1 >= limit1 && *iter2 >= limit2) break;
TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type);
if (src1->cols[i].len > 0 || target->cols[i].len > 0) {
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
target->maxPoints);
}
}
target->numOfRows++;
(*iter1)++;
} else if (key1 >= key2) {
if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type);
if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) {
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
target->maxPoints);
} else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
target->maxPoints);
} else if(target->cols[i].len > 0) {
dataColSetNullAt(&target->cols[i], target->numOfRows);
}
}
target->numOfRows++;
}
(*iter2)++;
if (key1 == key2) (*iter1)++;
}
ASSERT(target->numOfRows <= target->maxPoints);
}
}
#endif
SKVRow tdKVRowDup(SKVRow row) {
SKVRow trow = taosMemoryMalloc(kvRowLen(row));
@ -859,98 +504,3 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) {
return row;
}
#if 0
SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) {
#if 0
ASSERT(memRowKey(row1) == memRowKey(row2));
ASSERT(schemaVersion(pSchema1) == memRowVersion(row1));
ASSERT(schemaVersion(pSchema2) == memRowVersion(row2));
ASSERT(schemaVersion(pSchema1) >= schemaVersion(pSchema2));
#endif
SArray *stashRow = taosArrayInit(pSchema1->numOfCols, sizeof(SColInfo));
if (stashRow == NULL) {
return NULL;
}
SMemRow pRow = buffer;
SDataRow dataRow = memRowDataBody(pRow);
memRowSetType(pRow, SMEM_ROW_DATA);
dataRowSetVersion(dataRow, schemaVersion(pSchema1)); // use latest schema version
dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pSchema1->flen));
TDRowLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE;
int32_t i = 0; // row1
int32_t j = 0; // row2
int32_t nCols1 = schemaNCols(pSchema1);
int32_t nCols2 = schemaNCols(pSchema2);
SColInfo colInfo = {0};
int32_t kvIdx1 = 0, kvIdx2 = 0;
while (i < nCols1) {
STColumn *pCol = schemaColAt(pSchema1, i);
void * val1 = tdGetMemRowDataOfColEx(row1, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx1);
// if val1 != NULL, use val1;
if (val1 != NULL && !isNull(val1, pCol->type)) {
tdAppendColVal(dataRow, val1, pCol->type, pCol->offset);
kvLen += tdGetColAppendLen(SMEM_ROW_KV, val1, pCol->type);
setSColInfo(&colInfo, pCol->colId, pCol->type, val1);
taosArrayPush(stashRow, &colInfo);
++i; // next col
continue;
}
void *val2 = NULL;
while (j < nCols2) {
STColumn *tCol = schemaColAt(pSchema2, j);
if (tCol->colId < pCol->colId) {
++j;
continue;
}
if (tCol->colId == pCol->colId) {
val2 = tdGetMemRowDataOfColEx(row2, tCol->colId, tCol->type, TD_DATA_ROW_HEAD_SIZE + tCol->offset, &kvIdx2);
} else if (tCol->colId > pCol->colId) {
// set NULL
}
break;
} // end of while(j<nCols2)
if (val2 == NULL) {
val2 = (void *)getNullValue(pCol->type);
}
tdAppendColVal(dataRow, val2, pCol->type, pCol->offset);
if (!isNull(val2, pCol->type)) {
kvLen += tdGetColAppendLen(SMEM_ROW_KV, val2, pCol->type);
setSColInfo(&colInfo, pCol->colId, pCol->type, val2);
taosArrayPush(stashRow, &colInfo);
}
++i; // next col
}
dataLen = memRowTLen(pRow);
if (kvLen < dataLen) {
// scan stashRow and generate SKVRow
memset(buffer, 0, sizeof(dataLen));
SMemRow tRow = buffer;
memRowSetType(tRow, SMEM_ROW_KV);
SKVRow kvRow = (SKVRow)memRowKvBody(tRow);
int16_t nKvNCols = (int16_t) taosArrayGetSize(stashRow);
kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nKvNCols));
kvRowSetNCols(kvRow, nKvNCols);
memRowSetKvVersion(tRow, pSchema1->version);
int32_t toffset = 0;
int16_t k;
for (k = 0; k < nKvNCols; ++k) {
SColInfo *pColInfo = taosArrayGet(stashRow, k);
tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset);
toffset += sizeof(SColIdx);
}
ASSERT(kvLen == memRowTLen(tRow));
}
taosArrayDestroy(stashRow);
return buffer;
}
#endif

6
source/common/src/tmsg.c
View File

@ -1233,21 +1233,21 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
for (int32_t i = 0; i < numOfCreatedDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0};
if (tDecodeCStrTo(pDecoder, db) < 0) return -1;
int32_t len = strlen(db) + 1;
int32_t len = strlen(db);
taosHashPut(pRsp->createdDbs, db, len, db, len);
}
for (int32_t i = 0; i < numOfReadDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0};
if (tDecodeCStrTo(pDecoder, db) < 0) return -1;
int32_t len = strlen(db) + 1;
int32_t len = strlen(db);
taosHashPut(pRsp->readDbs, db, len, db, len);
}
for (int32_t i = 0; i < numOfWriteDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0};
if (tDecodeCStrTo(pDecoder, db) < 0) return -1;
int32_t len = strlen(db) + 1;
int32_t len = strlen(db);
taosHashPut(pRsp->writeDbs, db, len, db, len);
}

934
source/common/src/trow.c
View File

@ -33,6 +33,7 @@ const uint8_t tdVTypeByte[2][3] = {{
// declaration
static uint8_t tdGetBitmapByte(uint8_t byte);
static int32_t tdCompareColId(const void *arg1, const void *arg2);
// static void dataColSetNEleNull(SDataCol *pCol, int nEle);
@ -917,3 +918,936 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
return pRet;
}
void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) {
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pSchema);
tdSTSRowIterReset(&iter, row);
printf("%s >>>", tag);
for (int i = 0; i < pSchema->numOfCols; ++i) {
STColumn *stCol = pSchema->columns + i;
SCellVal sVal = {255, NULL};
if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
break;
}
ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2);
tdSCellValPrint(&sVal, stCol->type);
}
printf("\n");
}
void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
if (tdValTypeIsNull(pVal->valType)) {
printf("NULL ");
return;
} else if (tdValTypeIsNone(pVal->valType)) {
printf("NONE ");
return;
}
switch (colType) {
case TSDB_DATA_TYPE_BOOL:
printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true");
break;
case TSDB_DATA_TYPE_TINYINT:
printf("%" PRIi8 " ", *(int8_t *)pVal->val);
break;
case TSDB_DATA_TYPE_SMALLINT:
printf("%" PRIi16 " ", *(int16_t *)pVal->val);
break;
case TSDB_DATA_TYPE_INT:
printf("%" PRIi32 " ", *(int32_t *)pVal->val);
break;
case TSDB_DATA_TYPE_BIGINT:
printf("%" PRIi64 " ", *(int64_t *)pVal->val);
break;
case TSDB_DATA_TYPE_FLOAT:
printf("%f ", *(float *)pVal->val);
break;
case TSDB_DATA_TYPE_DOUBLE:
printf("%lf ", *(double *)pVal->val);
break;
case TSDB_DATA_TYPE_VARCHAR:
printf("VARCHAR ");
break;
case TSDB_DATA_TYPE_TIMESTAMP:
printf("%" PRIi64 " ", *(int64_t *)pVal->val);
break;
case TSDB_DATA_TYPE_NCHAR:
printf("NCHAR ");
break;
case TSDB_DATA_TYPE_UTINYINT:
printf("%" PRIu8 " ", *(uint8_t *)pVal->val);
break;
case TSDB_DATA_TYPE_USMALLINT:
printf("%" PRIu16 " ", *(uint16_t *)pVal->val);
break;
case TSDB_DATA_TYPE_UINT:
printf("%" PRIu32 " ", *(uint32_t *)pVal->val);
break;
case TSDB_DATA_TYPE_UBIGINT:
printf("%" PRIu64 " ", *(uint64_t *)pVal->val);
break;
case TSDB_DATA_TYPE_JSON:
printf("JSON ");
break;
case TSDB_DATA_TYPE_VARBINARY:
printf("VARBIN ");
break;
case TSDB_DATA_TYPE_DECIMAL:
printf("DECIMAL ");
break;
case TSDB_DATA_TYPE_BLOB:
printf("BLOB ");
break;
case TSDB_DATA_TYPE_MEDIUMBLOB:
printf("MedBLOB ");
break;
// case TSDB_DATA_TYPE_BINARY:
// printf("BINARY ");
// break;
case TSDB_DATA_TYPE_MAX:
printf("UNDEF ");
break;
default:
printf("UNDEF ");
break;
}
}
int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) {
ASSERT(rows > 0);
int32_t result = 0;
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
result += pDataCol->dataOff[rows - 1];
SCellVal val = {0};
if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) {
TASSERT(0);
}
// Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4
result += varDataTLen(val.val);
// TODO: later on, don't save Null/None for VarDataT for 3.0
// if (tdValTypeIsNorm(val.valType)) {
// result += varDataTLen(val.val);
// }
} else {
result += TYPE_BYTES[pDataCol->type] * rows;
}
ASSERT(pDataCol->len == result);
return result;
}
bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow));
return true;
}
void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow));
tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx);
return true;
}
bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx,
SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow));
return true;
}
void *pBitmap = tdGetBitmapAddrTp(pRow, flen);
tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx);
return true;
}
int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) {
if (isAllRowsNone(pCol)) {
pVal->valType = TD_VTYPE_NULL;
#ifdef TD_SUPPORT_READ2
pVal->val = (void *)getNullValue(pCol->type);
#else
pVal->val = NULL;
#endif
return TSDB_CODE_SUCCESS;
}
if (TD_COL_ROWS_NORM(pCol)) {
pVal->valType = TD_VTYPE_NORM;
} else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) {
return terrno;
}
if (tdValTypeIsNorm(pVal->valType)) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
} else {
pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
}
} else {
pVal->valType = TD_VTYPE_NULL;
#ifdef TD_SUPPORT_READ2
pVal->val = (void *)getNullValue(pCol->type);
#else
pVal->val = NULL;
#endif
}
return TSDB_CODE_SUCCESS;
}
bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
pVal->val = &pIter->pRow->ts;
pVal->valType = TD_VTYPE_NORM;
return true;
}
if (TD_IS_TP_ROW(pIter->pRow)) {
STColumn *pCol = NULL;
STSchema *pSchema = pIter->pSchema;
while (pIter->colIdx < pSchema->numOfCols) {
pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key
if (colId == pCol->colId) {
break;
} else if (pCol->colId < colId) {
++pIter->colIdx;
continue;
} else {
return false;
}
}
tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
++pIter->colIdx;
} else if (TD_IS_KV_ROW(pIter->pRow)) {
return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal);
} else {
pVal->valType = TD_VTYPE_NONE;
terrno = TSDB_CODE_INVALID_PARA;
if (COL_REACH_END(colId, pIter->maxColId)) return false;
}
return true;
}
bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) {
STSRow *pRow = pIter->pRow;
SKvRowIdx *pKvIdx = NULL;
bool colFound = false;
col_id_t kvNCols = tdRowGetNCols(pRow) - 1;
while (*nIdx < kvNCols) {
pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx));
if (pKvIdx->colId == colId) {
++(*nIdx);
pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset);
colFound = true;
break;
} else if (pKvIdx->colId > colId) {
pVal->valType = TD_VTYPE_NONE;
return true;
} else {
++(*nIdx);
}
}
if (!colFound) {
if (colId <= pIter->maxColId) {
pVal->valType = TD_VTYPE_NONE;
return true;
} else {
return false;
}
}
#ifdef TD_SUPPORT_BITMAP
int16_t colIdx = -1;
if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx);
if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) {
pVal->valType = TD_VTYPE_NONE;
}
#else
pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
#endif
return true;
}
bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) {
STSRow *pRow = pIter->pRow;
if (IS_VAR_DATA_TYPE(colType)) {
pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset));
} else {
pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset);
}
#ifdef TD_SUPPORT_BITMAP
if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) {
pVal->valType = TD_VTYPE_NONE;
}
#else
pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
#endif
return true;
}
static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) {
if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
return 1;
} else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
return -1;
} else {
return 0;
}
}
bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) {
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
pVal->val = &pIter->pRow->ts;
pVal->valType = TD_VTYPE_NORM;
return true;
}
STSRow *pRow = pIter->pRow;
int16_t colIdx = -1;
if (TD_IS_TP_ROW(pRow)) {
STSchema *pSchema = pIter->pSchema;
STColumn *pCol =
(STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ);
if (!pCol) {
pVal->valType = TD_VTYPE_NONE;
if (COL_REACH_END(colId, pIter->maxColId)) return false;
return true;
}
#ifdef TD_SUPPORT_BITMAP
colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn);
#endif
tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1);
} else if (TD_IS_KV_ROW(pRow)) {
SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx),
compareKvRowColId, TD_EQ);
#ifdef TD_SUPPORT_BITMAP
if (pIdx) {
colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx);
}
#endif
tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx);
} else {
if (COL_REACH_END(colId, pIter->maxColId)) return false;
pVal->valType = TD_VTYPE_NONE;
}
return true;
}
static int32_t tdCompareColId(const void *arg1, const void *arg2) {
int32_t colId = *(int32_t *)arg1;
STColumn *pCol = (STColumn *)arg2;
if (colId < pCol->colId) {
return -1;
} else if (colId == pCol->colId) {
return 0;
} else {
return 1;
}
}
int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
if (!pBitmap || colIdx < 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
int16_t nBytes = colIdx / TD_VTYPE_PARTS;
int16_t nOffset = colIdx & TD_VTYPE_OPTR;
char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
// use literal value directly and not use formula to simplify the codes
switch (nOffset) {
case 0:
*pValType = (((*pDestByte) & 0xC0) >> 6);
break;
case 1:
*pValType = (((*pDestByte) & 0x30) >> 4);
break;
case 2:
*pValType = (((*pDestByte) & 0x0C) >> 2);
break;
case 3:
*pValType = ((*pDestByte) & 0x03);
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
if (!pBitmap || colIdx < 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
int16_t nBytes = colIdx / TD_VTYPE_PARTS_I;
int16_t nOffset = colIdx & TD_VTYPE_OPTR_I;
char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
// use literal value directly and not use formula to simplify the codes
switch (nOffset) {
case 0:
*pValType = (((*pDestByte) & 0x80) >> 7);
break;
case 1:
*pValType = (((*pDestByte) & 0x40) >> 6);
break;
case 2:
*pValType = (((*pDestByte) & 0x20) >> 5);
break;
case 3:
*pValType = (((*pDestByte) & 0x10) >> 4);
break;
case 4:
*pValType = (((*pDestByte) & 0x08) >> 3);
break;
case 5:
*pValType = (((*pDestByte) & 0x04) >> 2);
break;
case 6:
*pValType = (((*pDestByte) & 0x02) >> 1);
break;
case 7:
*pValType = ((*pDestByte) & 0x01);
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
if (!pBitmap || colIdx < 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
int16_t nBytes = colIdx / TD_VTYPE_PARTS_I;
int16_t nOffset = colIdx & TD_VTYPE_OPTR_I;
char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
// use literal value directly and not use formula to simplify the codes
switch (nOffset) {
case 0:
*pDestByte = ((*pDestByte) & 0x7F) | (valType << 7);
// set the value and clear other partitions for offset 0
// *pDestByte |= (valType << 7);
break;
case 1:
*pDestByte = ((*pDestByte) & 0xBF) | (valType << 6);
// *pDestByte |= (valType << 6);
break;
case 2:
*pDestByte = ((*pDestByte) & 0xDF) | (valType << 5);
// *pDestByte |= (valType << 5);
break;
case 3:
*pDestByte = ((*pDestByte) & 0xEF) | (valType << 4);
// *pDestByte |= (valType << 4);
break;
case 4:
*pDestByte = ((*pDestByte) & 0xF7) | (valType << 3);
// *pDestByte |= (valType << 3);
break;
case 5:
*pDestByte = ((*pDestByte) & 0xFB) | (valType << 2);
// *pDestByte |= (valType << 2);
break;
case 6:
*pDestByte = ((*pDestByte) & 0xFD) | (valType << 1);
// *pDestByte |= (valType << 1);
break;
case 7:
*pDestByte = ((*pDestByte) & 0xFE) | valType;
// *pDestByte |= (valType);
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) {
#ifdef TD_SUPPORT_BITMAP
TASSERT(colIdx < tdRowGetNCols(pRow) - 1);
if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) {
output->valType = TD_VTYPE_NONE;
return terrno;
}
if (tdValTypeIsNorm(output->valType)) {
if (offset < 0) {
terrno = TSDB_CODE_INVALID_PARA;
output->valType = TD_VTYPE_NONE;
return terrno;
}
output->val = POINTER_SHIFT(pRow, offset);
}
#else
TASSERT(0);
if (offset < 0) {
terrno = TSDB_CODE_INVALID_PARA;
output->valType = TD_VTYPE_NONE;
return terrno;
}
output->val = POINTER_SHIFT(pRow, offset);
output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
#endif
return TSDB_CODE_SUCCESS;
}
int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset,
int16_t colIdx) {
#ifdef TD_SUPPORT_BITMAP
if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) {
output->valType = TD_VTYPE_NONE;
return terrno;
}
if (tdValTypeIsNorm(output->valType)) {
if (IS_VAR_DATA_TYPE(colType)) {
output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset));
} else {
output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset);
}
}
#else
if (IS_VAR_DATA_TYPE(colType)) {
output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset));
} else {
output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset);
}
output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
#endif
return TSDB_CODE_SUCCESS;
}
int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val,
bool isCopyVarData, int32_t offset, col_id_t colIdx) {
STSRow *pRow = pBuilder->pBuf;
if (!val) {
#ifdef TD_SUPPORT_BITMAP
if (tdValTypeIsNorm(valType)) {
terrno = TSDB_CODE_INVALID_PTR;
return terrno;
}
#else
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
#endif
}
// TS KEY is stored in STSRow.ts and not included in STSRow.data field.
if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
TD_ROW_KEY(pRow) = *(TSKEY *)val;
// The primary TS key is Norm all the time, thus its valType is not stored in bitmap.
return TSDB_CODE_SUCCESS;
}
// TODO: We can avoid the type judegement by FP, but would prevent the inline scheme.
if (TD_IS_TP_ROW(pRow)) {
tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset);
} else {
tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId);
}
return TSDB_CODE_SUCCESS;
}
int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) {
if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
offset -= sizeof(SKvRowIdx);
--colIdx;
#ifdef TD_SUPPORT_BITMAP
if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) {
return terrno;
}
#endif
STSRow *row = pBuilder->pBuf;
// No need to store None/Null values.
if (tdValIsNorm(valType, val, colType)) {
// ts key stored in STSRow.ts
SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
pColIdx->colId = colId;
pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN
if (IS_VAR_DATA_TYPE(colType)) {
if (isCopyVarData) {
memcpy(ptr, val, varDataTLen(val));
}
TD_ROW_LEN(row) += varDataTLen(val);
} else {
memcpy(ptr, val, TYPE_BYTES[colType]);
TD_ROW_LEN(row) += TYPE_BYTES[colType];
}
}
#ifdef TD_SUPPORT_BACK2
// NULL/None value
else {
SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
pColIdx->colId = colId;
pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN
const void *nullVal = getNullValue(colType);
if (IS_VAR_DATA_TYPE(colType)) {
if (isCopyVarData) {
memcpy(ptr, nullVal, varDataTLen(nullVal));
}
TD_ROW_LEN(row) += varDataTLen(nullVal);
} else {
memcpy(ptr, nullVal, TYPE_BYTES[colType]);
TD_ROW_LEN(row) += TYPE_BYTES[colType];
}
}
#endif
return 0;
}
int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData,
int8_t colType, int16_t colIdx, int32_t offset) {
if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
offset -= sizeof(TSKEY);
--colIdx;
#ifdef TD_SUPPORT_BITMAP
if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) {
return terrno;
}
#endif
STSRow *row = pBuilder->pBuf;
// 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap
// should be updated simultaneously if Norm val overwrite Null/None cols.
// 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null.
if (tdValIsNorm(valType, val, colType)) {
// TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar.
if (IS_VAR_DATA_TYPE(colType)) {
// ts key stored in STSRow.ts
*(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row);
if (isCopyVarData) {
memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val));
}
TD_ROW_LEN(row) += varDataTLen(val);
} else {
memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]);
}
}
#ifdef TD_SUPPORT_BACK2
// NULL/None value
else {
// TODO: Null value for new data types imported since 3.0 need to be defined.
const void *nullVal = getNullValue(colType);
if (IS_VAR_DATA_TYPE(colType)) {
// ts key stored in STSRow.ts
*(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row);
if (isCopyVarData) {
memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal));
}
TD_ROW_LEN(row) += varDataTLen(nullVal);
} else {
memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]);
}
}
#endif
return 0;
}
int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen,
int32_t allNullLen, int32_t boundNullLen) {
if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) {
uint32_t tpLen = allNullLen;
uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen;
if (isSelectKVRow(kvLen, tpLen)) {
pBuilder->rowType = TD_ROW_KV;
} else {
pBuilder->rowType = TD_ROW_TP;
}
} else {
pBuilder->rowType = TD_ROW_TP;
}
pBuilder->flen = flen;
pBuilder->nCols = nCols;
pBuilder->nBoundCols = nBoundCols;
if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#ifdef TD_SUPPORT_BITMAP
// the primary TS key is stored separatedly
pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
if (nBoundCols > 0) {
pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
} else {
pBuilder->nBoundBitmaps = 0;
}
#else
pBuilder->nBitmaps = 0;
pBuilder->nBoundBitmaps = 0;
#endif
return TSDB_CODE_SUCCESS;
}
int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
pBuilder->pBuf = (STSRow *)pBuf;
if (!pBuilder->pBuf) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
TD_ROW_SET_INFO(pBuilder->pBuf, 0);
TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType);
TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0);
uint32_t len = 0;
switch (pBuilder->rowType) {
case TD_ROW_TP:
#ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen);
memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps);
#endif
// the primary TS key is stored separatedly
len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps;
TD_ROW_SET_LEN(pBuilder->pBuf, len);
TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
break;
case TD_ROW_KV:
#ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps);
#endif
len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) +
pBuilder->nBoundBitmaps; // add
TD_ROW_SET_LEN(pBuilder->pBuf, len);
TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver);
TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols);
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) {
pBuilder->pBuf = (STSRow *)pBuf;
if (!pBuilder->pBuf) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0);
uint32_t len = 0;
switch (pBuilder->rowType) {
case TD_ROW_TP:
#ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen);
#endif
break;
case TD_ROW_KV:
#ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
#endif
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols,
int32_t nBoundCols, int32_t flen) {
if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) {
return terrno;
}
return tdSRowResetBuf(pBuilder, pBuf);
}
void tdSRowReset(SRowBuilder *pBuilder) {
pBuilder->rowType = TD_ROW_TP;
pBuilder->pBuf = NULL;
pBuilder->nBoundCols = -1;
pBuilder->nCols = -1;
pBuilder->flen = -1;
pBuilder->pBitmap = NULL;
}
int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) {
pBuilder->flen = flen;
pBuilder->nCols = nCols;
if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#ifdef TD_SUPPORT_BITMAP
// the primary TS key is stored separatedly
pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
#else
pBuilder->nBitmaps = 0;
pBuilder->nBoundBitmaps = 0;
#endif
return TSDB_CODE_SUCCESS;
}
int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) {
pBuilder->flen = flen;
pBuilder->nCols = nCols;
pBuilder->nBoundCols = nBoundCols;
if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
#ifdef TD_SUPPORT_BITMAP
// the primary TS key is stored separatedly
pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
if (nBoundCols > 0) {
pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
} else {
pBuilder->nBoundBitmaps = 0;
}
#else
pBuilder->nBitmaps = 0;
pBuilder->nBoundBitmaps = 0;
#endif
return TSDB_CODE_SUCCESS;
}
int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
switch (bitmapMode) {
case 0:
tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
break;
case -1:
case 1:
tdGetBitmapValTypeI(pBitmap, colIdx, pValType);
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) {
TDRowValT valType = 0;
tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode);
if (tdValTypeIsNorm(valType)) {
return true;
}
return false;
}
int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) {
if (!pBitmap || colIdx < 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
int16_t nBytes = colIdx / TD_VTYPE_PARTS;
int16_t nOffset = colIdx & TD_VTYPE_OPTR;
char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
// use literal value directly and not use formula to simplify the codes
switch (nOffset) {
case 0:
*pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
// set the value and clear other partitions for offset 0
// *pDestByte |= (valType << 6);
break;
case 1:
*pDestByte = ((*pDestByte) & 0xCF) | (valType << 4);
// *pDestByte |= (valType << 4);
break;
case 2:
*pDestByte = ((*pDestByte) & 0xF3) | (valType << 2);
// *pDestByte |= (valType << 2);
break;
case 3:
*pDestByte = ((*pDestByte) & 0xFC) | valType;
// *pDestByte |= (valType);
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) {
switch (bitmapMode) {
case 0:
tdSetBitmapValTypeII(pBitmap, colIdx, valType);
break;
case -1:
case 1:
tdSetBitmapValTypeI(pBitmap, colIdx, valType);
break;
default:
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) {
#ifdef TD_SUPPORT_BITMAP
switch (rowType) {
case TD_ROW_TP:
return tdGetBitmapAddrTp(pRow, flen);
case TD_ROW_KV:
return tdGetBitmapAddrKv(pRow, nKvCols);
default:
break;
}
#endif
return NULL;
}
void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) {
pIter->pRow = pRow;
pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow));
pIter->offset = 0;
pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID;
pIter->kvIdx = 0;
}
void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) {
pIter->pSchema = pSchema;
pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId;
}

4
source/dnode/mnode/impl/test/user/user.cpp
View File

@ -367,8 +367,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
EXPECT_EQ(numOfReadDbs, 1);
EXPECT_EQ(numOfWriteDbs, 0);
char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 5);
EXPECT_STREQ(dbname, "1.d2");
char* dbname = (char*)taosHashGet(authRsp.readDbs, "1.d2", 4);
EXPECT_TRUE(dbname != NULL);
taosHashCleanup(authRsp.readDbs);
taosHashCleanup(authRsp.writeDbs);

56
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -211,7 +211,7 @@ int tsdbCommit(STsdb *pRepo) {
void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
TSKEY minKey, midKey, maxKey, now;
TSKEY minKey, midKey, maxKey, now;
now = taosGetTimestamp(pCfg->precision);
minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision];
@ -1386,34 +1386,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
tSkipListIterNext(pCommitIter->pIter);
} else {
#if 0
if (update != TD_ROW_OVERWRITE_UPDATE) {
// copy disk data
for (int i = 0; i < pDataCols->numOfCols; ++i) {
// TODO: dataColAppendVal may fail
SCellVal sVal = {0};
if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
TASSERT(0);
}
tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode);
}
if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++;
}
if (update != TD_ROW_DISCARD_UPDATE) {
// copy mem data
if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row));
ASSERT(pSchema != NULL);
}
tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE);
}
++(*iter);
tSkipListIterNext(pCommitIter->pIter);
#endif
if(lastKey != key1) {
if (lastKey != key1) {
lastKey = key1;
++pTarget->numOfRows;
}
@ -1485,28 +1458,3 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
return false;
}
// int tsdbApplyRtn(STsdbRepo *pRepo) {
// SRtn rtn;
// SFSIter fsiter;
// STsdbFS * pfs = REPO_FS(pRepo);
// SDFileSet *pSet;
// // Get retention snapshot
// tsdbGetRtnSnap(pRepo, &rtn);
// tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
// while ((pSet = tsdbFSIterNext(&fsiter))) {
// if (pSet->fid < rtn.minFid) {
// tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
// TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
// continue;
// }
// if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) {
// return -1;
// }
// }
// return 0;
// }

5
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -326,6 +326,11 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo
tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext);
#ifdef TD_DEBUG_PRINT_ROW
printf("!!! %s:%d table %" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, pTbData->uid,
SL_SIZE(pTbData->pData));
#endif
// Set statistics
keyMax = TD_ROW_KEY(blkIter.row);

224
source/dnode/vnode/src/tsdb/tsdbMemTable2.c
View File

@ -21,16 +21,19 @@ typedef struct SMemSkipList SMemSkipList;
typedef struct SMemSkipListNode SMemSkipListNode;
typedef struct SMemSkipListCurosr SMemSkipListCurosr;
#define SL_MAX_LEVEL 5
struct SMemTable {
STsdb *pTsdb;
TSKEY minKey;
TSKEY maxKey;
int64_t minVer;
int64_t maxVer;
int64_t nRows;
int32_t nHash;
int32_t nBucket;
SMemData **pBuckets;
STsdb *pTsdb;
TSKEY minKey;
TSKEY maxKey;
int64_t minVer;
int64_t maxVer;
int64_t nRows;
int32_t nHash;
int32_t nBucket;
SMemData **pBuckets;
SMemSkipListCurosr *pSlc;
};
struct SMemSkipListNode {
@ -60,9 +63,15 @@ struct SMemData {
struct SMemSkipListCurosr {
SMemSkipList *pSl;
SMemSkipListNode *pNodeC;
SMemSkipListNode *pNodes[SL_MAX_LEVEL];
};
typedef struct {
int64_t version;
uint32_t szRow;
const STSRow *pRow;
} STsdbRow;
#define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET))
#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
@ -76,7 +85,19 @@ struct SMemSkipListCurosr {
#define SL_HEAD_NODE_FORWARD(n, l) SL_NODE_FORWARD(n, l)
#define SL_TAIL_NODE_BACKWARD(n, l) SL_NODE_FORWARD(n, l)
static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl);
static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl);
static int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow);
static int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow);
static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc);
static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc);
static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl);
static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode);
static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags);
static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc);
static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc);
static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc);
static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc);
static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow);
// SMemTable
int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
@ -102,6 +123,11 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
taosMemoryFree(pMemTb);
return -1;
}
if (tsdbMemSkipListCursorCreate(pTsdb->pVnode->config.tsdbCfg.slLevel, &pMemTb->pSlc) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pMemTb->pBuckets);
taosMemoryFree(pMemTb);
}
*ppMemTb = pMemTb;
return 0;
@ -110,6 +136,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) {
int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) {
if (pMemTb) {
// loop to destroy the contents (todo)
tsdbMemSkipListCursorDestroy(pMemTb->pSlc);
taosMemoryFree(pMemTb->pBuckets);
taosMemoryFree(pMemTb);
}
@ -177,52 +204,42 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p
}
// do insert data to SMemData
SMemSkipListCurosr slc = {0};
const STSRow *pRow;
uint32_t szRow;
SDecoder decoder = {0};
SMemSkipListNode *forwards[SL_MAX_LEVEL];
SMemSkipListNode *pNode;
int32_t iRow;
STsdbRow tRow = {.version = version};
SEncoder ec = {0};
SDecoder dc = {0};
tDecoderInit(&decoder, pSubmitBlk->pData, pSubmitBlk->nData);
for (;;) {
if (tDecodeIsEnd(&decoder)) break;
tDecoderInit(&dc, pSubmitBlk->pData, pSubmitBlk->nData);
tsdbMemSkipListCursorInit(pMemTb->pSlc, &pMemData->sl);
for (iRow = 0;; iRow++) {
if (tDecodeIsEnd(&dc)) break;
if (tDecodeBinary(&decoder, (const uint8_t **)&pRow, &szRow) < 0) {
// decode row
if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
// check the row (todo)
// // move the cursor to position to write (todo)
// int32_t c;
// tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c);
// ASSERT(c);
// move cursor
tsdbMemSkipListCursorMoveTo(pMemTb->pSlc, version, tRow.pRow->ts, 0);
// encode row
int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl);
int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (0 /*todo*/);
SMemSkipListNode *pNode = vnodeBufPoolMalloc(pPool, tsize);
pNode = tsdbMemSkipListNodeCreate(pPool, &pMemData->sl, &tRow);
if (pNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pNode->level = level;
// uint8_t *pData = SL_NODE_DATA(pSlNode);
// *(int64_t *)pData = version;
// pData += sizeof(version);
// memcpy(pData, pt, p - pt);
// // insert row
// tsdbMemSkipListCursorPut(&slc, pSlNode);
// put the node
tsdbMemSkipListCursorPut(pMemTb->pSlc, pNode);
// update status
if (pRow->ts < pMemData->minKey) pMemData->minKey = pRow->ts;
if (pRow->ts > pMemData->maxKey) pMemData->maxKey = pRow->ts;
if (tRow.pRow->ts < pMemData->minKey) pMemData->minKey = tRow.pRow->ts;
if (tRow.pRow->ts > pMemData->maxKey) pMemData->maxKey = tRow.pRow->ts;
}
tDecoderClear(&decoder);
// tsdbMemSkipListCursorClose(&slc);
tDecoderClear(&dc);
// update status
if (pMemData->minVer == -1) pMemData->minVer = version;
@ -236,17 +253,128 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p
return 0;
}
static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
int8_t level = 1;
int8_t tlevel;
int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
const uint32_t factor = 4;
if (pSl->size) {
tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
level++;
}
while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
level++;
}
return level;
}
static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) {
if (tEncodeI64(pEncoder, pRow->version) < 0) return -1;
if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1;
return 0;
}
static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) {
if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1;
if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1;
return 0;
}
static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc) {
*ppSlc = (SMemSkipListCurosr *)taosMemoryCalloc(1, sizeof(**ppSlc) + sizeof(SMemSkipListNode *) * maxLevel);
if (*ppSlc == NULL) {
return -1;
}
return 0;
}
static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc) { taosMemoryFree(pSlc); }
static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl) {
SMemSkipListNode *pHead = SL_HEAD_NODE(pSl);
pSlc->pSl = pSl;
// for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
// pSlc->forwards[iLevel] = pHead;
// }
}
static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode) {
SMemSkipList *pSl = pSlc->pSl;
SMemSkipListNode *pNodeNext;
for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) {
// todo
ASSERT(0);
}
if (pSl->level < pNode->level) {
pSl->level = pNode->level;
}
pSl->size += 1;
}
static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags) {
SMemSkipListNode **pForwards = NULL;
SMemSkipList *pSl = pSlc->pSl;
int8_t maxLevel = pSl->maxLevel;
SMemSkipListNode *pHead = SL_HEAD_NODE(pSl);
SMemSkipListNode *pTail = SL_TAIL_NODE(pSl);
if (pSl->size == 0) {
for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
pForwards[iLevel] = pHead;
}
}
return 0;
}
static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc) {
SMemSkipList *pSl = pSlc->pSl;
SMemSkipListNode *pHead = SL_HEAD_NODE(pSl);
for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
pSlc->pNodes[iLevel] = pHead;
}
tsdbMemSkipListCursorMoveToNext(pSlc);
}
static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc) {
SMemSkipList *pSl = pSlc->pSl;
SMemSkipListNode *pTail = SL_TAIL_NODE(pSl);
for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) {
pSlc->pNodes[iLevel] = pTail;
}
tsdbMemSkipListCursorMoveToPrev(pSlc);
}
static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) {
// TODO
return 0;
}
static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc) {
// TODO
return 0;
}
static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow) {
int32_t tsize;
int32_t ret;
int8_t level = tsdbMemSkipListRandLevel(pSl);
SMemSkipListNode *pNode = NULL;
SEncoder ec = {0};
tEncodeSize(tsdbEncodeRow, pTRow, tsize, ret);
pNode = vnodeBufPoolMalloc(pPool, tsize + SL_NODE_SIZE(level));
if (pNode) {
pNode->level = level;
tEncoderInit(&ec, (uint8_t *)SL_NODE_DATA(pNode), tsize);
tsdbEncodeRow(&ec, pTRow);
tEncoderClear(&ec);
}
return pNode;
}

4
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -3870,7 +3870,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
if (metaGetTableEntryByUid(&mr, uid) < 0) {
tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
goto _error;
} else {
tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
@ -3940,7 +3940,7 @@ int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGr
metaReaderInit(&mr, (SMeta*)pMeta, 0);
if (metaGetTableEntryByUid(&mr, uid) < 0) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
goto _error;
}

48
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -497,6 +497,47 @@ _exit:
return 0;
}
static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char* tags) {
ASSERT(pMsg != NULL);
SSubmitMsgIter msgIter = {0};
SMeta *pMeta = pVnode->pMeta;
SSubmitBlk *pBlock = NULL;
SSubmitBlkIter blkIter = {0};
STSRow *row = NULL;
STSchema *pSchema = NULL;
tb_uid_t suid = 0;
if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
while (true) {
if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
if (pBlock == NULL) break;
tInitSubmitBlkIter(&msgIter, pBlock, &blkIter);
if (blkIter.row == NULL) continue;
if (!pSchema || (suid != msgIter.suid)) {
if (pSchema) {
taosMemoryFreeClear(pSchema);
}
pSchema = metaGetTbTSchema(pMeta, msgIter.suid, 0); // TODO: use the real schema
if(pSchema) {
suid = msgIter.suid;
}
}
if(!pSchema) {
printf("%s:%d no valid schema\n", tags, __LINE__);
continue;
}
char __tags[128] = {0};
snprintf(__tags, 128, "%s: uid %" PRIi64 " ", tags, msgIter.uid);
while ((row = tGetSubmitBlkNext(&blkIter))) {
tdSRowPrint(row, pSchema, __tags);
}
}
taosMemoryFreeClear(pSchema);
return 0;
}
static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
SSubmitMsgIter msgIter = {0};
@ -508,6 +549,10 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
pRsp->code = 0;
#ifdef TD_DEBUG_PRINT_ROW
vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__);
#endif
// handle the request
if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) {
pRsp->code = TSDB_CODE_INVALID_MSG;
@ -550,7 +595,8 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
goto _exit;
}
rsp.numOfRows += nRows;
rsp.affectedRows += nRows;
}
_exit:

2
source/libs/catalog/inc/catalogInt.h
View File

@ -36,7 +36,7 @@ extern "C" {
#define CTG_DEFAULT_INVALID_VERSION (-1)
#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_TDB_INVALID_TABLE_ID
#define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_PAR_TABLE_NOT_EXIST
enum {
CTG_READ = 1,

5
source/libs/executor/src/executorimpl.c
View File

@ -4199,7 +4199,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
}
}
taosMemoryFree(pOperator->pExpr);
taosMemoryFreeClear(pOperator->pExpr);
taosMemoryFreeClear(pOperator->info);
taosMemoryFreeClear(pOperator);
}
@ -4384,6 +4384,9 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) {
}
static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
if (NULL == param) {
return;
}
SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*)param;
doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
cleanupAggSup(&pInfo->aggSup);

6
source/libs/function/inc/builtinsimpl.h
View File

@ -55,6 +55,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx);
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t stddevInvertFunction(SqlFunctionCtx* pCtx);
bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool leastSQRFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t leastSQRFunction(SqlFunctionCtx* pCtx);
int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx);
bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t percentileFunction(SqlFunctionCtx *pCtx);

28
source/libs/function/src/builtins.c
View File

@ -226,6 +226,23 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
return TSDB_CODE_SUCCESS;
}
static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (3 != numOfParams) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
for (int32_t i = 0; i < numOfParams; ++i) {
uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
if (!IS_NUMERIC_TYPE(colType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
}
pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY };
return TSDB_CODE_SUCCESS;
}
static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (4 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@ -537,6 +554,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.finalizeFunc = stddevFinalize,
.invertFunc = stddevInvertFunction
},
{
.name = "leastsquares",
.type = FUNCTION_TYPE_LEASTSQUARES,
.classification = FUNC_MGT_AGG_FUNC,
.translateFunc = translateLeastSQR,
.getEnvFunc = getLeastSQRFuncEnv,
.initFunc = leastSQRFunctionSetup,
.processFunc = leastSQRFunction,
.finalizeFunc = leastSQRFinalize,
.invertFunc = leastSQRInvertFunction
},
{
.name = "avg",
.type = FUNCTION_TYPE_AVG,

213
source/libs/function/src/builtinsimpl.c
View File

@ -65,6 +65,13 @@ typedef struct SStddevRes {
};
} SStddevRes;
typedef struct SLeastSQRInfo {
double matrix[2][3];
double startVal;
double stepVal;
int64_t num;
} SLeastSQRInfo;
typedef struct SPercentileInfo {
double result;
tMemBucket* pMemBucket;
@ -105,6 +112,7 @@ typedef struct SHistoFuncBin {
typedef struct SHistoFuncInfo {
int32_t numOfBins;
int32_t totalCount;
bool normalized;
SHistoFuncBin bins[];
} SHistoFuncInfo;
@ -1404,6 +1412,181 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
return functionFinalize(pCtx, pBlock);
}
bool getLeastSQRFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SLeastSQRInfo);
return true;
}
bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) {
return false;
}
SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
pInfo->startVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[1].param.d :
(double)pCtx->param[1].param.i;
pInfo->stepVal = IS_FLOAT_TYPE(pCtx->param[1].param.nType) ? pCtx->param[2].param.d :
(double)pCtx->param[1].param.i;
return true;
}
#define LEASTSQR_CAL(p, x, y, index, step) \
do { \
(p)[0][0] += (double)(x) * (x); \
(p)[0][1] += (double)(x); \
(p)[0][2] += (double)(x) * (y)[index]; \
(p)[1][2] += (y)[index]; \
(x) += step; \
} while (0)
int32_t leastSQRFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElem = 0;
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SColumnInfoData* pCol = pInput->pData[0];
double(*param)[3] = pInfo->matrix;
double x = pInfo->startVal;
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
switch (type) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem++;
LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal);
break;
}
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t* plist = (int16_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem++;
LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal);
}
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t* plist = (int32_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem++;
LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal);
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t* plist = (int64_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem++;
LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal);
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float* plist = (float*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem++;
LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal);
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double* plist = (double*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem++;
LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal);
}
break;
}
default:
break;
}
pInfo->startVal = x;
pInfo->num += numOfElem;
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS;
}
int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SLeastSQRInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
int32_t currentRow = pBlock->info.rows;
if (0 == pInfo->num) {
return 0;
}
double(*param)[3] = pInfo->matrix;
param[1][1] = (double)pInfo->num;
param[1][0] = param[0][1];
param[0][0] -= param[1][0] * (param[0][1] / param[1][1]);
param[0][2] -= param[1][2] * (param[0][1] / param[1][1]);
param[0][1] = 0;
param[1][2] -= param[0][2] * (param[1][0] / param[0][0]);
param[1][0] = 0;
param[0][2] /= param[0][0];
param[1][2] /= param[1][1];
char buf[64] = {0};
size_t len = snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%.6lf, intercept:%.6lf}", param[0][2], param[1][2]);
varDataSetLen(buf, len);
colDataAppend(pCol, currentRow, buf, false);
return pResInfo->numOfRes;
}
int32_t leastSQRInvertFunction(SqlFunctionCtx* pCtx) {
//TODO
return TSDB_CODE_SUCCESS;
}
bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SPercentileInfo);
return true;
@ -2398,6 +2581,9 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultIn
}
SHistoFuncInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
pInfo->numOfBins = 0;
pInfo->totalCount = 0;
pInfo->normalized = 0;
int8_t binType = getHistogramBinType(varDataVal(pCtx->param[1].param.pz));
if (binType == UNKNOWN_BIN) {
@ -2427,7 +2613,6 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) {
int32_t numOfRows = pInput->numOfRows;
int32_t numOfElems = 0;
int32_t totalElems = 0;
for (int32_t i = start; i < numOfRows + start; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
@ -2442,23 +2627,13 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) {
for (int32_t k = 0; k < pInfo->numOfBins; ++k) {
if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) {
pInfo->bins[k].count++;
totalElems++;
pInfo->totalCount++;
break;
}
}
}
if (pInfo->normalized) {
for (int32_t k = 0; k < pInfo->numOfBins; ++k) {
if(totalElems != 0) {
pInfo->bins[k].percentage = pInfo->bins[k].count / (double)totalElems;
} else {
pInfo->bins[k].percentage = 0;
}
}
}
SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins);
return TSDB_CODE_SUCCESS;
}
@ -2471,14 +2646,24 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t currentRow = pBlock->info.rows;
if (pInfo->normalized) {
for (int32_t k = 0; k < pResInfo->numOfRes; ++k) {
if(pInfo->totalCount != 0) {
pInfo->bins[k].percentage = pInfo->bins[k].count / (double)pInfo->totalCount;
} else {
pInfo->bins[k].percentage = 0;
}
}
}
for (int32_t i = 0; i < pResInfo->numOfRes; ++i) {
int32_t len;
char buf[512] = {0};
if (!pInfo->normalized) {
len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}",
len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}",
pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count);
} else {
len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}",
len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}",
pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].percentage);
}
varDataSetLen(buf, len);

12
source/libs/index/inc/indexComm.h
View File

@ -20,11 +20,23 @@
extern "C" {
#endif
#include "indexInt.h"
#include "tcompare.h"
extern char JSON_COLUMN[];
extern char JSON_VALUE_DELIM;
char* indexPackJsonData(SIndexTerm* itm);
char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b);
_cache_range_compare indexGetCompare(RangeType ty);
#ifdef __cplusplus
}
#endif

48
source/libs/index/src/indexCache.c
View File

@ -60,50 +60,6 @@ static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult
static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
RangeType type);
typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) {
// optime later
int32_t ret = func(a, b);
switch (comType) {
case QUERY_LESS_THAN: {
if (ret < 0) return MATCH;
} break;
case QUERY_LESS_EQUAL: {
if (ret <= 0) return MATCH;
break;
}
case QUERY_GREATER_THAN: {
if (ret > 0) return MATCH;
break;
}
case QUERY_GREATER_EQUAL: {
if (ret >= 0) return MATCH;
}
}
return CONTINUE;
}
static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_LESS_THAN, a, b);
}
static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
}
static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
}
static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
}
static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual,
tCompareGreaterThan, tCompareGreaterEqual};
static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
{cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual,
cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange},
@ -169,7 +125,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes
return 0;
}
_cache_range_compare cmpFn = rangeCompare[type];
_cache_range_compare cmpFn = indexGetCompare(type);
MemTable* mem = cache;
IndexCache* pCache = mem->pCache;
@ -295,7 +251,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
if (cache == NULL) {
return 0;
}
_cache_range_compare cmpFn = rangeCompare[type];
_cache_range_compare cmpFn = indexGetCompare(type);
MemTable* mem = cache;
IndexCache* pCache = mem->pCache;

47
source/libs/index/src/indexComm.c
View File

@ -13,12 +13,58 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "indexComm.h"
#include "index.h"
#include "indexInt.h"
#include "tcompare.h"
char JSON_COLUMN[] = "JSON";
char JSON_VALUE_DELIM = '&';
static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_LESS_THAN, a, b);
}
static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
}
static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
}
static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0);
return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
}
TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) {
// optime later
int32_t ret = func(a, b);
switch (comType) {
case QUERY_LESS_THAN: {
if (ret < 0) return MATCH;
} break;
case QUERY_LESS_EQUAL: {
if (ret <= 0) return MATCH;
break;
}
case QUERY_GREATER_THAN: {
if (ret > 0) return MATCH;
break;
}
case QUERY_GREATER_EQUAL: {
if (ret >= 0) return MATCH;
}
}
return CONTINUE;
}
static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual,
tCompareGreaterThan, tCompareGreaterEqual};
_cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; }
char* indexPackJsonData(SIndexTerm* itm) {
/*
* |<-----colname---->|<-----dataType---->|<--------colVal---------->|
@ -46,6 +92,7 @@ char* indexPackJsonData(SIndexTerm* itm) {
return buf;
}
char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
/*
* |<-----colname---->|<-----dataType---->|<--------colVal---------->|

149
source/libs/index/src/indexTfile.c
View File

@ -72,9 +72,23 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype);
static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = {
tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan,
tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange};
static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype);
static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = {
{tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual,
tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange},
{tfSearchTerm_JSON, tfSearchPrefix_JSON, tfSearchSuffix_JSON, tfSearchRegex_JSON, tfSearchLessThan_JSON,
tfSearchLessEqual_JSON, tfSearchGreaterThan_JSON, tfSearchGreaterEqual_JSON, tfSearchRange_JSON}};
TFileCache* tfileCacheCreate(const char* path) {
TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache));
@ -202,14 +216,10 @@ void tfileReaderDestroy(TFileReader* reader) {
taosMemoryFree(reader);
}
static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
int ret = 0;
char* p = tem->colVal;
uint64_t sz = tem->nColVal;
if (hasJson) {
p = indexPackJsonData(tem);
sz = strlen(p);
}
int64_t st = taosGetTimestampUs();
FstSlice key = fstSliceCreate(p, sz);
uint64_t offset;
@ -224,9 +234,6 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
tem->colName, tem->colVal, cost);
}
if (hasJson) {
taosMemoryFree(p);
}
fstSliceDestroy(&key);
return 0;
}
@ -308,14 +315,11 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
}
static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) {
bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
int ret = 0;
char* p = tem->colVal;
int skip = 0;
int ret = 0;
char* p = tem->colVal;
int skip = 0;
_cache_range_compare cmpFn = indexGetCompare(type);
if (hasJson) {
p = indexPackJsonDataPrefix(tem, &skip);
}
SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
@ -328,7 +332,16 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult
StreamWithState* st = streamBuilderIntoStream(sb);
StreamWithStateResult* rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
taosArrayPush(offsets, &(rt->out.out));
FstSlice* s = &rt->data;
char* ch = (char*)fstSliceData(s, NULL);
TExeCond cond = cmpFn(ch, p, tem->colType);
if (MATCH == cond) {
tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
} else if (CONTINUE == cond) {
} else if (BREAK == cond) {
swsResultDestroy(rt);
break;
}
swsResultDestroy(rt);
}
streamWithStateDestroy(st);
@ -376,17 +389,105 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
fstSliceDestroy(&key);
return 0;
}
static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
int ret = 0;
char* p = indexPackJsonData(tem);
int sz = strlen(p);
int64_t st = taosGetTimestampUs();
FstSlice key = fstSliceCreate(p, sz);
uint64_t offset;
if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) {
int64_t et = taosGetTimestampUs();
int64_t cost = et - st;
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us",
tem->suid, tem->colName, tem->colVal, cost);
ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
cost = taosGetTimestampUs() - et;
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
tem->colName, tem->colVal, cost);
}
fstSliceDestroy(&key);
return 0;
// deprecate api
return TSDB_CODE_SUCCESS;
}
static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
// impl later
return TSDB_CODE_SUCCESS;
}
static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
// impl later
return TSDB_CODE_SUCCESS;
}
static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
// impl later
return TSDB_CODE_SUCCESS;
}
static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
return tfSearchCompareFunc_JSON(reader, tem, tr, LT);
}
static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
return tfSearchCompareFunc_JSON(reader, tem, tr, LE);
}
static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
return tfSearchCompareFunc_JSON(reader, tem, tr, GT);
}
static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
return tfSearchCompareFunc_JSON(reader, tem, tr, GE);
}
static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
// impl later
return TSDB_CODE_SUCCESS;
}
static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype) {
int ret = 0;
int skip = 0;
char* p = indexPackJsonDataPrefix(tem, &skip);
_cache_range_compare cmpFn = indexGetCompare(ctype);
SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
FstSlice h = fstSliceCreate((uint8_t*)p, skip);
fstStreamBuilderSetRange(sb, &h, ctype);
fstSliceDestroy(&h);
StreamWithState* st = streamBuilderIntoStream(sb);
StreamWithStateResult* rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
FstSlice* s = &rt->data;
char* ch = (char*)fstSliceData(s, NULL);
TExeCond cond = cmpFn(ch, p, tem->colType);
if (MATCH == cond) {
tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
} else if (CONTINUE == cond) {
} else if (BREAK == cond) {
swsResultDestroy(rt);
break;
}
swsResultDestroy(rt);
}
streamWithStateDestroy(st);
fstStreamBuilderDestroy(sb);
return TSDB_CODE_SUCCESS;
}
int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) {
SIndexTerm* term = query->term;
EIndexQueryType qtype = query->qType;
if (qtype >= sizeof(tfSearch) / sizeof(tfSearch[0])) {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName,
term->colVal);
return -1;
if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
return tfSearch[1][qtype](reader, term, tr);
} else {
return tfSearch[qtype](reader, term, tr);
return tfSearch[0][qtype](reader, term, tr);
}
tfileReaderUnRef(reader);
return 0;
}

12
source/libs/parser/src/parInsert.c
View File

@ -241,6 +241,15 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool
SParseContext* pBasicCtx = pCxt->pComCxt;
SName name = {0};
createSName(&name, pTname, pBasicCtx->acctId, pBasicCtx->db, &pCxt->msg);
char dbFname[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFname);
bool pass = false;
CHECK_CODE(catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, dbFname, AUTH_TYPE_WRITE, &pass));
if (!pass) {
return TSDB_CODE_PAR_PERMISSION_DENIED;
}
if (isStb) {
CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name,
&pCxt->pTableMeta));
@ -1151,6 +1160,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
(*pCxt->pStmtCb->setExecInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj);
pCxt->pVgroupsHashObj = NULL;
pCxt->pTableBlockHashObj = NULL;
pCxt->pTableMeta = NULL;
return TSDB_CODE_SUCCESS;
}
@ -1276,7 +1286,7 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tN
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
SSchema* pSchema = pDataBlock->pTableMeta->schema;
SKvParam param = {.builder = &tagBuilder};
for (int c = 0; c < tags->numOfBound; ++c) {

3
source/libs/parser/src/parInsertData.c
View File

@ -469,7 +469,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
// the maximum expanded size in byte when a row-wise data is converted to SDataRow format
int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize +
sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta);
sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + pOneTableBlock->createTbReqLen;
if (dataBuf->nAllocSize < destSize) {
dataBuf->nAllocSize = (uint32_t)(destSize * 1.5);
@ -601,6 +601,7 @@ int32_t qResetStmtDataBlock(void* block, bool keepBuf) {
pBlock->numOfTables = 1;
pBlock->nAllocSize = TSDB_PAYLOAD_SIZE;
pBlock->headerSize = pBlock->size;
pBlock->createTbReqLen = 0;
memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder));

4
source/libs/parser/src/parTranslater.c
View File

@ -2579,7 +2579,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt
SName tableName;
int32_t code = getTableMetaImpl(
pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta);
if ((TSDB_CODE_TDB_INVALID_TABLE_ID == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) {
if ((TSDB_CODE_PAR_TABLE_NOT_EXIST == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) {
return TSDB_CODE_SUCCESS;
}
if (TSDB_CODE_SUCCESS == code) {
@ -3150,7 +3150,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB;
}
strcpy(req.user, pStmt->userName);
strcpy(req.dbname, pStmt->dbName);
sprintf(req.dbname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->dbName);
return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
}

16
source/libs/scalar/src/filter.c
View File

@ -3577,6 +3577,22 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
}
if (QUERY_NODE_NODE_LIST == nodeType(*pNode)) {
SNodeListNode *listNode = (SNodeListNode *)*pNode;
if (QUERY_NODE_VALUE != nodeType(listNode->pNodeList->pHead->pNode)) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
SValueNode *valueNode = (SValueNode *)listNode->pNodeList->pHead->pNode;
uint8_t type = valueNode->node.resType.type;
SNode *node = NULL;
FOREACH(node, listNode->pNodeList) {
if (type != ((SValueNode *)node)->node.resType.type) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
}
return DEAL_RES_CONTINUE;
}

117
source/libs/scalar/src/sclvector.c
View File

@ -265,13 +265,53 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r
}
static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) {
int64_t value = strtoll(buf, NULL, 10);
colDataAppendInt64(pOut->columnData, rowIndex, &value);
switch (pOut->columnData->info.type) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t value = (int8_t)strtoll(buf, NULL, 10);
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value);
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t value = (int16_t)strtoll(buf, NULL, 10);
colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value);
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t value = (int32_t)strtoll(buf, NULL, 10);
colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value);
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t value = (int64_t)strtoll(buf, NULL, 10);
colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value);
break;
}
}
}
static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) {
uint64_t value = strtoull(buf, NULL, 10);
colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*) &value);
switch (pOut->columnData->info.type) {
case TSDB_DATA_TYPE_UTINYINT: {
uint8_t value = (uint8_t)strtoull(buf, NULL, 10);
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value);
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
uint16_t value = (uint16_t)strtoull(buf, NULL, 10);
colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value);
break;
}
case TSDB_DATA_TYPE_UINT: {
uint32_t value = (uint32_t)strtoull(buf, NULL, 10);
colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value);
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
uint64_t value = (uint64_t)strtoull(buf, NULL, 10);
colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value);
break;
}
}
}
static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) {
@ -453,6 +493,71 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
}
}
int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int16_t inType, int16_t outType) {
SColumnInfoData* pInputCol = pIn->columnData;
SColumnInfoData* pOutputCol = pOut->columnData;
char tmp[128] = {0};
if (IS_SIGNED_NUMERIC_TYPE(inType) || inType == TSDB_DATA_TYPE_BOOL || inType == TSDB_DATA_TYPE_TIMESTAMP) {
for (int32_t i = 0; i < pIn->numOfRows; ++i) {
if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
int64_t value = 0;
GET_TYPED_DATA(value, int64_t, inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%" PRId64, value);
varDataLen(tmp) = len;
if (outType == TSDB_DATA_TYPE_NCHAR) {
varToNchar(tmp, pOut, i);
} else {
colDataAppend(pOutputCol, i, (char *)&value, false);
}
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(inType)) {
for (int32_t i = 0; i < pIn->numOfRows; ++i) {
if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
uint64_t value = 0;
GET_TYPED_DATA(value, uint64_t, inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%" PRIu64, value);
varDataLen(tmp) = len;
if (outType == TSDB_DATA_TYPE_NCHAR) {
varToNchar(tmp, pOut, i);
} else {
colDataAppend(pOutputCol, i, (char *)&value, false);
}
}
} else if (IS_FLOAT_TYPE(inType)) {
for (int32_t i = 0; i < pIn->numOfRows; ++i) {
if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
colDataAppendNULL(pOutputCol, i);
continue;
}
double value = 0;
GET_TYPED_DATA(value, double, inType, colDataGetData(pInputCol, i));
int32_t len = sprintf(varDataVal(tmp), "%lf", value);
varDataLen(tmp) = len;
if (outType == TSDB_DATA_TYPE_NCHAR) {
varToNchar(tmp, pOut, i);
} else {
colDataAppend(pOutputCol, i, (char *)&value, false);
}
}
} else {
sclError("not supported input type:%d", inType);
return TSDB_CODE_QRY_APP_ERROR;
}
return TSDB_CODE_SUCCESS;
}
// TODO opt performance
int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
SColumnInfoData* pInputCol = pIn->columnData;
@ -610,6 +715,10 @@ int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
}
break;
}
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
return vectorConvertToVarData(pIn, pOut, inType, outType);
}
default:
sclError("invalid convert output type:%d", outType);
return TSDB_CODE_QRY_APP_ERROR;

22
source/libs/sync/src/syncRaftLog.c
View File

@ -60,7 +60,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
if (code != 0) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr);
sError("walWriteWithSyncInfo error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno));
ASSERT(0);
}
//assert(code == 0);
@ -79,7 +79,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
if (code != 0) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
sError("walReadWithHandle error, err:%d, msg:%s", err, errStr);
sError("walReadWithHandle error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno));
ASSERT(0);
}
//assert(walReadWithHandle(pWalHandle, index) == 0);
@ -108,7 +108,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
assert(walRollback(pWal, fromIndex) == 0);
//assert(walRollback(pWal, fromIndex) == 0);
int32_t code = walRollback(pWal, fromIndex);
if (code != 0) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
sError("walRollback error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno));
ASSERT(0);
}
return 0; // to avoid compiler error
}
@ -132,7 +139,14 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) {
int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
assert(walCommit(pWal, index) == 0);
//assert(walCommit(pWal, index) == 0);
int32_t code = walCommit(pWal, index);
if (code != 0) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
sError("walCommit error, err:%d, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, errStr, errno, strerror(errno));
ASSERT(0);
}
return 0; // to avoid compiler error
}

2
source/libs/transport/src/transSrv.c
View File

@ -1108,7 +1108,7 @@ void transSendResponse(const STransMsg* msg) {
SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg));
srvMsg->msg = tmsg;
srvMsg->type = Normal;
tTrace("server conn %p start to send resp (1/2)", exh->handle);
tDebug("server conn %p start to send resp (1/2)", exh->handle);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
uvReleaseExHandle(refId);
return;

1
source/util/src/terror.c
View File

@ -94,6 +94,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed
TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found")
TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization")
TAOS_DEFINE_ERROR(TSDB_CODE_DUP_KEY, "Cannot add duplicate keys to hash")
TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs")

175
source/util/src/tskiplist2.c
View File

@ -1,175 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tskiplist2.h"
struct SSLNode {
int8_t level;
SSLNode *forwards[];
};
struct SSkipList2 {
int8_t level;
uint32_t seed;
int32_t size;
const SSLCfg *pCfg;
SSLNode *pHead[];
};
static void *slMalloc(void *pPool, int32_t size);
static void slFree(void *pPool, void *p);
static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData);
const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL,
.nKey = -1,
.nData = -1,
.cmprFn = slCmprFn,
.pPool = NULL,
.xMalloc = slMalloc,
.xFree = slFree};
int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) {
SSkipList2 *pSl = NULL;
int32_t size;
*ppSl = NULL;
if (pCfg == NULL) pCfg = &slDefaultCfg;
// check config (TODO)
// malloc handle
size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2;
pSl = pCfg->xMalloc(pCfg->pPool, size);
if (pSl == NULL) {
return -1;
}
pSl->level = 0;
pSl->seed = taosRand();
pSl->size = 0;
pSl->pCfg = pCfg;
// init an empty skiplist
for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) {
pSl->pHead[i] = NULL;
}
*ppSl = pSl;
return 0;
}
int32_t slClose(SSkipList2 *pSl) {
if (pSl) {
slClear(pSl);
if (pSl->pCfg->xFree) {
pSl->pCfg->xFree(pSl->pCfg->pPool, pSl);
}
}
return 0;
}
int32_t slClear(SSkipList2 *pSl) {
// loop to clear sl
for (;;) {
// (TODO)
}
// init sl (TODO)
return 0;
}
int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) {
pSlc->pSl = pSl;
for (int i = 0; i < SL_MAX_LEVEL; i++) {
if (i < pSl->pCfg->maxLevel) {
} else {
pSlc->forwards[i] = NULL;
}
}
// TODO
return 0;
}
int32_t slcClose(SSLCursor *pSlc) {
// TODO
return 0;
}
int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) {
// TODO
return 0;
}
int32_t slcMoveToNext(SSLCursor *pSlc) {
// TODO
return 0;
}
int32_t slcMoveToPrev(SSLCursor *pSlc) {
// TODO
return 0;
}
int32_t slcMoveToFirst(SSLCursor *pSlc) {
// TODO
return 0;
}
int32_t slcMoveToLast(SSLCursor *pSlc) {
// TODO
return 0;
}
int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) {
// TODO
return 0;
}
int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) {
// TODO
return 0;
}
int32_t slcDrop(SSLCursor *pSlc) {
// TODO
return 0;
}
static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); }
static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); }
static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) {
ASSERT(nKey1 >= 0 && nKey2 >= 0);
int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1;
int32_t c;
c = memcmp(pKey1, pKey2, nKey);
if (c == 0) {
if (nKey1 > nKey2) {
c = 1;
} else if (nKey1 < nKey2) {
c = -1;
}
}
return c;
}

617
tests/script/api/batchprepare.c
View File

@ -11,7 +11,7 @@
int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT};
int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR};
int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE};
int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_NCHAR};
int32_t optrIdxList[] = {0, 1, 2};
typedef struct {
@ -20,6 +20,11 @@ typedef struct {
bool enclose;
} OperInfo;
typedef enum {
BP_BIND_TAG = 1,
BP_BIND_COL,
} BP_BIND_TYPE;
OperInfo operInfo[] = {
{">", 2, false},
{">=", 2, false},
@ -41,6 +46,8 @@ int32_t varoperatorList[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
char *bpStbPrefix = "st";
char *bpTbPrefix = "t";
int32_t bpDefaultStbId = 1;
//char *operatorList[] = {">", ">=", "<", "<=", "=", "<>", "in", "not in"};
@ -65,6 +72,7 @@ typedef struct {
char* isNull;
int32_t* binaryLen;
TAOS_MULTI_BIND* pBind;
TAOS_MULTI_BIND* pTags;
char* sql;
int32_t* colTypes;
int32_t colNum;
@ -83,6 +91,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos);
int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos);
int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos);
int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos);
int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos);
int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos);
enum {
@ -95,52 +104,55 @@ typedef struct {
int32_t colNum;
int32_t *colList; // full table column list
int32_t testType;
bool prepareStb;
bool autoCreateTbl;
bool fullCol;
int32_t (*runFn)(TAOS_STMT*, TAOS*);
int32_t tblNum;
int32_t rowNum;
int32_t bindRowNum;
int32_t bindColNum; // equal colNum in full column case
int32_t bindTagNum; // equal colNum in full column case
int32_t bindNullNum;
int32_t runTimes;
int32_t preCaseIdx;
} CaseCfg;
CaseCfg gCase[] = {
{"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 1, -1},
{"insert:MBSE1-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 1, -1},
{"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1},
{"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1},
{"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 1, -1},
{"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 1, -1},
{"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 1, -1},
{"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1},
{"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1},
{"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1},
{"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 1, -1},
{"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 1, -1},
{"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 1, -1},
{"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1},
{"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1},
{"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1},
{"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 1, -1},
{"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 1, -1},
{"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 1, -1},
{"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1},
{"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1},
{"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1},
// 11
{"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 1, -1},
{"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 1, -1},
{"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 1, -1},
{"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1},
{"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1},
{"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1},
{"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 1, -1},
{"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 1, -1},
{"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 1, -1},
{"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1},
{"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1},
{"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1},
{"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 1, -1},
{"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 1, -1},
{"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 1, -1},
{"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1},
{"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1},
{"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1},
{"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 1, -1},
{"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 1, -1},
{"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1},
{"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1},
// 22
{"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 1, 2},
{"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1},
{"query:SUBT-FULL", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, querySUBTTest1, 10, 10, 1, 3, 0, 0, 1, 2},
};
@ -149,17 +161,20 @@ CaseCfg *gCurCase = NULL;
typedef struct {
char caseCatalog[255];
int32_t bindNullNum;
bool prepareStb;
bool checkParamNum;
bool printRes;
bool printCreateTblSql;
bool printQuerySql;
bool printStmtSql;
bool autoCreateTbl;
int32_t rowNum; //row num for one table
int32_t bindColNum;
int32_t bindTagNum;
int32_t bindRowNum; //row num for once bind
int32_t bindColTypeNum;
int32_t* bindColTypeList;
int32_t bindTagTypeNum;
int32_t* bindTagTypeList;
int32_t optrIdxListNum;
int32_t* optrIdxList;
int32_t runTimes;
@ -172,15 +187,18 @@ typedef struct {
#if 0
CaseCtrl gCaseCtrl = { // default
.bindNullNum = 0,
.prepareStb = false,
.printCreateTblSql = false,
.printQuerySql = true,
.printStmtSql = true,
.autoCreateTbl = false,
.rowNum = 0,
.bindColNum = 0,
.bindTagNum = 0,
.bindRowNum = 0,
.bindColTypeNum = 0,
.bindColTypeList = NULL,
.bindTagTypeNum = 0,
.bindTagTypeList = NULL,
.optrIdxListNum = 0,
.optrIdxList = NULL,
.checkParamNum = false,
@ -193,10 +211,37 @@ CaseCtrl gCaseCtrl = { // default
};
#endif
#if 1
CaseCtrl gCaseCtrl = { // default
.bindNullNum = 0,
.printCreateTblSql = true,
.printQuerySql = true,
.printStmtSql = true,
.autoCreateTbl = false,
.rowNum = 0,
.bindColNum = 0,
.bindTagNum = 0,
.bindRowNum = 0,
.bindColTypeNum = 0,
.bindColTypeList = NULL,
.bindTagTypeNum = 0,
.bindTagTypeList = NULL,
.optrIdxListNum = 0,
.optrIdxList = NULL,
.checkParamNum = false,
.printRes = true,
.runTimes = 0,
.caseIdx = -1,
.caseNum = 1,
.caseRunIdx = 11,
.caseRunNum = 1,
};
#endif
#if 0
CaseCtrl gCaseCtrl = { // query case with specified col&oper
.bindNullNum = 0,
.prepareStb = false,
.printCreateTblSql = false,
.printQuerySql = true,
.printStmtSql = true,
@ -221,28 +266,29 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper
};
#endif
#if 1
#if 0
CaseCtrl gCaseCtrl = { // query case with specified col&oper
.bindNullNum = 0,
.prepareStb = false,
.printCreateTblSql = false,
.printCreateTblSql = true,
.printQuerySql = true,
.printStmtSql = true,
.autoCreateTbl = true,
.rowNum = 0,
.bindColNum = 0,
.bindTagNum = 0,
.bindRowNum = 0,
.bindColTypeNum = 0,
.bindColTypeList = NULL,
.optrIdxListNum = 0,
.optrIdxList = NULL,
.checkParamNum = true,
.checkParamNum = false,
.printRes = true,
.runTimes = 0,
.caseRunIdx = -1,
.optrIdxListNum = tListLen(optrIdxList),
.optrIdxList = optrIdxList,
.bindColTypeNum = tListLen(bindColTypeList),
.bindColTypeList = bindColTypeList,
//.optrIdxListNum = tListLen(optrIdxList),
//.optrIdxList = optrIdxList,
//.bindColTypeNum = tListLen(bindColTypeList),
//.bindColTypeList = bindColTypeList,
.caseIdx = 22,
.caseNum = 1,
.caseRunNum = 1,
@ -299,6 +345,78 @@ void generateInsertSQL(BindData *data) {
} else {
len = sprintf(data->sql, "insert into %s0 ", bpTbPrefix);
}
if (gCurCase->bindTagNum > 0) {
len += sprintf(data->sql + len, "using %s%d ", bpStbPrefix, bpDefaultStbId);
if (!gCurCase->fullCol) {
len += sprintf(data->sql + len, "(");
for (int c = 0; c < gCurCase->bindTagNum; ++c) {
if (c) {
len += sprintf(data->sql + len, ",");
}
switch (data->pTags[c].buffer_type) {
case TSDB_DATA_TYPE_BOOL:
len += sprintf(data->sql + len, "tbooldata");
break;
case TSDB_DATA_TYPE_TINYINT:
len += sprintf(data->sql + len, "ttinydata");
break;
case TSDB_DATA_TYPE_SMALLINT:
len += sprintf(data->sql + len, "tsmalldata");
break;
case TSDB_DATA_TYPE_INT:
len += sprintf(data->sql + len, "tintdata");
break;
case TSDB_DATA_TYPE_BIGINT:
len += sprintf(data->sql + len, "tbigdata");
break;
case TSDB_DATA_TYPE_FLOAT:
len += sprintf(data->sql + len, "tfloatdata");
break;
case TSDB_DATA_TYPE_DOUBLE:
len += sprintf(data->sql + len, "tdoubledata");
break;
case TSDB_DATA_TYPE_VARCHAR:
len += sprintf(data->sql + len, "tbinarydata");
break;
case TSDB_DATA_TYPE_TIMESTAMP:
len += sprintf(data->sql + len, "tts");
break;
case TSDB_DATA_TYPE_NCHAR:
len += sprintf(data->sql + len, "tnchardata");
break;
case TSDB_DATA_TYPE_UTINYINT:
len += sprintf(data->sql + len, "tutinydata");
break;
case TSDB_DATA_TYPE_USMALLINT:
len += sprintf(data->sql + len, "tusmalldata");
break;
case TSDB_DATA_TYPE_UINT:
len += sprintf(data->sql + len, "tuintdata");
break;
case TSDB_DATA_TYPE_UBIGINT:
len += sprintf(data->sql + len, "tubigdata");
break;
default:
printf("!!!invalid tag type:%d", data->pTags[c].buffer_type);
exit(1);
}
}
len += sprintf(data->sql + len, ") ");
}
len += sprintf(data->sql + len, "tags (");
for (int c = 0; c < gCurCase->bindTagNum; ++c) {
if (c) {
len += sprintf(data->sql + len, ",");
}
len += sprintf(data->sql + len, "?");
}
len += sprintf(data->sql + len, ") ");
}
if (!gCurCase->fullCol) {
len += sprintf(data->sql + len, "(");
for (int c = 0; c < gCurCase->bindColNum; ++c) {
@ -528,6 +646,7 @@ void generateQueryMiscSQL(BindData *data, int32_t tblIdx) {
}
void generateErrorSQL(BindData *data, int32_t tblIdx) {
int32_t len = 0;
data->sql = taosMemoryCalloc(1, 1024);
@ -552,7 +671,7 @@ void generateErrorSQL(BindData *data, int32_t tblIdx) {
}
}
void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) {
void generateColDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) {
if (bindIdx < gCurCase->bindColNum) {
if (gCaseCtrl.bindColTypeNum) {
*dataType = gCaseCtrl.bindColTypeList[colIdx];
@ -584,103 +703,145 @@ void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *
}
}
int32_t prepareColData(BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) {
int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP;
void generateTagDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) {
if (bindIdx < gCurCase->bindTagNum) {
if (gCaseCtrl.bindTagTypeNum) {
*dataType = gCaseCtrl.bindTagTypeList[colIdx];
return;
} else if (gCurCase->fullCol) {
*dataType = gCurCase->colList[bindIdx];
return;
} else {
while (true) {
*dataType = rand() % (TSDB_DATA_TYPE_MAX - 1) + 1;
if (*dataType == TSDB_DATA_TYPE_JSON || *dataType == TSDB_DATA_TYPE_DECIMAL
|| *dataType == TSDB_DATA_TYPE_BLOB || *dataType == TSDB_DATA_TYPE_MEDIUMBLOB
|| *dataType == TSDB_DATA_TYPE_VARBINARY) {
continue;
}
if (colExists(data->pTags, *dataType)) {
continue;
}
break;
}
}
} else {
*dataType = data->pTags[bindIdx%gCurCase->bindTagNum].buffer_type;
}
}
int32_t prepareColData(BP_BIND_TYPE bType, BindData *data, int32_t bindIdx, int32_t rowIdx, int32_t colIdx) {
int32_t dataType = TSDB_DATA_TYPE_TIMESTAMP;
TAOS_MULTI_BIND *pBase = NULL;
if (bType == BP_BIND_TAG) {
pBase = data->pTags;
generateTagDataType(data, bindIdx, colIdx, &dataType);
} else {
pBase = data->pBind;
generateColDataType(data, bindIdx, colIdx, &dataType);
}
generateDataType(data, bindIdx, colIdx, &dataType);
switch (dataType) {
case TSDB_DATA_TYPE_BOOL:
data->pBind[bindIdx].buffer_length = sizeof(bool);
data->pBind[bindIdx].buffer = data->boolData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(bool);
pBase[bindIdx].buffer = data->boolData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_TINYINT:
data->pBind[bindIdx].buffer_length = sizeof(int8_t);
data->pBind[bindIdx].buffer = data->tinyData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(int8_t);
pBase[bindIdx].buffer = data->tinyData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_SMALLINT:
data->pBind[bindIdx].buffer_length = sizeof(int16_t);
data->pBind[bindIdx].buffer = data->smallData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(int16_t);
pBase[bindIdx].buffer = data->smallData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_INT:
data->pBind[bindIdx].buffer_length = sizeof(int32_t);
data->pBind[bindIdx].buffer = data->intData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(int32_t);
pBase[bindIdx].buffer = data->intData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_BIGINT:
data->pBind[bindIdx].buffer_length = sizeof(int64_t);
data->pBind[bindIdx].buffer = data->bigData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(int64_t);
pBase[bindIdx].buffer = data->bigData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_FLOAT:
data->pBind[bindIdx].buffer_length = sizeof(float);
data->pBind[bindIdx].buffer = data->floatData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(float);
pBase[bindIdx].buffer = data->floatData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_DOUBLE:
data->pBind[bindIdx].buffer_length = sizeof(double);
data->pBind[bindIdx].buffer = data->doubleData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(double);
pBase[bindIdx].buffer = data->doubleData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_VARCHAR:
data->pBind[bindIdx].buffer_length = gVarCharSize;
data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize;
data->pBind[bindIdx].length = data->binaryLen;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = gVarCharSize;
pBase[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize;
pBase[bindIdx].length = data->binaryLen;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_TIMESTAMP:
data->pBind[bindIdx].buffer_length = sizeof(int64_t);
data->pBind[bindIdx].buffer = data->tsData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = NULL;
pBase[bindIdx].buffer_length = sizeof(int64_t);
pBase[bindIdx].buffer = data->tsData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = NULL;
break;
case TSDB_DATA_TYPE_NCHAR:
data->pBind[bindIdx].buffer_length = gVarCharSize;
data->pBind[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize;
data->pBind[bindIdx].length = data->binaryLen;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = gVarCharSize;
pBase[bindIdx].buffer = data->binaryData + rowIdx * gVarCharSize;
pBase[bindIdx].length = data->binaryLen;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_UTINYINT:
data->pBind[bindIdx].buffer_length = sizeof(uint8_t);
data->pBind[bindIdx].buffer = data->utinyData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(uint8_t);
pBase[bindIdx].buffer = data->utinyData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_USMALLINT:
data->pBind[bindIdx].buffer_length = sizeof(uint16_t);
data->pBind[bindIdx].buffer = data->usmallData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(uint16_t);
pBase[bindIdx].buffer = data->usmallData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_UINT:
data->pBind[bindIdx].buffer_length = sizeof(uint32_t);
data->pBind[bindIdx].buffer = data->uintData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(uint32_t);
pBase[bindIdx].buffer = data->uintData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
case TSDB_DATA_TYPE_UBIGINT:
data->pBind[bindIdx].buffer_length = sizeof(uint64_t);
data->pBind[bindIdx].buffer = data->ubigData + rowIdx;
data->pBind[bindIdx].length = NULL;
data->pBind[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
pBase[bindIdx].buffer_length = sizeof(uint64_t);
pBase[bindIdx].buffer = data->ubigData + rowIdx;
pBase[bindIdx].length = NULL;
pBase[bindIdx].is_null = data->isNull ? (data->isNull + rowIdx) : NULL;
break;
default:
printf("!!!invalid col type:%d", dataType);
exit(1);
}
data->pBind[bindIdx].buffer_type = dataType;
data->pBind[bindIdx].num = gCurCase->bindRowNum;
pBase[bindIdx].buffer_type = dataType;
pBase[bindIdx].num = gCurCase->bindRowNum;
if (bType == BP_BIND_TAG) {
pBase[bindIdx].num = 1;
}
return 0;
}
@ -694,6 +855,7 @@ int32_t prepareInsertData(BindData *data) {
data->colTypes = taosMemoryCalloc(30, sizeof(int32_t));
data->sql = taosMemoryCalloc(1, 1024);
data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND));
data->pTags = taosMemoryCalloc(gCurCase->tblNum*gCurCase->bindTagNum, sizeof(TAOS_MULTI_BIND));
data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t));
data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool));
data->tinyData = taosMemoryMalloc(allRowNum * sizeof(int8_t));
@ -734,10 +896,17 @@ int32_t prepareInsertData(BindData *data) {
for (int b = 0; b < (allRowNum/gCurCase->bindRowNum); b++) {
for (int c = 0; c < gCurCase->bindColNum; ++c) {
prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c);
prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c);
}
}
for (int b = 0; b < gCurCase->tblNum; b++) {
for (int c = 0; c < gCurCase->bindTagNum; ++c) {
prepareColData(BP_BIND_TAG, data, b*gCurCase->bindTagNum+c, b, c);
}
}
generateInsertSQL(data);
return 0;
@ -791,11 +960,11 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) {
for (int b = 0; b < bindNum; b++) {
for (int c = 0; c < gCurCase->bindColNum; ++c) {
prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c);
prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c);
}
}
generateQuerySQL(data, tblIdx);
generateQueryCondSQL(data, tblIdx);
return 0;
}
@ -849,7 +1018,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) {
for (int b = 0; b < bindNum; b++) {
for (int c = 0; c < gCurCase->bindColNum; ++c) {
prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c);
prepareColData(BP_BIND_COL, data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c);
}
}
@ -1099,6 +1268,15 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B
printf("***sql res num match stmt res num %d\n", stmtResNum);
}
int32_t bpSetTableNameTags(BindData *data, int32_t tblIdx, char *tblName, TAOS_STMT *stmt) {
if (gCurCase->bindTagNum > 0) {
return taos_stmt_set_tbname_tags(stmt, tblName, data->pTags + tblIdx * gCurCase->bindTagNum);
} else {
return taos_stmt_set_tbname(stmt, tblName);
}
}
/* prepare [settbname [bind add]] exec */
int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) {
BindData data = {0};
@ -1117,7 +1295,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1174,7 +1352,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1227,7 +1405,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1281,7 +1459,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1335,7 +1513,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1350,7 +1528,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1402,7 +1580,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1458,7 +1636,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname(stmt, buf);
code = bpSetTableNameTags(&data, t, buf, stmt);
if (code != 0){
printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt));
exit(1);
@ -1502,6 +1680,74 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) {
return 0;
}
/* [prepare [settbnametag [bind add] exec]] */
int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) {
int32_t loop = 0;
while (gCurCase->bindTagNum > 0 && gCurCase->bindColNum > 0) {
BindData data = {0};
prepareInsertData(&data);
int code = taos_stmt_prepare(stmt, data.sql, 0);
if (code != 0){
printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt));
exit(1);
}
bpCheckIsInsert(stmt, 1);
int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum;
for (int32_t t = 0; t< gCurCase->tblNum; ++t) {
if (gCurCase->tblNum > 1) {
char buf[32];
sprintf(buf, "t%d", t);
code = taos_stmt_set_tbname_tags(stmt, buf, data.pTags + t * gCurCase->bindTagNum);
if (code != 0){
printf("!!!taos_stmt_set_tbname_tags error:%s\n", taos_stmt_errstr(stmt));
exit(1);
}
}
if (gCaseCtrl.checkParamNum) {
bpCheckParamNum(stmt);
}
for (int32_t b = 0; b <bindTimes; ++b) {
if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) {
exit(1);
}
if (taos_stmt_add_batch(stmt)) {
printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt));
exit(1);
}
}
if (taos_stmt_execute(stmt) != 0) {
printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt));
exit(1);
}
}
bpCheckIsInsert(stmt, 1);
destroyData(&data);
gCurCase->bindColNum -= 2;
gCurCase->bindTagNum -= 2;
gCurCase->fullCol = false;
loop++;
}
bpCheckAffectedRows(stmt, loop);
gExecLoopTimes = loop;
return 0;
}
/* select * from table */
int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) {
BindData data = {0};
@ -2911,63 +3157,6 @@ int sql_s_perf1(TAOS *taos) {
void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *colList, bool stable) {
int32_t blen = 0;
blen = sprintf(buf, "create table %s%d ", (stable ? bpStbPrefix : bpTbPrefix), tblIdx);
if (stable) {
blen += sprintf(buf + blen, "tags (");
for (int c = 0; c < colNum; ++c) {
if (c > 0) {
blen += sprintf(buf + blen, ",");
}
switch (colList[c]) {
case TSDB_DATA_TYPE_BOOL:
blen += sprintf(buf + blen, "tbooldata bool");
break;
case TSDB_DATA_TYPE_TINYINT:
blen += sprintf(buf + blen, "ttinydata tinyint");
break;
case TSDB_DATA_TYPE_SMALLINT:
blen += sprintf(buf + blen, "tsmalldata smallint");
break;
case TSDB_DATA_TYPE_INT:
blen += sprintf(buf + blen, "tintdata int");
break;
case TSDB_DATA_TYPE_BIGINT:
blen += sprintf(buf + blen, "tbigdata bigint");
break;
case TSDB_DATA_TYPE_FLOAT:
blen += sprintf(buf + blen, "tfloatdata float");
break;
case TSDB_DATA_TYPE_DOUBLE:
blen += sprintf(buf + blen, "tdoubledata double");
break;
case TSDB_DATA_TYPE_VARCHAR:
blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
blen += sprintf(buf + blen, "tts ts");
break;
case TSDB_DATA_TYPE_NCHAR:
blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize);
break;
case TSDB_DATA_TYPE_UTINYINT:
blen += sprintf(buf + blen, "tutinydata tinyint unsigned");
break;
case TSDB_DATA_TYPE_USMALLINT:
blen += sprintf(buf + blen, "tusmalldata smallint unsigned");
break;
case TSDB_DATA_TYPE_UINT:
blen += sprintf(buf + blen, "tuintdata int unsigned");
break;
case TSDB_DATA_TYPE_UBIGINT:
blen += sprintf(buf + blen, "tubigdata bigint unsigned");
break;
default:
printf("invalid col type:%d", colList[c]);
exit(1);
}
}
blen += sprintf(buf + blen, ")");
}
blen += sprintf(buf + blen, " (");
@ -3027,6 +3216,64 @@ void generateCreateTableSQL(char *buf, int32_t tblIdx, int32_t colNum, int32_t *
blen += sprintf(buf + blen, ")");
if (stable) {
blen += sprintf(buf + blen, "tags (");
for (int c = 0; c < colNum; ++c) {
if (c > 0) {
blen += sprintf(buf + blen, ",");
}
switch (colList[c]) {
case TSDB_DATA_TYPE_BOOL:
blen += sprintf(buf + blen, "tbooldata bool");
break;
case TSDB_DATA_TYPE_TINYINT:
blen += sprintf(buf + blen, "ttinydata tinyint");
break;
case TSDB_DATA_TYPE_SMALLINT:
blen += sprintf(buf + blen, "tsmalldata smallint");
break;
case TSDB_DATA_TYPE_INT:
blen += sprintf(buf + blen, "tintdata int");
break;
case TSDB_DATA_TYPE_BIGINT:
blen += sprintf(buf + blen, "tbigdata bigint");
break;
case TSDB_DATA_TYPE_FLOAT:
blen += sprintf(buf + blen, "tfloatdata float");
break;
case TSDB_DATA_TYPE_DOUBLE:
blen += sprintf(buf + blen, "tdoubledata double");
break;
case TSDB_DATA_TYPE_VARCHAR:
blen += sprintf(buf + blen, "tbinarydata binary(%d)", gVarCharSize);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
blen += sprintf(buf + blen, "tts timestamp");
break;
case TSDB_DATA_TYPE_NCHAR:
blen += sprintf(buf + blen, "tnchardata nchar(%d)", gVarCharSize);
break;
case TSDB_DATA_TYPE_UTINYINT:
blen += sprintf(buf + blen, "tutinydata tinyint unsigned");
break;
case TSDB_DATA_TYPE_USMALLINT:
blen += sprintf(buf + blen, "tusmalldata smallint unsigned");
break;
case TSDB_DATA_TYPE_UINT:
blen += sprintf(buf + blen, "tuintdata int unsigned");
break;
case TSDB_DATA_TYPE_UBIGINT:
blen += sprintf(buf + blen, "tubigdata bigint unsigned");
break;
default:
printf("invalid col type:%d", colList[c]);
exit(1);
}
}
blen += sprintf(buf + blen, ")");
}
if (gCaseCtrl.printCreateTblSql) {
printf("\tCreate Table SQL:%s\n", buf);
}
@ -3067,7 +3314,7 @@ void prepare(TAOS *taos, int32_t colNum, int32_t *colList, int prepareStb) {
}
} else {
char buf[1024];
generateCreateTableSQL(buf, 1, colNum, colList, true);
generateCreateTableSQL(buf, bpDefaultStbId, colNum, colList, true);
result = taos_query(taos, buf);
code = taos_errno(result);
@ -3111,16 +3358,33 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) {
gCurCase->rowNum = gCaseCtrl.rowNum;
}
if (gCaseCtrl.autoCreateTbl) {
if (gCurCase->testType == TTYPE_INSERT && gCurCase->tblNum > 1) {
gCurCase->autoCreateTbl = true;
if (gCurCase->bindTagNum <= 0) {
gCurCase->bindTagNum = gCurCase->colNum;
}
} else {
return 1;
}
}
if (gCurCase->fullCol) {
gCurCase->bindColNum = gCurCase->colNum;
if (gCurCase->autoCreateTbl) {
gCurCase->bindTagNum = gCurCase->colNum;
}
}
gCurCase->bindNullNum = gCaseCtrl.bindNullNum;
gCurCase->prepareStb = gCaseCtrl.prepareStb;
if (gCaseCtrl.bindColNum) {
gCurCase->bindColNum = gCaseCtrl.bindColNum;
gCurCase->fullCol = false;
}
if (gCaseCtrl.bindTagNum) {
gCurCase->bindTagNum = gCaseCtrl.bindTagNum;
gCurCase->fullCol = false;
}
if (gCaseCtrl.bindRowNum) {
gCurCase->bindRowNum = gCaseCtrl.bindRowNum;
}
@ -3128,6 +3392,10 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) {
gCurCase->bindColNum = gCaseCtrl.bindColTypeNum;
gCurCase->fullCol = false;
}
if (gCaseCtrl.bindTagTypeNum) {
gCurCase->bindTagNum = gCaseCtrl.bindTagTypeNum;
gCurCase->fullCol = false;
}
if (!silent) {
printf("* Case %d - [%s]%s Begin *\n", caseRunIdx, gCaseCtrl.caseCatalog, gCurCase->caseDesc);
@ -3136,7 +3404,7 @@ int32_t runCase(TAOS *taos, int32_t caseIdx, int32_t caseRunIdx, bool silent) {
totalUs = 0;
for (int32_t n = 0; n < gCurCase->runTimes; ++n) {
if (gCurCase->preCaseIdx < 0) {
prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->prepareStb);
prepare(taos, gCurCase->colNum, gCurCase->colList, gCurCase->autoCreateTbl);
}
beginUs = taosGetTimestampUs();
@ -3197,10 +3465,18 @@ void* runCaseList(TAOS *taos) {
}
void runAll(TAOS *taos) {
/*
strcpy(gCaseCtrl.caseCatalog, "Normal Test");
printf("%s Begin\n", gCaseCtrl.caseCatalog);
runCaseList(taos);
*/
strcpy(gCaseCtrl.caseCatalog, "Auto Create Table Test");
gCaseCtrl.autoCreateTbl = true;
printf("%s Begin\n", gCaseCtrl.caseCatalog);
runCaseList(taos);
gCaseCtrl.autoCreateTbl = false;
/*
strcpy(gCaseCtrl.caseCatalog, "Null Test");
printf("%s Begin\n", gCaseCtrl.caseCatalog);
gCaseCtrl.bindNullNum = 1;
@ -3244,6 +3520,7 @@ void runAll(TAOS *taos) {
gCaseCtrl.bindColTypeNum = tListLen(bindColTypeList);
gCaseCtrl.bindColTypeList = bindColTypeList;
runCaseList(taos);
*/
printf("All Test End\n");
}

26
tests/system-test/1-insert/insertWithMoreVgroup.py
View File

@ -199,7 +199,8 @@ class TDTestCase:
os.system("%s -f %s -y " %(taosBenchbin,jsonFile))
return
def taosBenchCreate(self,dbname,stbname,vgroups,threadNumbers,count):
def taosBenchCreate(self,host,dropdb,dbname,stbname,vgroups,threadNumbers,count):
# count=50000
buildPath = self.getBuildPath()
if (buildPath == ""):
@ -207,20 +208,28 @@ class TDTestCase:
else:
tdLog.info("taosd found in %s" % buildPath)
taosBenchbin = buildPath+ "/build/bin/taosBenchmark"
buildPath = self.getBuildPath()
config = buildPath+ "../sim/dnode1/cfg/"
tsql=self.newcur(host,config)
# insert: create one or mutiple tables per sql and insert multiple rows per sql
tdSql.execute("drop database if exists %s"%dbname)
tsql.execute("drop database if exists %s"%dbname)
tdSql.execute("create database %s vgroups %d"%(dbname,vgroups))
tdSql.execute("use %s" %dbname)
tsql.execute("create database %s vgroups %d"%(dbname,vgroups))
print("db has been created")
# tsql.getResult("show databases")
# print(tdSql.queryResult)
tsql.execute("use %s" %dbname)
threads = []
# threadNumbers=2
for i in range(threadNumbers):
jsonfile="1-insert/Vgroups%d%d.json"%(vgroups,i)
os.system("cp -f 1-insert/manyVgroups.json %s"%(jsonfile))
os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s%d\",/g' %s"%(dbname,i,jsonfile))
os.system("sed -i 's/\"childtable_count\": 300000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile))
os.system("sed -i 's/\"name\": \"db\",/\"name\": \"%s\",/g' %s"%(dbname,jsonfile))
os.system("sed -i 's/\"drop\": \"no\",/\"drop\": \"%s\",/g' %s"%(dropdb,jsonfile))
os.system("sed -i 's/\"host\": \"127.0.0.1\",/\"host\": \"%s\",/g' %s"%(host,jsonfile))
os.system("sed -i 's/\"childtable_count\": 10000,/\"childtable_count\": %d,/g' %s "%(count,jsonfile))
os.system("sed -i 's/\"name\": \"stb1\",/\"name\": \"%s%d\",/g' %s "%(stbname,i,jsonfile))
os.system("sed -i 's/\"childtable_prefix\": \"stb1_\",/\"childtable_prefix\": \"%s%d_\",/g' %s "%(stbname,i,jsonfile))
threads.append(mp.Process(target=self.taosBench, args=("%s"%jsonfile,)))
@ -337,8 +346,11 @@ class TDTestCase:
return
def test_case3(self):
# self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000)
self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*1000)
self.taosBenchCreate("db1", "stb1", 1, 2, 1*50000)
# self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000)
# self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000)
return

6
tests/system-test/1-insert/manyVgroups.json
View File

@ -1,7 +1,7 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos/",
"host": "test216",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
@ -16,14 +16,14 @@
{
"dbinfo": {
"name": "db",
"drop": "yes",
"drop": "no",
"vgroups": 1
},
"super_tables": [
{
"name": "stb1",
"child_table_exists": "no",
"childtable_count": 300000,
"childtable_count": 10000,
"childtable_prefix": "stb1_",
"auto_create_table": "no",
"batch_create_tbl_num": 50000,

49
tests/system-test/2-query/diff.py
View File

@ -83,17 +83,20 @@ class TDTestCase:
tdSql.query("select diff(col6) from stb_1")
tdSql.checkRows(0)
tdSql.query("select diff(col7) from stb_1")
tdSql.checkRows(0)
for i in range(self.rowNum):
tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)"
% (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
tdSql.error("select diff(ts) from stb")
# tdSql.error("select diff(ts) from stb")
tdSql.error("select diff(ts) from stb_1")
tdSql.error("select diff(col7) from stb")
tdSql.error("select diff(col7) from stb_1")
tdSql.error("select diff(col8) from stb")
# tdSql.error("select diff(col7) from stb")
# tdSql.error("select diff(col8) from stb")
tdSql.error("select diff(col8) from stb_1")
tdSql.error("select diff(col9) from stb")
# tdSql.error("select diff(col9) from stb")
tdSql.error("select diff(col9) from stb_1")
tdSql.error("select diff(col11) from stb_1")
tdSql.error("select diff(col12) from stb_1")
@ -101,31 +104,12 @@ class TDTestCase:
tdSql.error("select diff(col14) from stb_1")
tdSql.query("select ts,diff(col1),ts from stb_1")
tdSql.checkRows(10)
tdSql.checkRows(11)
tdSql.checkData(0, 0, "2018-09-17 09:00:00.000")
tdSql.checkData(0, 1, "2018-09-17 09:00:00.000")
tdSql.checkData(0, 3, "2018-09-17 09:00:00.000")
tdSql.checkData(1, 0, "2018-09-17 09:00:00.000")
tdSql.checkData(1, 2, "2018-09-17 09:00:00.000")
tdSql.checkData(9, 0, "2018-09-17 09:00:00.009")
tdSql.checkData(9, 1, "2018-09-17 09:00:00.009")
tdSql.checkData(9, 3, "2018-09-17 09:00:00.009")
# tdSql.query("select ts,diff(col1),ts from stb group by tbname")
# tdSql.checkRows(10)
# tdSql.checkData(0, 0, "2018-09-17 09:00:00.000")
# tdSql.checkData(0, 1, "2018-09-17 09:00:00.000")
# tdSql.checkData(0, 3, "2018-09-17 09:00:00.000")
# tdSql.checkData(9, 0, "2018-09-17 09:00:00.009")
# tdSql.checkData(9, 1, "2018-09-17 09:00:00.009")
# tdSql.checkData(9, 3, "2018-09-17 09:00:00.009")
tdSql.query("select ts,diff(col1),ts from stb_1")
tdSql.checkRows(10)
tdSql.checkData(0, 0, "2018-09-17 09:00:00.000")
tdSql.checkData(0, 1, "2018-09-17 09:00:00.000")
tdSql.checkData(0, 3, "2018-09-17 09:00:00.000")
tdSql.checkData(9, 0, "2018-09-17 09:00:00.009")
tdSql.checkData(9, 1, "2018-09-17 09:00:00.009")
tdSql.checkData(9, 3, "2018-09-17 09:00:00.009")
tdSql.checkData(9, 2, "2018-09-17 09:00:00.009")
# tdSql.query("select ts,diff(col1),ts from stb group by tbname")
# tdSql.checkRows(10)
@ -154,15 +138,6 @@ class TDTestCase:
tdSql.query("select diff(col6) from stb_1")
tdSql.checkRows(10)
self.insertData()
tdSql.query("select diff(col) from st group by tbname")
tdSql.checkRows(185)
tdSql.error("select diff(col) from st group by dev")
tdSql.error("select diff(col) from st group by col")
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)

1
tests/system-test/fulltest.sh
View File

@ -49,3 +49,4 @@ python3 ./test.py -f 2-query/tan.py
python3 ./test.py -f 2-query/arcsin.py
python3 ./test.py -f 2-query/arccos.py
python3 ./test.py -f 2-query/arctan.py
python3 ./test.py -f 2-query/query_cols_tags_and_or.py

76
tests/system-test/insert.json Normal file
View File

@ -0,0 +1,76 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 16,
"create_table_thread_count": 1,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 0,
"num_of_records_per_req": 10000,
"prepared_rand": 10000,
"chinese": "no",
"databases": [
{
"dbinfo": {
"name": "db",
"drop": "yes",
"vgroups":4,
"replica": 1,
"precision": "ms"
},
"super_tables": [
{
"name": "stb",
"child_table_exists": "no",
"childtable_count": 1000,
"childtable_prefix": "stb_",
"escape_character": "no",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"non_stop_mode": "no",
"line_protocol": "line",
"insert_rows": 100000,
"interlace_rows": 0,
"insert_interval": 0,
"disorder_ratio": 0,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"use_sample_ts": "no",
"tags_file": "",
"columns": [
{
"type": "FLOAT",
"name": "current",
"count": 4,
"max": 12,
"min": 8
},
{ "type": "INT", "name": "voltage", "max": 225, "min": 215 },
{ "type": "FLOAT", "name": "phase", "max": 1, "min": 0 }
],
"tags": [
{
"type": "TINYINT",
"name": "groupid",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "BINARY",
"len": 16,
"values": ["beijing", "shanghai"]
}
]
}
]
}
]
}