xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into merge/3.0to3.3.6

This commit is contained in:
Simon Guan 2025-03-15 09:27:22 +08:00
parent 6d9a4a57da 2de6736160
commit 8c8f0be8c5
215 changed files with 25163 additions and 2033 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/14-reference/09-error-code.md
View File

@ -73,6 +73,8 @@ This document details the server error codes that may be encountered when using
| 0x80000134 | Invalid value | Invalid value | Preserve the scene and logs, report issue on github |
| 0x80000135 | Invalid fqdn | Invalid FQDN | Check if the configured or input FQDN value is correct |
| 0x8000013C | Invalid disk id | Invalid disk id | Check users whether the mounted disk is invalid or use the parameter diskIDCheckEnabled to skip the disk check. |
| 0x8000013D | Decimal value overflow | Decimal value overflow | Check query expression and decimal values |
| 0x8000013E | Division by zero error | Division by zero | Check division expression |
## tsc

2
docs/zh/14-reference/09-error-code.md
View File

@ -76,6 +76,8 @@ description: TDengine 服务端的错误码列表和详细说明
| 0x80000134 | Invalid value | 无效值 | 保留现场和日志，github 上报 issue |
| 0x80000135 | Invalid fqdn | 无效 FQDN | 检查配置或输入的 FQDN 值是否正确 |
| 0x8000013C | Invalid disk id | 不合法的 disk id | 建议用户检查挂载磁盘是否失效或者使用参数 diskIDCheckEnabled 来跳过磁盘检查 |
| 0x8000013D | Decimal value overflow | Decimal 计算溢出 | 检查计算表达式和参数值是否计算结果导致类型溢出 |
| 0x8000013E | Division by zero error | Division by zero | 检查除法操作是否除以0 |

4
include/client/taos.h
View File

@ -52,7 +52,8 @@ typedef void TAOS_SUB;
#define TSDB_DATA_TYPE_MEDIUMBLOB 19
#define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string
#define TSDB_DATA_TYPE_GEOMETRY 20 // geometry
#define TSDB_DATA_TYPE_MAX 21
#define TSDB_DATA_TYPE_DECIMAL64 21 // decimal64
#define TSDB_DATA_TYPE_MAX 22
typedef enum {
TSDB_OPTION_LOCALE,
@ -270,6 +271,7 @@ DLL_EXPORT int taos_affected_rows(TAOS_RES *res);
DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res);
DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
DLL_EXPORT TAOS_FIELD_E *taos_fetch_fields_e(TAOS_RES *res);
DLL_EXPORT int taos_select_db(TAOS *taos, const char *db);
DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);

1
include/common/streamMsg.h
View File

@ -40,6 +40,7 @@ typedef struct SStreamUpstreamEpInfo {
SEpSet epSet;
bool dataAllowed; // denote if the data from this upstream task is allowed to put into inputQ, not serialize it
int64_t stage; // upstream task stage value, to denote if the upstream node has restart/replica changed/transfer
int64_t lastMsgId;
} SStreamUpstreamEpInfo;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo);

2
include/common/tcol.h
View File

@ -79,7 +79,7 @@ uint8_t columnLevelVal(const char* level);
uint8_t columnEncodeVal(const char* encode);
uint16_t columnCompressVal(const char* compress);
bool useCompress(uint8_t tableType);
bool withExtSchema(uint8_t tableType);
bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]);

37
include/common/tcommon.h
View File

@ -132,7 +132,7 @@ enum {
STREAM_INPUT__DATA_SUBMIT = 1,
STREAM_INPUT__DATA_BLOCK,
STREAM_INPUT__MERGED_SUBMIT,
STREAM_INPUT__TQ_SCAN,
STREAM_INPUT__RECALCULATE,
STREAM_INPUT__DATA_RETRIEVE,
STREAM_INPUT__GET_RES,
STREAM_INPUT__CHECKPOINT,
@ -162,18 +162,43 @@ typedef enum EStreamType {
STREAM_GET_RESULT,
STREAM_DROP_CHILD_TABLE,
STREAM_NOTIFY_EVENT,
STREAM_RECALCULATE_DATA,
STREAM_RECALCULATE_DELETE,
STREAM_RECALCULATE_START,
STREAM_RECALCULATE_END,
} EStreamType;
#pragma pack(push, 1)
typedef struct SColumnDataAgg {
int16_t colId;
int32_t colId;
int16_t numOfNull;
union {
struct {
int64_t sum;
int64_t max;
int64_t min;
};
struct {
uint64_t decimal128Sum[2];
uint64_t decimal128Max[2];
uint64_t decimal128Min[2];
uint8_t overflow;
};
};
} SColumnDataAgg;
#pragma pack(pop)
#define DECIMAL_AGG_FLAG 0x80000000
#define COL_AGG_GET_SUM_PTR(pAggs, dataType) \
(!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->sum : (void*)pAggs->decimal128Sum)
#define COL_AGG_GET_MAX_PTR(pAggs, dataType) \
(!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->max : (void*)pAggs->decimal128Max)
#define COL_AGG_GET_MIN_PTR(pAggs, dataType) \
(!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->min : (void*)pAggs->decimal128Min)
typedef struct SBlockID {
// The uid of table, from which current data block comes. And it is always 0, if current block is the
// result of calculation.
@ -430,6 +455,14 @@ static inline bool isTsmaResSTb(const char* stbName) {
return false;
}
static inline STypeMod typeGetTypeModFromColInfo(const SColumnInfo* pCol) {
return typeGetTypeMod(pCol->type, pCol->precision, pCol->scale, pCol->bytes);
}
static inline STypeMod typeGetTypeModFromCol(const SColumn* pCol) {
return typeGetTypeMod(pCol->type, pCol->precision, pCol->scale, pCol->bytes);
}
#ifdef __cplusplus
}
#endif

3
include/common/tdatablock.h
View File

@ -80,6 +80,9 @@ typedef struct SBlockOrderInfo {
#define IS_JSON_NULL(type, data) \
((type) == TSDB_DATA_TYPE_JSON && (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data)))
#define GET_COL_DATA_TYPE(col) \
{ .type = (col).type, .precision = (col).precision, .bytes = (col).bytes, .scale = (col).scale }
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (!pColumnInfoData->hasNull) {
return false;

16
include/common/tdataformat.h
View File

@ -43,6 +43,8 @@ typedef struct SColData SColData;
typedef struct SRowKey SRowKey;
typedef struct SValueColumn SValueColumn;
struct SColumnDataAgg;
typedef struct SColumnDataAgg* SColumnDataAggPtr;
#define HAS_NONE ((uint8_t)0x1)
#define HAS_NULL ((uint8_t)0x2)
@ -187,7 +189,7 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg);
void tColDataArrGetRowKey(SColData *aColData, int32_t nColData, int32_t iRow, SRowKey *key);
extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull);
extern void (*tColDataCalcSMA[])(SColData *pColData, SColumnDataAggPtr pAggs);
int32_t tColDataCompress(SColData *colData, SColDataCompressInfo *info, SBuffer *output, SBuffer *assist);
int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *colData, SBuffer *assist);
@ -244,6 +246,8 @@ typedef struct {
uint32_t offset;
} SPrimaryKeyIndex;
#define DATUM_MAX_SIZE 16
struct SValue {
int8_t type;
union {
@ -255,6 +259,16 @@ struct SValue {
};
};
#define VALUE_GET_DATUM(pVal, type) \
(IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) ? (pVal)->pData : (void*)&(pVal)->val
#define VALUE_GET_TRIVIAL_DATUM(pVal) ((pVal)->val)
#define VALUE_SET_TRIVIAL_DATUM(pVal, v) (pVal)->val = v
void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len);
void valueCloneDatum(SValue *pDst, const SValue *pSrc, int8_t type);
void valueClearDatum(SValue *pVal, int8_t type);
#define TD_MAX_PK_COLS 2
struct SRowKey {
TSKEY ts;

4
include/common/tglobal.h
View File

@ -303,6 +303,10 @@ extern int32_t tsStreamNotifyMessageSize;
extern int32_t tsStreamNotifyFrameSize;
extern bool tsCompareAsStrInGreatest;
extern char tsAdapterFqdn[];
extern uint16_t tsAdapterPort;
extern char tsAdapterToken[];
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)

41
include/common/tmsg.h
View File

@ -485,6 +485,14 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC,
QUERY_NODE_RESET_STREAM_STMT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT,
} ENodeType;
typedef struct {
@ -515,6 +523,7 @@ typedef struct SFieldWithOptions {
int8_t flags;
int32_t bytes;
uint32_t compress;
STypeMod typeMod;
} SFieldWithOptions;
typedef struct SRetention {
@ -596,6 +605,7 @@ struct SSchema {
struct SSchemaExt {
col_id_t colId;
uint32_t compress;
STypeMod typeMod;
};
//
@ -659,6 +669,7 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define COL_SET_NULL ((int8_t)0x10)
#define COL_SET_VAL ((int8_t)0x20)
#define COL_IS_SYSINFO ((int8_t)0x40)
#define COL_HAS_TYPE_MOD ((int8_t)0x80)
#define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0)
#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL)))
@ -677,6 +688,13 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp);
(s)->flags &= (~COL_IDX_ON); \
} while (0)
#define SSCHEMA_SET_TYPE_MOD(s) \
do { \
(s)->flags |= COL_HAS_TYPE_MOD; \
} while (0)
#define HAS_TYPE_MOD(s) (((s)->flags & COL_HAS_TYPE_MOD))
#define SSCHMEA_TYPE(s) ((s)->type)
#define SSCHMEA_FLAGS(s) ((s)->flags)
#define SSCHMEA_COLID(s) ((s)->colId)
@ -693,6 +711,12 @@ typedef struct {
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN];
} SMonitorParas;
typedef struct {
STypeMod typeMod;
} SExtSchema;
bool hasExtSchema(const SExtSchema* pExtSchema);
typedef struct {
int32_t nCols;
int32_t version;
@ -843,12 +867,14 @@ static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema)
static FORCE_INLINE int32_t tEncodeSSchemaExt(SEncoder* pEncoder, const SSchemaExt* pSchemaExt) {
TAOS_CHECK_RETURN(tEncodeI16v(pEncoder, pSchemaExt->colId));
TAOS_CHECK_RETURN(tEncodeU32(pEncoder, pSchemaExt->compress));
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pSchemaExt->typeMod));
return 0;
}
static FORCE_INLINE int32_t tDecodeSSchemaExt(SDecoder* pDecoder, SSchemaExt* pSchemaExt) {
TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &pSchemaExt->colId));
TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &pSchemaExt->compress));
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pSchemaExt->typeMod));
return 0;
}
@ -881,6 +907,7 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp
}
static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) {
if (pSW == NULL) {return TSDB_CODE_INVALID_PARA;}
TAOS_CHECK_RETURN(tEncodeI32v(pEncoder, pSW->nCols));
TAOS_CHECK_RETURN(tEncodeI32v(pEncoder, pSW->version));
for (int32_t i = 0; i < pSW->nCols; i++) {
@ -890,6 +917,7 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSch
}
static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) {
if (pSW == NULL) {return TSDB_CODE_INVALID_PARA;}
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pSW->nCols));
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pSW->version));
@ -988,6 +1016,7 @@ typedef struct {
int32_t sqlLen;
char* sql;
int64_t keep;
SArray* pTypeMods;
} SMAlterStbReq;
int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq);
@ -2940,6 +2969,7 @@ typedef struct {
#define STREAM_TRIGGER_WINDOW_CLOSE 2
#define STREAM_TRIGGER_MAX_DELAY 3
#define STREAM_TRIGGER_FORCE_WINDOW_CLOSE 4
#define STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE 5
#define STREAM_DEFAULT_IGNORE_EXPIRED 1
#define STREAM_FILL_HISTORY_ON 1
@ -2992,6 +3022,11 @@ typedef struct {
int32_t notifyEventTypes;
int32_t notifyErrorHandle;
int8_t notifyHistory;
int64_t recalculateInterval;
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} SCMCreateStreamReq;
typedef struct STaskNotifyEventStat {
@ -3243,6 +3278,7 @@ typedef struct SVCreateStbReq {
int8_t colCmpred;
SColCmprWrapper colCmpr;
int64_t keep;
SExtSchema* pExtSchemas;
} SVCreateStbReq;
int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@ -3283,6 +3319,7 @@ typedef struct SVCreateTbReq {
int32_t sqlLen;
char* sql;
SColCmprWrapper colCmpr;
SExtSchema* pExtSchemas;
} SVCreateTbReq;
int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq);
@ -3306,6 +3343,7 @@ static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) {
taosMemoryFreeClear(req->ntb.schemaRow.pSchema);
}
taosMemoryFreeClear(req->colCmpr.pColCmpr);
taosMemoryFreeClear(req->pExtSchemas);
}
typedef struct {
@ -3422,6 +3460,8 @@ typedef struct {
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
uint32_t compress; // TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS
SArray* pMultiTag; // TSDB_ALTER_TABLE_ADD_MULTI_TAGS
// for Add column
STypeMod typeMod;
} SVAlterTbReq;
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@ -4316,7 +4356,6 @@ typedef struct {
};
void* data; //for free in client, only effected if type is data or metadata. raw data not effected
bool blockDataElementFree; // if true, free blockDataElement in blockData,(true in server, false in client)
} SMqDataRsp;
int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pObj);

59
include/common/ttypes.h
View File

@ -45,6 +45,9 @@ typedef struct {
} SNCharNullT;
#pragma pack(pop)
#define STypeMod int32_t
void extractTypeFromTypeMod(uint8_t type, STypeMod typeMod, uint8_t *prec, uint8_t* scale, int32_t *bytes);
#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
#define varDataCopy(dst, v) (void)memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
@ -53,7 +56,26 @@ typedef struct {
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
#define GET_TYPED_DATA(_v, _finalType, _type, _data) \
#define DEFINE_TYPE_FROM_DECIMAL_FUNC(oType, decimalType) \
oType oType##From##decimalType(const void* pDec, uint8_t prec, uint8_t scale)
#define DEFINE_TYPE_FROM_DECIMAL_FUNCS(prefix, decimalType) \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(bool, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int8_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint8_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int16_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint16_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int32_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint32_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int64_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint64_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(float, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(double, decimalType);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(extern, Decimal64);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(extern, Decimal128);
#define GET_TYPED_DATA(_v, _finalType, _type, _data, inputTypeMod) \
do { \
switch (_type) { \
case TSDB_DATA_TYPE_BOOL: \
@ -88,6 +110,16 @@ typedef struct {
case TSDB_DATA_TYPE_INT: \
(_v) = (_finalType)GET_INT32_VAL(_data); \
break; \
case TSDB_DATA_TYPE_DECIMAL: { \
uint8_t prec = 0, scale = 0; \
extractTypeFromTypeMod(_type, inputTypeMod, &prec, &scale, NULL); \
(_v) = _finalType##FromDecimal128(_data, prec, scale); \
} break; \
case TSDB_DATA_TYPE_DECIMAL64: { \
uint8_t prec = 0, scale = 0; \
extractTypeFromTypeMod(_type, inputTypeMod, &prec, &scale, NULL); \
(_v) = _finalType##FromDecimal64(_data, prec, scale); \
} break; \
default: \
(_v) = (_finalType)varDataLen(_data); \
break; \
@ -265,8 +297,9 @@ typedef struct {
#define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)))
#define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP)
#define IS_BOOLEAN_TYPE(_t) ((_t) == TSDB_DATA_TYPE_BOOL)
#define IS_DECIMAL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_DECIMAL || (_t) == TSDB_DATA_TYPE_DECIMAL64)
#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)) || (IS_DECIMAL_TYPE(_t)))
#define IS_MATHABLE_TYPE(_t) \
(IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
@ -364,6 +397,13 @@ typedef struct tDataTypeCompress {
extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
extern tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX];
typedef struct SDataType {
uint8_t type;
uint8_t precision;
uint8_t scale;
int32_t bytes;
} SDataType;
bool isValidDataType(int32_t type);
int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
@ -371,6 +411,21 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type);
void *getDataMin(int32_t type, void *value);
void *getDataMax(int32_t type, void *value);
STypeMod typeGetTypeMod(uint8_t type, uint8_t prec, uint8_t scale, int32_t bytes);
STypeMod typeGetTypeModFromDataType(const SDataType* pDataType);
uint8_t decimalTypeFromPrecision(uint8_t precision);
STypeMod decimalCalcTypeMod(uint8_t prec, uint8_t scale);
void decimalFromTypeMod(STypeMod typeMod, uint8_t *precision, uint8_t *scale);
// pType->type should has been set
void fillTypeFromTypeMod(SDataType *pType, STypeMod mod);
uint8_t getScaleFromTypeMod(int32_t type, STypeMod mod);
// TODO fix me!! for compatibility issue, save precision in scale in bytes, move it to somewhere else
void fillBytesForDecimalType(int32_t *pBytes, int32_t type, uint8_t precision, uint8_t scale);
void extractDecimalTypeInfoFromBytes(int32_t *pBytes, uint8_t *precision, uint8_t *scale);
int32_t calcTypeBytesFromSchemaBytes(int32_t type, int32_t schemaBytes, bool isStmt);
int32_t calcSchemaBytesFromTypeBytes(int32_t type, int32_t varTypeBytes, bool isStmt);
#ifdef __cplusplus
}
#endif

143
include/libs/decimal/decimal.h Normal file
View File

@ -0,0 +1,143 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_DECIMAL_H_
#define _TD_DECIMAL_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tdef.h"
#include "ttypes.h"
typedef struct SValue SValue;
typedef void DecimalType;
typedef struct Decimal64 {
DecimalWord words[1]; // do not touch it directly, use DECIMAL64_GET_VALUE MACRO
} Decimal64;
#define DECIMAL64_GET_VALUE(pDec) (int64_t)((pDec)->words[0])
#define DECIMAL64_SET_VALUE(pDec, val) (*(int64_t*)((pDec)->words)) = (int64_t)(val)
#define DECIMAL64_CLONE(pDst, pFrom) ((Decimal64*)(pDst))->words[0] = ((Decimal64*)(pFrom))->words[0]
static const Decimal64 decimal64Zero = {0};
static const Decimal64 decimal64Two = {2};
static const Decimal64 decimal64Min = {(uint64_t)-999999999999999999LL};
static const Decimal64 decimal64Max = {(uint64_t)999999999999999999LL};
#define DECIMAL64_ZERO decimal64Zero
#define DECIMAL64_MAX decimal64Max
#define DECIMAL64_MIN decimal64Min
typedef struct Decimal128 {
DecimalWord words[2]; // do not touch it directly, use DECIMAL128_HIGH_WORD/DECIMAL128_LOW_WORD
} Decimal128;
#define Decimal Decimal128
#define decimalFromStr decimal128FromStr
#define makeDecimal makeDecimal128
#define DEFINE_DECIMAL128(lo, hi) {lo, hi}
static const Decimal128 decimal128Zero = DEFINE_DECIMAL128(0, 0);
static const Decimal128 decimal128Two = DEFINE_DECIMAL128(2, 0);
static const Decimal128 decimal128Max = DEFINE_DECIMAL128(687399551400673280ULL - 1, 5421010862427522170LL);
static const Decimal128 decimal128Min = DEFINE_DECIMAL128(17759344522308878337ULL, 13025733211282029445ULL);
#define DECIMAL128_LOW_WORD(pDec) (uint64_t)((pDec)->words[0])
#define DECIMAL128_SET_LOW_WORD(pDec, val) (pDec)->words[0] = val
#define DECIMAL128_HIGH_WORD(pDec) (int64_t)((pDec)->words[1])
#define DECIMAL128_SET_HIGH_WORD(pDec, val) *(int64_t*)((pDec)->words + 1) = val
#define DECIMAL128_ZERO decimal128Zero
#define DECIMAL128_MAX decimal128Max
#define DECIMAL128_MIN decimal128Min
#define DECIMAL128_CLONE(pDst, pFrom) makeDecimal128(pDst, DECIMAL128_HIGH_WORD(pFrom), DECIMAL128_LOW_WORD(pFrom))
typedef struct SDecimalCompareCtx {
const void* pData;
int8_t type;
STypeMod typeMod;
} SDecimalCompareCtx;
void makeDecimal64(Decimal64* pDec64, int64_t w);
void makeDecimal128(Decimal128* pDec128, int64_t hi, uint64_t low);
void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale);
int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* result);
int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale,
Decimal128* result);
int32_t decimal64ToDataVal(const Decimal64* dec, SValue* pVal);
int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal);
int32_t decimalToStr(const DecimalType* pDec, int8_t type, int8_t precision, int8_t scale, char* pBuf, int32_t bufLen);
int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOperatorType opType, SDataType* pOutType);
bool decimal64Compare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight);
bool decimalCompare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight);
int32_t decimalOp(EOperatorType op, const SDataType* pLeftT, const SDataType* pRightT, const SDataType* pOutT,
const void* pLeftData, const void* pRightData, void* pOutputData);
int32_t convertToDecimal(const void* pData, const SDataType* pInputType, void* pOut, const SDataType* pOutType);
bool decimal128AddCheckOverflow(const Decimal128* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void encodeDecimal(const DecimalType* pDec, int8_t type, void* pBuf);
void decodeDecimal(const void* pBuf, int8_t type, DecimalType* pDec);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal64);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal128);
// word num use DECIMAL_WORD_NUM(Decimal64) or DECIMAL_WORD_NUM(Decimal128)
typedef struct SDecimalOps {
void (*negate)(DecimalType* pWord);
void (*abs)(DecimalType* pWord);
void (*add)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void (*subtract)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void (*multiply)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void (*divide)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum, DecimalType* pRemainder);
void (*mod)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
bool (*lt)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
bool (*gt)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
bool (*eq)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
int32_t (*toStr)(const DecimalType* pInt, uint8_t scale, char* pBuf, int32_t bufLen);
} SDecimalOps;
// all these ops only used for operations on decimal types with same scale
const SDecimalOps* getDecimalOps(int8_t dataType);
#if 0
__int128 decimal128ToInt128(const Decimal128* pDec);
#endif
int32_t TEST_decimal64From_int64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, int64_t v);
int32_t TEST_decimal64From_uint64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, uint64_t v);
int32_t TEST_decimal64From_double(Decimal64* pDec, uint8_t prec, uint8_t scale, double v);
double TEST_decimal64ToDouble(Decimal64* pDec, uint8_t prec, uint8_t scale);
int32_t TEST_decimal64FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale, Decimal64* pOutput,
uint8_t outputPrec, uint8_t outputScale);
int32_t TEST_decimal64FromDecimal128(const Decimal128* pInput, uint8_t prec, uint8_t scale, Decimal64* pOutput,
uint8_t outputPrec, uint8_t outputScale);
int32_t TEST_decimal128From_int64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, int64_t v);
int32_t TEST_decimal128From_uint64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, uint64_t v);
int32_t TEST_decimal128From_double(Decimal128* pDec, uint8_t prec, uint8_t scale, double v);
double TEST_decimal128ToDouble(Decimal128* pDec, uint8_t prec, uint8_t scale);
int32_t TEST_decimal128FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale,
Decimal128* pOutput, uint8_t outputPrec, uint8_t outputScale);
int32_t TEST_decimal128FromDecimal128(const Decimal128* pDec, uint8_t prec, uint8_t scale, Decimal128* pOutput,
uint8_t outputPrec, uint8_t outputScale);
#ifdef __cplusplus
}
#endif
#endif /*_TD_DECIMAL_H_*/

4
include/libs/executor/executor.h
View File

@ -53,6 +53,7 @@ typedef struct {
int32_t numOfVgroups;
void* sContext; // SSnapContext*
void* pStateBackend;
void* pOtherBackend;
int8_t fillHistory;
STimeWindow winRange;
@ -219,7 +220,6 @@ const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);
void* qExtractReaderFromStreamScanner(void* scanner);
void qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo);
@ -229,7 +229,7 @@ int32_t qStreamRecoverFinish(qTaskInfo_t tinfo);
bool qStreamScanhistoryFinished(qTaskInfo_t tinfo);
int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo);
void qResetTaskInfoCode(qTaskInfo_t tinfo);
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow);
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow, TSKEY* pRecInteral);
int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo);
int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo);

87
include/libs/executor/storageapi.h
View File

@ -84,6 +84,7 @@ typedef struct SMetaEntry {
uint8_t* pBuf;
SColCmprWrapper colCmpr; // col compress alg
SExtSchema* pExtSchemas;
} SMetaEntry;
typedef struct SMetaReader {
@ -126,6 +127,7 @@ typedef struct SRowBuffPos {
bool beUsed;
bool needFree;
bool beUpdated;
bool invalid;
} SRowBuffPos;
// tq
@ -136,6 +138,11 @@ typedef struct SMetaTableInfo {
char tbName[TSDB_TABLE_NAME_LEN];
} SMetaTableInfo;
static FORCE_INLINE void destroyMetaTableInfo(SMetaTableInfo* mtInfo){
if (mtInfo == NULL) return;
tDeleteSchemaWrapper(mtInfo->schema);
}
typedef struct SSnapContext {
struct SMeta* pMeta;
int64_t snapVersion;
@ -202,6 +209,8 @@ typedef struct TsdReader {
int32_t (*fileSetGetEntryField)(struct SFileSetReader *, const char *, void *);
void (*fileSetReaderClose)(struct SFileSetReader **);
int32_t (*getProgress)(const void* pReader, void** pBuf, uint64_t* pLen);
int32_t (*setProgress)(void *pReader, const void *pBuf, uint64_t len);
} TsdReader;
typedef struct SStoreCacheReader {
@ -320,7 +329,7 @@ typedef struct SUpdateInfo {
TSKEY minTS;
SScalableBf* pCloseWinSBF;
SHashObj* pMap;
uint64_t maxDataVersion;
int64_t maxDataVersion;
int8_t pkColType;
int32_t pkColLen;
char* pKeyBuff;
@ -330,6 +339,27 @@ typedef struct SUpdateInfo {
__compar_fn_t comparePkCol;
} SUpdateInfo;
typedef struct SRecDataInfo {
STimeWindow calWin;
uint64_t tableUid;
int64_t dataVersion;
EStreamType mode;
char pPkColData[];
} SRecDataInfo;
typedef struct SScanRange {
STimeWindow win;
STimeWindow calWin;
SSHashObj* pGroupIds;
SSHashObj* pUIds;
} SScanRange;
typedef struct SResultWindowInfo {
SRowBuffPos* pStatePos;
SSessionKey sessionWin;
bool isOutput;
} SResultWindowInfo;
typedef struct {
void* iter; // rocksdb_iterator_t* iter;
void* snapshot; // rocksdb_snapshot_t* snapshot;
@ -344,23 +374,44 @@ typedef struct {
int64_t minGpId;
} SStreamStateCur;
typedef struct STableTsDataState {
SSHashObj* pTableTsDataMap;
__compar_fn_t comparePkColFn;
void* pPkValBuff;
int32_t pkValLen;
SStreamState* pState;
int32_t curRecId;
void* pStreamTaskState;
SArray* pScanRanges;
SRecDataInfo* pRecValueBuff;
int32_t recValueLen;
SStreamStateCur* pRecCur;
int32_t cfgIndex;
void* pBatch;
int32_t batchBufflen;
void* pBatchBuff;
} STableTsDataState;
typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache,
int32_t* pWinCode);
int32_t (*streamStateDeleteParName)(SStreamState* pState, int64_t groupId);
void (*streamStateSetParNameInvalid)(SStreamState* pState);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
void (*streamStateClearBuff)(SStreamState* pState, void* pVal);
void (*streamStateFreeVal)(void* val);
int32_t (*streamStateGetPrev)(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateGetAllPrev)(SStreamState* pState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key, bool hasLimit, bool* pIsLast);
bool (*streamStateCheckSessionState)(SStreamState* pState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
void (*streamStateDel)(SStreamState* pState, const SWinKey* key);
void (*streamStateDelByGroupId)(SStreamState* pState, uint64_t groupId);
@ -368,6 +419,8 @@ typedef struct SStateStore {
void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex);
void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t (*streamStateGetNumber)(SStreamState* pState);
int32_t (*streamStateDeleteInfo)(SStreamState* pState, void* pKey, int32_t keyLen);
int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
@ -392,7 +445,10 @@ typedef struct SStateStore {
int32_t (*streamStateFillGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
void (*streamStateClearExpiredState)(SStreamState* pState);
void (*streamStateClearExpiredState)(SStreamState* pState, int32_t numOfKeep, TSKEY minTs);
void (*streamStateClearExpiredSessionState)(SStreamState* pState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup);
int32_t (*streamStateSetRecFlag)(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t (*streamStateGetRecFlag)(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t* pMode);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
@ -409,6 +465,9 @@ typedef struct SStateStore {
int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionSaveToDisk)(STableTsDataState* pTblState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t (*streamStateFlushReaminInfoToDisk)(STableTsDataState* pTblState);
int32_t (*streamStateSessionDeleteAll)(SStreamState* pState);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen, int32_t* pWinCode);
@ -454,12 +513,32 @@ typedef struct SStateStore {
bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
SStreamState* (*streamStateRecalatedOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState);
void (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
int32_t (*streamStateGetAndSetTsData)(STableTsDataState* pState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t (*streamStateTsDataCommit)(STableTsDataState* pState);
int32_t (*streamStateInitTsDataState)(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState, void* pOtherState);
void (*streamStateDestroyTsDataState)(STableTsDataState* pTsDataState);
int32_t (*streamStateRecoverTsData)(STableTsDataState* pTsDataState);
int32_t (*streamStateReloadTsDataState)(STableTsDataState* pTsDataState);
int32_t (*streamStateMergeAndSaveScanRange)(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId,
SRecDataInfo* pRecData, int32_t len);
int32_t (*streamStateMergeAllScanRange)(STableTsDataState* pTsDataState);
int32_t (*streamStatePopScanRange)(STableTsDataState* pTsDataState, SScanRange* pRange);
SStreamStateCur* (*streamStateGetLastStateCur)(SStreamState* pState);
void (*streamStateLastStateCurNext)(SStreamStateCur* pCur);
int32_t (*streamStateNLastStateGetKVByCur)(SStreamStateCur* pCur, int32_t num, SArray* pRes);
SStreamStateCur* (*streamStateGetLastSessionStateCur)(SStreamState* pState);
void (*streamStateLastSessionStateCurNext)(SStreamStateCur* pCur);
int32_t (*streamStateNLastSessionStateGetKVByCur)(SStreamStateCur* pCur, int32_t num, SArray* pRes);
} SStateStore;
typedef struct SStorageAPI {

20
include/libs/function/function.h
View File

@ -20,10 +20,10 @@
extern "C" {
#endif
#include "functionResInfo.h"
#include "tcommon.h"
#include "tsimplehash.h"
#include "tvariant.h"
#include "functionResInfo.h"
struct SqlFunctionCtx;
struct SResultRowEntryInfo;
@ -44,7 +44,8 @@ typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef int32_t (*FExecCombine)(struct SqlFunctionCtx *pDestCtx, struct SqlFunctionCtx *pSourceCtx);
typedef int32_t (*FExecDecode)(struct SqlFunctionCtx *pCtx, const char *buf, struct SResultRowEntryInfo *pResultCellInfo, int32_t version);
typedef int32_t (*FExecDecode)(struct SqlFunctionCtx *pCtx, const char *buf,
struct SResultRowEntryInfo *pResultCellInfo, int32_t version);
typedef int32_t (*processFuncByRow)(SArray *pCtx); // array of SqlFunctionCtx
typedef struct SScalarFuncExecFuncs {
@ -168,11 +169,14 @@ typedef struct STdbState {
void *pParNameDb;
void *pParTagDb;
void *txn;
int8_t recalc;
} STdbState;
typedef struct SResultRowStore {
int32_t (*resultRowPut)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t (*resultRowGet)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t (*resultRowPut)(struct SExprSupp *pSup, const char *inBuf, size_t inBufSize, char **outBuf,
size_t *outBufSize);
int32_t (*resultRowGet)(struct SExprSupp *pSup, const char *inBuf, size_t inBufSize, char **outBuf,
size_t *outBufSize);
} SResultRowStore;
struct SStreamState {
@ -187,6 +191,7 @@ struct SStreamState {
int32_t tsIndex;
SResultRowStore pResultRowStore;
struct SExprSupp *pExprSupp;
char pTaskIdStr[65];
};
typedef struct SFunctionStateStore {
@ -265,6 +270,7 @@ typedef struct SqlFunctionCtx {
bool bInputFinished;
bool hasWindowOrGroup; // denote that the function is used with time window or group
bool needCleanup; // denote that the function need to be cleaned up
int32_t inputType; // save the fuction input type funcs like finalize
} SqlFunctionCtx;
typedef struct tExprNode {
@ -291,12 +297,14 @@ struct SScalarParam {
SColumnInfoData *columnData;
SHashObj *pHashFilter;
SHashObj *pHashFilterOthers;
int32_t hashValueType;
int32_t filterValueType;
void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value
int32_t numOfRows;
int32_t numOfQualified; // number of qualified elements in the final results
timezone_t tz;
void *charsetCxt;
SArray *pFilterArr; // for types that can't filter with hash
STypeMod filterValueTypeMod;
};
static inline void setTzCharset(SScalarParam *param, timezone_t tz, void *charsetCxt) {
@ -315,7 +323,7 @@ typedef struct SPoint {
} SPoint;
void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
int32_t inputType);
int32_t inputType, STypeMod inputTypeMod);
#define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25
#define LEASTSQUARES_BUFF_LENGTH 128

3
include/libs/function/functionMgt.h
View File

@ -159,6 +159,8 @@ typedef enum EFunctionType {
FUNCTION_TYPE_FORECAST_ROWTS,
FUNCTION_TYPE_COLS,
FUNCTION_TYPE_IROWTS_ORIGIN,
FUNCTION_TYPE_GROUP_ID,
FUNCTION_TYPE_IS_WINDOW_FILLED,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,
@ -300,6 +302,7 @@ bool fmIsElapsedFunc(int32_t funcId);
bool fmIsDBUsageFunc(int32_t funcId);
bool fmIsRowTsOriginFunc(int32_t funcId);
bool fmIsSelectColsFunc(int32_t funcId);
bool fmIsGroupIdFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);

2
include/libs/nodes/cmdnodes.h
View File

@ -57,6 +57,7 @@ extern "C" {
#define BIT_FLAG_MASK(n) (1 << n)
#define BIT_FLAG_SET_MASK(val, mask) ((val) |= (mask))
#define BIT_FLAG_UNSET_MASK(val, mask) ((val) &= ~(mask))
#define BIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
#define PRIVILEGE_TYPE_ALL BIT_FLAG_MASK(0)
@ -565,6 +566,7 @@ typedef struct SStreamOptions {
SNode* pDelay;
SNode* pWatermark;
SNode* pDeleteMark;
SNode* pRecInterval;
int8_t fillHistory;
int8_t ignoreExpired;
int8_t ignoreUpdate;

13
include/libs/nodes/plannodes.h
View File

@ -316,6 +316,12 @@ typedef enum EWindowAlgorithm {
SESSION_ALGO_STREAM_SINGLE,
SESSION_ALGO_MERGE,
INTERVAL_ALGO_STREAM_MID,
INTERVAL_ALGO_STREAM_CONTINUE_SINGLE,
INTERVAL_ALGO_STREAM_CONTINUE_FINAL,
INTERVAL_ALGO_STREAM_CONTINUE_SEMI,
SESSION_ALGO_STREAM_CONTINUE_SINGLE,
SESSION_ALGO_STREAM_CONTINUE_FINAL,
SESSION_ALGO_STREAM_CONTINUE_SEMI,
} EWindowAlgorithm;
typedef struct SWindowLogicNode {
@ -347,6 +353,7 @@ typedef struct SWindowLogicNode {
SNodeList* pTsmaSubplans;
SNode* pAnomalyExpr;
char anomalyOpt[TSDB_ANALYTIC_ALGO_OPTION_LEN];
int64_t recalculateInterval;
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -508,6 +515,11 @@ typedef struct STableScanPhysiNode {
bool needCountEmptyTable;
bool paraTablesSort;
bool smallDataTsSort;
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;
@ -683,6 +695,7 @@ typedef struct SWindowPhysiNode {
int8_t igExpired;
int8_t destHasPrimaryKey;
bool mergeDataBlock;
int64_t recalculateInterval;
} SWindowPhysiNode;
typedef struct SIntervalPhysiNode {

14
include/libs/nodes/querynodes.h
View File

@ -25,6 +25,7 @@ extern "C" {
#include "tmsg.h"
#include "tsimplehash.h"
#include "tvariant.h"
#include "ttypes.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) \
@ -45,13 +46,6 @@ typedef struct SRawExprNode {
bool isPseudoColumn;
} SRawExprNode;
typedef struct SDataType {
uint8_t type;
uint8_t precision;
uint8_t scale;
int32_t bytes;
} SDataType;
typedef struct SExprNode {
ENodeType type;
SDataType resType;
@ -76,7 +70,8 @@ typedef enum EColumnType {
COLUMN_TYPE_WINDOW_START,
COLUMN_TYPE_WINDOW_END,
COLUMN_TYPE_WINDOW_DURATION,
COLUMN_TYPE_GROUP_KEY
COLUMN_TYPE_GROUP_KEY,
COLUMN_TYPE_IS_WINDOW_FILLED,
} EColumnType;
typedef struct SColumnNode {
@ -203,6 +198,8 @@ typedef struct SFunctionNode {
bool dual; // whether select stmt without from stmt, true for without.
timezone_t tz;
void *charsetCxt;
const struct SFunctionNode* pSrcFuncRef;
SDataType srcFuncInputType;
} SFunctionNode;
typedef struct STableNode {
@ -655,6 +652,7 @@ typedef struct SQuery {
SArray* pDbList;
SArray* pPlaceholderValues;
SNode* pPrepareRoot;
SExtSchema* pResExtSchema;
} SQuery;
void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);

2
include/libs/parser/parser.h
View File

@ -189,7 +189,7 @@ int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen);
int32_t checkSchema(SSchema* pColSchema, SSchemaExt* pColExtSchema, int8_t* fields, char* errstr, int32_t errstrLen);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);

6
include/libs/planner/planner.h
View File

@ -47,6 +47,12 @@ typedef struct SPlanContext {
bool destHasPrimaryKey;
bool sourceHasPrimaryKey;
void* timezone;
int64_t recalculateInterval;
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} SPlanContext;
// Create the physical plan for the query, according to the AST.

6
include/libs/scalar/scalar.h
View File

@ -41,11 +41,12 @@ pDst need to freed in caller
int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst);
int32_t scalarGetOperatorParamNum(EOperatorType type);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, STypeMod typeMod, int8_t processType);
int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod, int32_t startIndex, int32_t numOfRows);
STypeMod getConvertTypeMod(int32_t type, const SColumnInfo *pCol1, const SColumnInfo *pCol2);
/* Math functions */
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@ -122,6 +123,7 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t isWinFilledFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qPseudoTagFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

42
include/libs/stream/streamState.h
View File

@ -32,6 +32,7 @@ extern "C" {
#include "storageapi.h"
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
SStreamState* streamStateRecalatedOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
void streamStateCommit(SStreamState* pState);
@ -43,7 +44,7 @@ int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVa
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool streamStateCheck(SStreamState* pState, const SWinKey* key);
bool streamStateCheck(SStreamState* pState, const SWinKey* key, bool hasLimit, bool* pIsLast);
int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal);
void streamStateDel(SStreamState* pState, const SWinKey* key);
void streamStateDelByGroupId(SStreamState* pState, uint64_t groupId);
@ -51,8 +52,11 @@ void streamStateClear(SStreamState* pState);
void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex);
void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t streamStateGetNumber(SStreamState* pState);
int32_t streamStateDeleteInfo(SStreamState* pState, void* pKey, int32_t keyLen);
int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateGetAllPrev(SStreamState* pState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
// session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen,
@ -67,6 +71,9 @@ int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey*
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t streamStateSessionSaveToDisk(STableTsDataState* pTblState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t streamStateFlushReaminInfoToDisk(STableTsDataState* pTblState);
int32_t streamStateSessionDeleteAll(SStreamState* pState);
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key);
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
@ -89,7 +96,7 @@ int32_t streamStateFillGetNext(SStreamState* pState, const SWinKey* pKey, SWinKe
int32_t streamStateFillGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
void streamStateClearBuff(SStreamState* pState, void* pVal);
@ -98,7 +105,8 @@ void streamStateFreeVal(void* val);
// count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);
@ -111,7 +119,10 @@ int32_t streamStateFillGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, con
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
// twa
void streamStateClearExpiredState(SStreamState* pState);
void streamStateClearExpiredState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs);
void streamStateClearExpiredSessionState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup);
int32_t streamStateSetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t streamStateGetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t* pMode);
void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
@ -119,6 +130,7 @@ void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode);
int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId);
void streamStateSetParNameInvalid(SStreamState* pState);
// group id
int32_t streamStateGroupPut(SStreamState* pState, int64_t groupId, void* value, int32_t vLen);
@ -126,6 +138,28 @@ SStreamStateCur* streamStateGroupGetCur(SStreamState* pState);
void streamStateGroupCurNext(SStreamStateCur* pCur);
int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** pVal, int32_t* pVLen);
// ts data
int32_t streamStateGetAndSetTsData(STableTsDataState* pState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t streamStateTsDataCommit(STableTsDataState* pState);
int32_t streamStateInitTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState, void* pOtherState);
void streamStateDestroyTsDataState(STableTsDataState* pTsDataState);
int32_t streamStateRecoverTsData(STableTsDataState* pTsDataState);
int32_t streamStateReloadTsDataState(STableTsDataState* pTsDataState);
int32_t streamStateMergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId,
SRecDataInfo* pRecData, int32_t len);
int32_t streamStateMergeAllScanRange(STableTsDataState* pTsDataState);
int32_t streamStatePopScanRange(STableTsDataState* pTsDataState, SScanRange* pRange);
// continuous
bool streamStateCheckSessionState(SStreamState* pState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
SStreamStateCur* streamStateGetLastStateCur(SStreamState* pState);
void streamStateLastStateCurNext(SStreamStateCur* pCur);
int32_t streamStateNLastStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
SStreamStateCur* streamStateGetLastSessionStateCur(SStreamState* pState);
void streamStateLastSessionStateCurNext(SStreamStateCur* pCur);
int32_t streamStateNLastSessionStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
void streamStateCopyBackend(SStreamState* src, SStreamState* dst);

32
include/libs/stream/tstream.h
View File

@ -318,9 +318,10 @@ typedef struct SSTaskBasicInfo {
int32_t selfChildId;
int32_t trigger;
int8_t taskLevel;
int8_t fillHistory; // is fill history task or not
int8_t fillHistory; // enum , 1. is fill history task or not 2. recal
int64_t delaySchedParam; // in msec
int64_t watermark; // extracted from operators
int8_t hasAggTasks; // has agg tasks in current stream
SInterval interval;
} SSTaskBasicInfo;
@ -461,7 +462,6 @@ struct SStreamTask {
STaskExecStatisInfo execInfo;
TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ
SMsgCb* pMsgCb; // msg handle
SStreamState* pState; // state backend
SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo;
SNotifyInfo notifyInfo;
@ -470,12 +470,18 @@ struct SStreamTask {
// the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo;
int32_t transferStateAlignCnt;
int32_t recalculateAlignCnt;
struct SStreamMeta* pMeta;
SSHashObj* pNameMap;
void* pBackend;
int8_t subtableWithoutMd5; // only for tsma stream tasks
char reserve[256];
char* backendPath;
void* pBackend;
void* pRecalBackend;
SStreamState* pRecalState;
SStreamState* pState; // state backend
};
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
@ -558,16 +564,23 @@ typedef struct STaskUpdateEntry {
int32_t transId;
} STaskUpdateEntry;
typedef enum {
STREAM_NORMAL_TASK = 0,
STREAM_HISTORY_TASK = 1,
STREAM_RECALCUL_TASK = 2,
} EStreamTaskType;
typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int32_t trigger,
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, EStreamTaskType type, int32_t trigger,
int64_t triggerParam, SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5,
SStreamTask** pTask);
int8_t hasAggTasks, SStreamTask** pTask);
void tFreeStreamTask(void* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
void streamFreeTaskState(SStreamTask* pTask, int8_t remove);
int32_t streamCreateAddRecalculateEndBlock(SStreamTask* pTask);
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo);
int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId);
@ -640,12 +653,6 @@ typedef struct STaskStatusEntry {
STaskNotifyEventStat notifyEventStat;
} STaskStatusEntry;
//typedef struct SNodeUpdateInfo {
// int32_t nodeId;
// SEpSet prevEp;
// SEpSet newEp;
//} SNodeUpdateInfo;
typedef struct SStreamTaskState {
ETaskStatus state;
char* name;
@ -753,9 +760,10 @@ int32_t streamTaskReloadState(SStreamTask* pTask);
void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId);
void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
void streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key);
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key, uint8_t recalated);
bool streamTaskIsSinkTask(const SStreamTask* pTask);
void streamTaskSetRemoveBackendFiles(SStreamTask* pTask);
int8_t streamTaskShouldRecalated(SStreamTask* pTask);
void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);

40
include/libs/stream/tstreamFileState.h
View File

@ -56,12 +56,14 @@ void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuf
int32_t addRowBuffIfNotExist(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t createRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
void deleteRowBuffByGroupId(SStreamFileState* pFileState, uint64_t groupId);
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal);
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen);
bool hasRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, bool hasLimit, bool* pIsLast);
int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
@ -80,6 +82,7 @@ bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap);
TSKEY getFlushMark(SStreamFileState* pFileState);
SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState);
int32_t getRowStateRowSize(SStreamFileState* pFileState);
void freeArrayPtr(void* ptr);
// session window
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
@ -147,12 +150,43 @@ SSHashObj* getGroupIdCache(SStreamFileState* pFileState);
int fillStateKeyCompare(const void* pWin1, const void* pDatas, int pos);
int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey);
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey, bool* pIsEnd);
int32_t getRowStateAllPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
//twa
void setFillInfo(SStreamFileState* pFileState);
void clearExpiredState(SStreamFileState* pFileState);
void clearExpiredState(SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs);
int32_t addArrayBuffIfNotExist(SSHashObj* pSearchBuff, uint64_t groupId, SArray** ppResStates);
int32_t recoverHashSortBuff(SStreamFileState* pFileState, SArray* pWinStates, uint64_t groupId);
int32_t getAndSetTsData(STableTsDataState* pTsDataState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t doTsDataCommit(STableTsDataState* pTsDataState);
int32_t doRangeDataCommit(STableTsDataState* pTsDataState);
int32_t initTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState,
void* pOtherState);
void destroyTsDataState(STableTsDataState* pTsDataState);
int32_t recoverTsData(STableTsDataState* pTsDataState);
int32_t mergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId, SRecDataInfo* pRecData,
int32_t len);
int32_t mergeAllScanRange(STableTsDataState* pTsDataState);
int32_t popScanRange(STableTsDataState* pTsDataState, SScanRange* pRange);
// continuous
typedef void* (*getStateBuffFn)(SStreamFileState* pFileState);
SStreamStateCur* getLastStateCur(SStreamFileState* pFileState, getStateBuffFn fn);
void moveLastStateCurNext(SStreamStateCur* pCur, getStateBuffFn fn);
void moveOneStateCurNext(SStreamStateCur* pCur);
int32_t getNLastStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
int32_t getNLastSessionStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
int32_t reloadTsDataState(STableTsDataState* pTsDataState);
int32_t setStateRecFlag(SStreamFileState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t getStateRecFlag(SStreamFileState* pFileState, const void* pKey, int32_t keyLen, int32_t* pMode);
void clearExpiredSessionState(struct SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs,
SSHashObj* pFlushGroup);
bool hasSessionState(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
int32_t saveRecInfoToDisk(STableTsDataState* pTsDataState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t flushRemainRecInfoToDisk(STableTsDataState* pTsDataState);
#ifdef __cplusplus
}

2
include/util/taoserror.h
View File

@ -166,6 +166,8 @@ int32_t taosGetErrSize();
#define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A)
#define TSDB_CODE_TIME_ERROR TAOS_DEF_ERROR_CODE(0, 0x013B)
#define TSDB_CODE_INVALID_DISK_ID TAOS_DEF_ERROR_CODE(0, 0x013C)
#define TSDB_CODE_DECIMAL_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x013D)
#define TSDB_CODE_DIVISION_BY_ZERO TAOS_DEF_ERROR_CODE(0, 0x013E)
//client
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)

8
include/util/tcompare.h
View File

@ -60,11 +60,13 @@ int32_t setChkInBytes1(const void *pLeft, const void *pRight);
int32_t setChkInBytes2(const void *pLeft, const void *pRight);
int32_t setChkInBytes4(const void *pLeft, const void *pRight);
int32_t setChkInBytes8(const void *pLeft, const void *pRight);
int32_t setChkInDecimalHash(const void* pLeft, const void* pRight);
int32_t setChkNotInBytes1(const void *pLeft, const void *pRight);
int32_t setChkNotInBytes2(const void *pLeft, const void *pRight);
int32_t setChkNotInBytes4(const void *pLeft, const void *pRight);
int32_t setChkNotInBytes8(const void *pLeft, const void *pRight);
int32_t setChkNotInDecimalHash(const void* pLeft, const void* pRight);
int32_t compareChkInString(const void *pLeft, const void *pRight);
int32_t compareChkNotInString(const void *pLeft, const void *pRight);
@ -86,6 +88,12 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareLenBinaryVal(const void *pLeft, const void *pRight);
int32_t compareDecimal64(const void* pleft, const void* pright);
int32_t compareDecimal128(const void* pleft, const void* pright);
int32_t compareDecimal64SameScale(const void* pleft, const void* pright);
int32_t compareDecimal128SameScale(const void* pleft, const void* pright);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);

8
include/util/tcompression.h
View File

@ -198,6 +198,14 @@ int32_t tsCompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3
int32_t nBuf);
int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsDecompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsDecompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
/*************************************************************************
* STREAM COMPRESSION

23
include/util/tdef.h
View File

@ -47,7 +47,7 @@ extern "C" {
#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle
// Bytes for each type.
extern const int32_t TYPE_BYTES[21];
extern const int32_t TYPE_BYTES[22];
#define CHAR_BYTES sizeof(char)
#define SHORT_BYTES sizeof(int16_t)
@ -60,6 +60,9 @@ extern const int32_t TYPE_BYTES[21];
#define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(TdUcs4)
#define DECIMAL64_BYTES 8
#define DECIMAL128_BYTES 16
// NULL definition
#define TSDB_DATA_BOOL_NULL 0x02
#define TSDB_DATA_TINYINT_NULL 0x80
@ -694,6 +697,24 @@ typedef enum {
#define MIN_RESERVE_MEM_SIZE 1024 // MB
// Decimal
#define TSDB_DECIMAL64_MAX_PRECISION 18
#define TSDB_DECIMAL64_MAX_SCALE TSDB_DECIMAL64_MAX_PRECISION
#define TSDB_DECIMAL128_MAX_PRECISION 38
#define TSDB_DECIMAL128_MAX_SCALE TSDB_DECIMAL128_MAX_PRECISION
#define TSDB_DECIMAL_MIN_PRECISION 1
#define TSDB_DECIMAL_MAX_PRECISION TSDB_DECIMAL128_MAX_PRECISION
#define TSDB_DECIMAL_MIN_SCALE 0
#define TSDB_DECIMAL_MAX_SCALE TSDB_DECIMAL_MAX_PRECISION
#define GET_DEICMAL_MAX_PRECISION(type) (type) == TSDB_DATA_TYPE_DECIMAL64 ? TSDB_DECIMAL64_MAX_PRECISION : TSDB_DECIMAL_MAX_SCALE
typedef uint64_t DecimalWord;
#define DECIMAL_WORD_NUM(TYPE) (sizeof(TYPE) / sizeof(DecimalWord))
#define COMPILE_TIME_ASSERT(pred) switch(0) {case 0: case pred:;}
#ifdef __cplusplus
}
#endif

2
include/util/tutil.h
View File

@ -34,7 +34,7 @@ char **strsplit(char *src, const char *delim, int32_t *num);
char *strtolower(char *dst, const char *src);
char *strntolower(char *dst, const char *src, int32_t n);
char *strntolower_s(char *dst, const char *src, int32_t n);
int64_t strnatoi(char *num, int32_t len);
int64_t strnatoi(const char *num, int32_t len);
size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize);
size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize);

4
source/client/CMakeLists.txt
View File

@ -28,7 +28,7 @@ endif()
target_link_libraries(
${TAOS_LIB}
INTERFACE api
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ${TAOSD_MODULE}
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ${TAOSD_MODULE} decimal
)
if(TD_WINDOWS)
@ -67,7 +67,7 @@ target_include_directories(
target_link_libraries(
${TAOS_LIB_STATIC}
INTERFACE api
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry decimal
)
if(${BUILD_TEST})

10
source/client/inc/clientInt.h
View File

@ -205,7 +205,7 @@ typedef struct SReqResultInfo {
SExecResult execRes;
const char* pRspMsg;
const char* pData;
TAOS_FIELD* fields; // todo, column names are not needed.
TAOS_FIELD_E* fields; // todo, column names are not needed.
TAOS_FIELD* userFields; // the fields info that return to user
uint32_t numOfCols;
int32_t* length;
@ -313,11 +313,11 @@ typedef struct SSyncQueryParam {
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
void doSetOneRowPtr(SReqResultInfo* pResultInfo);
void doSetOneRowPtr(SReqResultInfo* pResultInfo, bool isStmt);
void setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4);
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool isStmt);
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4, bool isStmt);
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema, bool isStmt);
void doFreeReqResultInfo(SReqResultInfo* pResInfo);
int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
void syncCatalogFn(SMetaData* pResult, void* param, int32_t code);

2
source/client/inc/clientStmt.h
View File

@ -48,7 +48,7 @@ typedef struct SStmtTableCache {
} SStmtTableCache;
typedef struct SStmtQueryResInfo {
TAOS_FIELD *fields;
TAOS_FIELD_E *fields;
TAOS_FIELD *userFields;
uint32_t numOfCols;
int32_t precision;

98
source/client/src/clientImpl.c
View File

@ -28,6 +28,8 @@
#include "tref.h"
#include "tsched.h"
#include "tversion.h"
#include "decimal.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo);
@ -313,7 +315,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
code = qParseSql(&cxt, pQuery);
if (TSDB_CODE_SUCCESS == code) {
if ((*pQuery)->haveResultSet) {
code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols, (*pQuery)->pResExtSchema, pRequest->isStmtBind);
setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision);
}
}
@ -335,7 +337,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int8_t biMode = atomic_load_8(&pRequest->pTscObj->biMode);
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode, pRequest->pTscObj->optionInfo.charsetCxt);
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4, pRequest->isStmtBind);
}
return code;
@ -373,7 +375,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp,
atomic_load_8(&pRequest->pTscObj->biMode), pRequest->pTscObj->optionInfo.charsetCxt);
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4, pRequest->isStmtBind);
}
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@ -515,7 +517,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
return qCreateQueryPlan(&cxt, pPlan, pNodeList);
}
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) {
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema, bool isStmt) {
if (pResInfo == NULL || pSchema == NULL || numOfCols <= 0) {
tscError("invalid paras, pResInfo == NULL || pSchema == NULL || numOfCols <= 0");
return TSDB_CODE_INVALID_PARA;
@ -528,7 +530,7 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32
if (pResInfo->userFields != NULL) {
taosMemoryFree(pResInfo->userFields);
}
pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD_E));
if (NULL == pResInfo->fields) return terrno;
pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
if (NULL == pResInfo->userFields) {
@ -541,17 +543,14 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32
}
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
pResInfo->fields[i].bytes = pSchema[i].bytes;
pResInfo->fields[i].type = pSchema[i].type;
pResInfo->userFields[i].bytes = pSchema[i].bytes;
pResInfo->userFields[i].type = pSchema[i].type;
if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY ||
pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
// userFields must convert to type bytes, no matter isStmt or not
pResInfo->userFields[i].bytes = calcTypeBytesFromSchemaBytes(pSchema[i].type, pSchema[i].bytes, false);
pResInfo->fields[i].bytes = calcTypeBytesFromSchemaBytes(pSchema[i].type, pSchema[i].bytes, isStmt);
if (IS_DECIMAL_TYPE(pSchema[i].type) && pExtSchema) {
decimalFromTypeMod(pExtSchema[i].typeMod, &pResInfo->fields[i].precision, &pResInfo->fields[i].scale);
}
tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name));
@ -1942,15 +1941,15 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
// }
void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
void doSetOneRowPtr(SReqResultInfo* pResultInfo, bool isStmt) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
SResultColumn* pCol = &pResultInfo->pCol[i];
int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes;
int32_t schemaBytes = calcSchemaBytesFromTypeBytes(type, pResultInfo->fields[i].bytes, isStmt);
if (IS_VAR_DATA_TYPE(type)) {
if (!IS_VAR_NULL_TYPE(type, bytes) && pCol->offset[pResultInfo->current] != -1) {
if (!IS_VAR_NULL_TYPE(type, schemaBytes) && pCol->offset[pResultInfo->current] != -1) {
char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
pResultInfo->length[i] = varDataLen(pStart);
@ -1961,8 +1960,8 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
}
} else {
if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) {
pResultInfo->row[i] = pResultInfo->pCol[i].pData + bytes * pResultInfo->current;
pResultInfo->length[i] = bytes;
pResultInfo->row[i] = pResultInfo->pCol[i].pData + schemaBytes * pResultInfo->current;
pResultInfo->length[i] = schemaBytes;
} else {
pResultInfo->row[i] = NULL;
pResultInfo->length[i] = 0;
@ -1994,7 +1993,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
pRequest->code =
setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4);
setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4, pRequest->isStmtBind);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
return NULL;
@ -2013,7 +2012,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
if (setupOneRowPtr) {
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, pRequest->isStmtBind);
pResultInfo->current += 1;
}
@ -2060,7 +2059,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
return NULL;
} else {
if (setupOneRowPtr) {
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, pRequest->isStmtBind);
pResultInfo->current += 1;
}
@ -2087,14 +2086,14 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
return TSDB_CODE_SUCCESS;
}
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength, bool isStmt) {
int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, C2M, pResultInfo->charsetCxt);
if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR;
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes;
int32_t schemaBytes = calcSchemaBytesFromTypeBytes(pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, isStmt);
if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
@ -2111,11 +2110,11 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
char* pStart = pCol->offset[j] + pCol->pData;
int32_t len = taosUcs4ToMbsEx((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p), conv);
if (len < 0 || len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) {
if (len < 0 || len > schemaBytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) {
tscError(
"doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + "
"colLength[i]):%p",
len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
len, schemaBytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -2134,6 +2133,36 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
return TSDB_CODE_SUCCESS;
}
static int32_t convertDecimalType(SReqResultInfo* pResultInfo) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
TAOS_FIELD_E* pField = pResultInfo->fields + i;
int32_t type = pField->type;
int32_t bufLen = 0;
char* p = NULL;
if (!IS_DECIMAL_TYPE(type) || !pResultInfo->pCol[i].pData) {
continue;
} else {
bufLen = 64;
p = taosMemoryRealloc(pResultInfo->convertBuf[i], bufLen * pResultInfo->numOfRows);
pField->bytes = bufLen;
}
if (!p) return terrno;
pResultInfo->convertBuf[i] = p;
for (int32_t j = 0; j < pResultInfo->numOfRows; ++j) {
int32_t code = decimalToStr((DecimalWord*)(pResultInfo->pCol[i].pData + j * tDataTypes[type].bytes), type,
pField->precision, pField->scale, p, bufLen);
p += bufLen;
if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}
return 0;
}
int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) {
return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) * 3 + sizeof(uint64_t) +
numOfCols * (sizeof(int8_t) + sizeof(int32_t));
@ -2365,7 +2394,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) {
return TSDB_CODE_SUCCESS;
}
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4, bool isStmt) {
if (pResultInfo == NULL || pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL) {
tscError("setResultDataPtr paras error");
return TSDB_CODE_TSC_INTERNAL_ERROR;
@ -2423,6 +2452,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
int32_t bytes = *(int32_t*)p;
p += sizeof(int32_t);
if (IS_DECIMAL_TYPE(type) && pResultInfo->fields[i].precision == 0) {
extractDecimalTypeInfoFromBytes(&bytes, &pResultInfo->fields[i].precision, &pResultInfo->fields[i].scale);
}
}
int32_t* colLength = (int32_t*)p;
@ -2454,7 +2487,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
}
pResultInfo->pCol[i].pData = pStart;
pResultInfo->length[i] = pResultInfo->fields[i].bytes;
pResultInfo->length[i] = calcSchemaBytesFromTypeBytes(pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, isStmt);
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
pStart += colLength[i];
@ -2471,9 +2504,12 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV
if (convertUcs4) {
code = doConvertUCS4(pResultInfo, colLength);
code = doConvertUCS4(pResultInfo, colLength, isStmt);
}
#endif
if (TSDB_CODE_SUCCESS == code && convertUcs4) {
code = convertDecimalType(pResultInfo);
}
return code;
}
@ -2514,7 +2550,7 @@ void resetConnectDB(STscObj* pTscObj) {
(void)taosThreadMutexUnlock(&pTscObj->mutex);
}
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) {
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool isStmt) {
if (pResultInfo == NULL || pRsp == NULL) {
tscError("setQueryResultFromRsp paras is null");
return TSDB_CODE_TSC_INTERNAL_ERROR;
@ -2583,7 +2619,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
// TODO handle the compressed case
pResultInfo->totalRows += pResultInfo->numOfRows;
int32_t code = setResultDataPtr(pResultInfo, convertUcs4);
int32_t code = setResultDataPtr(pResultInfo, convertUcs4, isStmt);
return code;
}
@ -3046,7 +3082,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
}
pRequest->code =
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, pRequest->isStmtBind);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
tscError("req:0x%" PRIx64 ", fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),

22
source/client/src/clientMain.c
View File

@ -608,6 +608,14 @@ TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) {
return taosQueryImplWithReqid(taos, sql, false, reqid);
}
TAOS_FIELD_E *taos_fetch_fields_e(TAOS_RES *res) {
if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
}
SReqResultInfo* pResInfo = tscGetCurResInfo(res);
return pResInfo->fields;
}
TAOS_ROW taos_fetch_row(TAOS_RES *res) {
if (res == NULL) {
return NULL;
@ -639,7 +647,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
}
if (pResultInfo->current < pResultInfo->numOfRows) {
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, false);
pResultInfo->current += 1;
return pResultInfo->row;
} else {
@ -647,7 +655,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL;
}
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, false);
pResultInfo->current += 1;
return pResultInfo->row;
}
@ -760,6 +768,14 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
case TSDB_DATA_TYPE_BOOL:
len += tsnprintf(str + len, size - len, "%d", *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_DECIMAL: {
uint32_t decimalLen = strlen(row[i]);
uint32_t copyLen = TMIN(size - len - 1, decimalLen);
(void)memcpy(str + len, row[i], copyLen);
len += copyLen;
} break;
default:
break;
}
@ -1269,7 +1285,7 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta
}
if (pQuery->haveResultSet) {
code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols);
code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols, pQuery->pResExtSchema, pRequest->isStmtBind);
setResPrecision(&pRequest->body.resInfo, pQuery->precision);
}
}

4
source/client/src/clientMsgHandler.c
View File

@ -682,7 +682,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = buildShowVariablesRsp(rsp.variables, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, pRequest->isStmtBind);
}
if (code != 0) {
@ -837,7 +837,7 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, pRequest->isStmtBind);
}
if (code != 0) {

6
source/client/src/clientRawBlockWrite.c
View File

@ -516,7 +516,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(tvalue);
} else {
double val = 0;
GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64);
GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64, 0); // currently tag type can't be decimal, so pass 0 as typeMod
tvalue = cJSON_CreateNumber(val);
RAW_NULL_CHECK(tvalue);
}
@ -1060,6 +1060,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, uint32_t metaLen) {
SColCmpr* pCmp = &req.colCmpr.pColCmpr[i];
field.compress = pCmp->alg;
}
if (req.pExtSchemas) field.typeMod = req.pExtSchemas[i].typeMod;
RAW_NULL_CHECK(taosArrayPush(pReq.pColumns, &field));
}
pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField));
@ -1974,10 +1975,11 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
int j = 0;
for (; j < pTableMeta->tableInfo.numOfColumns; j++) {
SSchema* pColSchema = &pTableMeta->schema[j];
SSchemaExt* pColExtSchema = &pTableMeta->schemaExt[j];
char* fieldName = pSW->pSchema[i].name;
if (strcmp(pColSchema->name, fieldName) == 0) {
if (checkSchema(pColSchema, fields, NULL, 0) != 0){
if (checkSchema(pColSchema, pColExtSchema, fields, NULL, 0) != 0){
return true;
}
break;

7
source/client/src/clientStmt.c
View File

@ -213,7 +213,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) {
int32_t stmtRestoreQueryFields(STscStmt* pStmt) {
SStmtQueryResInfo* pRes = &pStmt->sql.queryRes;
int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD);
int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD_E);
pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols;
pStmt->exec.pRequest->body.resInfo.precision = pRes->precision;
@ -1270,8 +1270,9 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
if (pStmt->sql.pQuery->haveResultSet) {
STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema,
pStmt->sql.pQuery->numOfResCols));
pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema, true));
taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema);
taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema);
setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision);
}
@ -1861,7 +1862,7 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
}
*type = pField[idx].type;
*bytes = pField[idx].bytes;
*bytes = calcSchemaBytesFromTypeBytes(pField[idx].type, pField[idx].bytes, true);
_return:

3
source/client/src/clientStmt2.c
View File

@ -1498,8 +1498,9 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx, SVCr
if (pStmt->sql.pQuery->haveResultSet) {
STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema,
pStmt->sql.pQuery->numOfResCols));
pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema, true));
taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema);
taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema);
setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision);
}

4
source/client/src/clientTmq.c
View File

@ -3012,7 +3012,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
doFreeReqResultInfo(&pRspObj->resInfo);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(data->blockSchema, pRspObj->resIter);
if (pSW) {
TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols));
TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols, NULL, false));
}
}
@ -3028,7 +3028,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
pRspObj->resInfo.precision = precision;
pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows;
int32_t code = setResultDataPtr(&pRspObj->resInfo, convertUcs4);
int32_t code = setResultDataPtr(&pRspObj->resInfo, convertUcs4, false);
if (code != 0) {
return code;
}

151
source/common/src/msg/tmsg.c
View File

@ -673,6 +673,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name));
TAOS_CHECK_EXIT(tEncodeU32(&encoder, pField->compress));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->typeMod));
}
for (int32_t i = 0; i < pReq->numOfTags; ++i) {
@ -760,6 +761,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.bytes));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, field.name));
TAOS_CHECK_EXIT(tDecodeU32(&decoder, &field.compress));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.typeMod));
if (taosArrayPush(pReq->pColumns, &field) == NULL) {
TAOS_CHECK_EXIT(terrno);
}
@ -922,6 +924,19 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
}
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->keep));
ENCODESQL();
if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN ||
pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
if (taosArrayGetSize(pReq->pTypeMods) > 0) {
int8_t hasTypeMod = 1;
TAOS_CHECK_EXIT(tEncodeI8(&encoder, hasTypeMod));
for (int32_t i = 0; i < pReq->pTypeMods->size; ++i) {
const STypeMod *pTypeMod = taosArrayGet(pReq->pTypeMods, i);
TAOS_CHECK_ERRNO(tEncodeI32(&encoder, *pTypeMod));
}
} else {
TAOS_CHECK_EXIT(tEncodeI8(&encoder, 0));
}
}
tEndEncode(&encoder);
_exit:
@ -987,7 +1002,24 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->keep));
}
DECODESQL();
if (!tDecodeIsEnd(&decoder) && (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN ||
pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION)) {
int8_t hasTypeMod = 0;
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &hasTypeMod));
if (hasTypeMod == 1) {
pReq->pTypeMods = taosArrayInit(pReq->numOfFields, sizeof(STypeMod));
if (!pReq->pTypeMods) {
TAOS_CHECK_EXIT(terrno);
}
for (int32_t i = 0; i < pReq->numOfFields; ++i) {
STypeMod typeMod = 0;
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &typeMod));
if (taosArrayPush(pReq->pTypeMods, &typeMod) == NULL) {
TAOS_CHECK_EXIT(terrno);
}
}
}
}
tEndDecode(&decoder);
_exit:
@ -1000,6 +1032,7 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) {
pReq->pFields = NULL;
taosMemoryFreeClear(pReq->comment);
FREESQL();
taosArrayDestroy(pReq->pTypeMods);
}
int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) {
@ -3914,7 +3947,7 @@ int32_t tSerializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->tagsLen));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, pRsp->pTags, pRsp->tagsLen));
if (useCompress(pRsp->tableType)) {
if (withExtSchema(pRsp->tableType)) {
for (int32_t i = 0; i < pRsp->numOfColumns; ++i) {
SSchemaExt *pSchemaExt = &pRsp->pSchemaExt[i];
TAOS_CHECK_EXIT(tEncodeSSchemaExt(&encoder, pSchemaExt));
@ -3990,7 +4023,7 @@ int32_t tDeserializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp)
TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pRsp->pTags, NULL));
if (!tDecodeIsEnd(&decoder)) {
if (useCompress(pRsp->tableType) && pRsp->numOfColumns > 0) {
if (withExtSchema(pRsp->tableType) && pRsp->numOfColumns > 0) {
pRsp->pSchemaExt = taosMemoryMalloc(sizeof(SSchemaExt) * pRsp->numOfColumns);
if (pRsp->pSchemaExt == NULL) {
TAOS_CHECK_EXIT(terrno);
@ -6085,7 +6118,7 @@ static int32_t tEncodeSTableMetaRsp(SEncoder *pEncoder, STableMetaRsp *pRsp) {
TAOS_CHECK_RETURN(tEncodeSSchema(pEncoder, pSchema));
}
if (useCompress(pRsp->tableType)) {
if (withExtSchema(pRsp->tableType)) {
for (int32_t i = 0; i < pRsp->numOfColumns; ++i) {
SSchemaExt *pSchemaExt = &pRsp->pSchemaExt[i];
TAOS_CHECK_RETURN(tEncodeSSchemaExt(pEncoder, pSchemaExt));
@ -6126,7 +6159,7 @@ static int32_t tDecodeSTableMetaRsp(SDecoder *pDecoder, STableMetaRsp *pRsp) {
}
if (!tDecodeIsEnd(pDecoder)) {
if (useCompress(pRsp->tableType) && pRsp->numOfColumns > 0) {
if (withExtSchema(pRsp->tableType) && pRsp->numOfColumns > 0) {
pRsp->pSchemaExt = taosMemoryMalloc(sizeof(SSchemaExt) * pRsp->numOfColumns);
if (pRsp->pSchemaExt == NULL) {
TAOS_CHECK_RETURN(terrno);
@ -10124,7 +10157,13 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
SFieldWithOptions *pField = taosArrayGet(pReq->pCols, i);
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pField->type));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pField->flags));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes));
int32_t bytes = pField->bytes;
if (IS_DECIMAL_TYPE(pField->type)) {
uint8_t prec = 0, scale = 0;
extractTypeFromTypeMod(pField->type, pField->typeMod, &prec, &scale, NULL);
fillBytesForDecimalType(&bytes, pField->type, prec, scale);
}
TAOS_CHECK_EXIT(tEncodeI32(&encoder, bytes));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name));
}
@ -10139,6 +10178,12 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->notifyEventTypes));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->notifyErrorHandle));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->notifyHistory));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->recalculateInterval));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pWstartName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pWendName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pGroupIdName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pIsWindowFilledName));
tEndEncode(&encoder);
_exit:
@ -10297,6 +10342,14 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->notifyHistory));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->recalculateInterval));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pWstartName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pWendName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pGroupIdName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pIsWindowFilledName));
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
@ -10437,6 +10490,44 @@ _exit:
return code;
}
static int32_t tEncodeSExtSchema(SEncoder* pCoder, const SExtSchema* pExtSchema) {
int32_t code = 0, lino;
TAOS_CHECK_EXIT(tEncodeI32v(pCoder, pExtSchema->typeMod));
_exit:
return code;
}
int32_t tDecodeSExtSchema(SDecoder* pCoder, SExtSchema* pExtSchema) {
int32_t code = 0, lino;
TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &pExtSchema->typeMod));
_exit:
return code;
}
static int32_t tEncodeSExtSchemas(SEncoder* pCoder, const SExtSchema* pExtSchemas, int32_t nCol) {
int32_t code = 0, lino;
for (int32_t i = 0; i < nCol; ++i) {
TAOS_CHECK_EXIT(tEncodeSExtSchema(pCoder, pExtSchemas + i));
}
_exit:
return code;
}
static int32_t tDecodeSExtSchemas(SDecoder* pCoder, SExtSchema** ppExtSchema, int32_t nCol) {
int32_t code = 0, lino;
*ppExtSchema = tDecoderMalloc(pCoder, sizeof(SExtSchema) * nCol);
if (!*ppExtSchema) TAOS_CHECK_EXIT(terrno);
for (int32_t i = 0; i < nCol; ++i) {
TAOS_CHECK_EXIT(tDecodeSExtSchema(pCoder, (*ppExtSchema) + i));
}
_exit:
return code;
}
int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
int32_t code = 0;
int32_t lino;
@ -10461,6 +10552,12 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, pReq->colCmpred));
TAOS_CHECK_EXIT(tEncodeSColCmprWrapper(pCoder, &pReq->colCmpr));
TAOS_CHECK_EXIT(tEncodeI64(pCoder, pReq->keep));
if (pReq->pExtSchemas) {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 1));
TAOS_CHECK_EXIT(tEncodeSExtSchemas(pCoder, pReq->pExtSchemas, pReq->schemaRow.nCols));
} else {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 0));
}
tEndEncode(pCoder);
_exit:
@ -10498,6 +10595,13 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
if (!tDecodeIsEnd(pCoder)) {
TAOS_CHECK_EXIT(tDecodeI64(pCoder, &pReq->keep));
}
if (!tDecodeIsEnd(pCoder)) {
int8_t hasExtSchema = 0;
TAOS_CHECK_EXIT(tDecodeI8(pCoder, &hasExtSchema));
if (hasExtSchema) {
TAOS_CHECK_EXIT(tDecodeSExtSchemas(pCoder, &pReq->pExtSchemas, pReq->schemaRow.nCols));
}
}
}
tEndDecode(pCoder);
@ -10547,6 +10651,12 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
// Encode Column Options: encode compress level
if (pReq->type == TSDB_SUPER_TABLE || pReq->type == TSDB_NORMAL_TABLE) {
TAOS_CHECK_EXIT(tEncodeSColCmprWrapper(pCoder, &pReq->colCmpr));
if (pReq->pExtSchemas) {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 1));
TAOS_CHECK_EXIT(tEncodeSExtSchemas(pCoder, pReq->pExtSchemas, pReq->ntb.schemaRow.nCols));
} else {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 0));
}
}
tEndEncode(pCoder);
@ -10611,6 +10721,14 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
if (!tDecodeIsEnd(pCoder)) {
TAOS_CHECK_EXIT(tDecodeSColCmprWrapperEx(pCoder, &pReq->colCmpr));
}
if (!tDecodeIsEnd(pCoder)) {
int8_t hasExtSchema = 0;
TAOS_CHECK_EXIT(tDecodeI8(pCoder, &hasExtSchema));
if (hasExtSchema) {
TAOS_CHECK_EXIT(tDecodeSExtSchemas(pCoder, &pReq->pExtSchemas, pReq->ntb.schemaRow.nCols));
}
}
}
tEndDecode(pCoder);
@ -10968,6 +11086,9 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
}
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->ctimeMs));
TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pReq->source));
if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN) {
TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->typeMod));
}
tEndEncode(pEncoder);
_exit:
@ -11071,6 +11192,11 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
if (!tDecodeIsEnd(pDecoder)) {
TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pReq->source));
}
if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
if (!tDecodeIsEnd(pDecoder)) {
TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->typeMod));
}
}
tEndDecode(pDecoder);
_exit:
@ -11250,7 +11376,7 @@ int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal)
if (IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)) {
TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, pOffsetVal->primaryKey.pData, pOffsetVal->primaryKey.nData));
} else {
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pOffsetVal->primaryKey.val));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, VALUE_GET_TRIVIAL_DATUM(&pOffsetVal->primaryKey)));
}
} else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
@ -11281,7 +11407,7 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) {
TAOS_CHECK_EXIT(
tDecodeBinaryAlloc32(pDecoder, (void **)&pOffsetVal->primaryKey.pData, &pOffsetVal->primaryKey.nData));
} else {
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pOffsetVal->primaryKey.val));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &VALUE_GET_TRIVIAL_DATUM(&pOffsetVal->primaryKey)));
}
}
} else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
@ -11312,7 +11438,7 @@ void tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
taosMemoryFree(tmp);
} else {
(void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts,
pVal->primaryKey.type, pVal->primaryKey.val);
pVal->primaryKey.type, VALUE_GET_TRIVIAL_DATUM(&pVal->primaryKey));
}
}
}
@ -11694,7 +11820,7 @@ void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); }
int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
int32_t code = 0;
int32_t lino;
int32_t lino = 0;
TAOS_CHECK_EXIT(tEncodeMqDataRspCommon(pEncoder, pRsp));
TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->createTableNum));
@ -11736,7 +11862,6 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
}
}
}
_exit:
return code;
}
@ -13403,3 +13528,7 @@ void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp *pRsp) {
pRsp->batchMetaReq = NULL;
pRsp->batchMetaLen = NULL;
}
bool hasExtSchema(const SExtSchema *pExtSchema) {
return pExtSchema->typeMod != 0;
}

10
source/common/src/tcol.c
View File

@ -62,8 +62,9 @@ uint8_t getDefaultEncode(uint8_t type) {
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_VARBINARY:
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_DECIMAL:
return TSDB_COLVAL_ENCODE_DELTAD;
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_BLOB:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_MEDIUMBLOB:
@ -110,8 +111,9 @@ uint16_t getDefaultCompress(uint8_t type) {
return TSDB_COLVAL_COMPRESS_LZ4;
case TSDB_DATA_TYPE_VARBINARY:
return TSDB_COLVAL_COMPRESS_ZSTD;
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_DECIMAL:
return TSDB_COLVAL_COMPRESS_LZ4;
return TSDB_COLVAL_COMPRESS_ZSTD;
case TSDB_DATA_TYPE_BLOB:
return TSDB_COLVAL_COMPRESS_LZ4;
case TSDB_DATA_TYPE_MEDIUMBLOB:
@ -348,7 +350,7 @@ int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressTy
return TSDB_CODE_SUCCESS;
}
bool useCompress(uint8_t tableType) {
bool withExtSchema(uint8_t tableType) {
return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType;
}
@ -413,6 +415,8 @@ int8_t validColEncode(uint8_t type, uint8_t l1) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 || TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0;
} else if (type == TSDB_DATA_TYPE_GEOMETRY) {
return 1;
} else if (type == TSDB_DATA_TYPE_DECIMAL64 || type == TSDB_DATA_TYPE_DECIMAL) {
return l1 == TSDB_COLVAL_ENCODE_DISABLED ? 1 : 0;
}
return 0;
}

70
source/common/src/tdatablock.c
View File

@ -725,10 +725,13 @@ int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, b
void* skey = colDataGetData(pColInfoData, 0);
void* ekey = colDataGetData(pColInfoData, (pInfo->rows - 1));
int64_t val = 0;
if (asc) {
if (IS_NUMERIC_TYPE(pColInfoData->info.type)) {
GET_TYPED_DATA(pInfo->pks[0].val, int64_t, pColInfoData->info.type, skey);
GET_TYPED_DATA(pInfo->pks[1].val, int64_t, pColInfoData->info.type, ekey);
GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, skey, typeGetTypeModFromColInfo(&pColInfoData->info));
VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[0], val);
GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, ekey, typeGetTypeModFromColInfo(&pColInfoData->info));
VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[1], val);
} else { // todo refactor
memcpy(pInfo->pks[0].pData, varDataVal(skey), varDataLen(skey));
pInfo->pks[0].nData = varDataLen(skey);
@ -738,8 +741,10 @@ int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, b
}
} else {
if (IS_NUMERIC_TYPE(pColInfoData->info.type)) {
GET_TYPED_DATA(pInfo->pks[0].val, int64_t, pColInfoData->info.type, ekey);
GET_TYPED_DATA(pInfo->pks[1].val, int64_t, pColInfoData->info.type, skey);
GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, ekey, typeGetTypeModFromColInfo(&pColInfoData->info));
VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[0], val);
GET_TYPED_DATA(val, int64_t, pColInfoData->info.type, skey, typeGetTypeModFromColInfo(&pColInfoData->info));
VALUE_SET_TRIVIAL_DATUM(&pInfo->pks[1], val);
} else { // todo refactor
memcpy(pInfo->pks[0].pData, varDataVal(ekey), varDataLen(ekey));
pInfo->pks[0].nData = varDataLen(ekey);
@ -2803,7 +2808,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
return terrno;
}
SColVal cv = COL_VAL_VALUE(pCol->colId, ((SValue){.type = pCol->type, .val = *(TSKEY*)var}));
SValue val = {.type = pCol->type};
VALUE_SET_TRIVIAL_DATUM(&val, *(TSKEY*)var);
SColVal cv = COL_VAL_VALUE(pCol->colId, val);
void* px = taosArrayPush(pVals, &cv);
if (px == NULL) {
return terrno;
@ -2816,7 +2823,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat
return terrno;
}
} else {
SColVal cv = COL_VAL_VALUE(pCol->colId, ((SValue){.type = pCol->type, .val = *(int64_t*)var}));
SValue val = {.type = pCol->type};
VALUE_SET_TRIVIAL_DATUM(&val, *(int64_t*)var);
SColVal cv = COL_VAL_VALUE(pCol->colId, val);
void* px = taosArrayPush(pVals, &cv);
if (px == NULL) {
return terrno;
@ -2869,31 +2878,31 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat
} else {
SValue sv = {.type = pCol->type};
if (pCol->type == pColInfoData->info.type) {
memcpy(&sv.val, var, tDataTypes[pCol->type].bytes);
valueSetDatum(&sv, sv.type, var, tDataTypes[pCol->type].bytes);
} else {
/**
* 1. sum/avg would convert to int64_t/uint64_t/double during aggregation
* 2. below conversion may lead to overflow or loss, the app should select the right data type.
*/
char tv[8] = {0};
char tv[DATUM_MAX_SIZE] = {0};
if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) {
float v = 0;
GET_TYPED_DATA(v, float, pColInfoData->info.type, var);
GET_TYPED_DATA(v, float, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info));
SET_TYPED_DATA(&tv, pCol->type, v);
} else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) {
double v = 0;
GET_TYPED_DATA(v, double, pColInfoData->info.type, var);
GET_TYPED_DATA(v, double, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info));
SET_TYPED_DATA(&tv, pCol->type, v);
} else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) {
int64_t v = 0;
GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var);
GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info));
SET_TYPED_DATA(&tv, pCol->type, v);
} else {
uint64_t v = 0;
GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var);
GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var, typeGetTypeModFromColInfo(&pColInfoData->info));
SET_TYPED_DATA(&tv, pCol->type, v);
}
memcpy(&sv.val, tv, tDataTypes[pCol->type].bytes);
valueSetDatum(&sv, sv.type, tv, tDataTypes[pCol->type].bytes);
}
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
void* px = taosArrayPush(pVals, &cv);
@ -3187,7 +3196,12 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, size_t dataBuflen, in
*((int8_t*)data) = pColInfoData->info.type;
data += sizeof(int8_t);
*((int32_t*)data) = pColInfoData->info.bytes;
int32_t bytes = pColInfoData->info.bytes;
*((int32_t*)data) = bytes;
if (IS_DECIMAL_TYPE(pColInfoData->info.type)) {
fillBytesForDecimalType((int32_t*)data, pColInfoData->info.type, pColInfoData->info.precision,
pColInfoData->info.scale);
}
data += sizeof(int32_t);
}
@ -3328,6 +3342,10 @@ int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos
pStart += sizeof(int8_t);
pColInfoData->info.bytes = *(int32_t*)pStart;
if (IS_DECIMAL_TYPE(pColInfoData->info.type)) {
extractDecimalTypeInfoFromBytes(&pColInfoData->info.bytes, &pColInfoData->info.precision,
&pColInfoData->info.scale);
}
pStart += sizeof(int32_t);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
@ -3574,6 +3592,22 @@ int32_t trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolL
j += 1;
}
break;
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_DECIMAL:
while (j < totalRows) {
if (pBoolList[j] == 0) {
j += 1;
continue;
}
if (colDataIsNull_f(pBitmap, j)) {
colDataSetNull_f(pDst->nullbitmap, numOfRows);
} else {
memcpy(pDst->pData + numOfRows * pDst->info.bytes, pDst->pData + j * pDst->info.bytes, pDst->info.bytes);
}
numOfRows += 1;
j += 1;
}
break;
}
}
@ -3700,12 +3734,14 @@ int32_t blockDataCheck(const SSDataBlock* pDataBlock) {
} else {
if (TSDB_DATA_TYPE_FLOAT == pCol->info.type) {
float v = 0;
GET_TYPED_DATA(v, float, pCol->info.type, colDataGetNumData(pCol, r));
GET_TYPED_DATA(v, float, pCol->info.type, colDataGetNumData(pCol, r), typeGetTypeModFromColInfo(&pCol->info));
} else if (TSDB_DATA_TYPE_DOUBLE == pCol->info.type) {
double v = 0;
GET_TYPED_DATA(v, double, pCol->info.type, colDataGetNumData(pCol, r));
GET_TYPED_DATA(v, double, pCol->info.type, colDataGetNumData(pCol, r), typeGetTypeModFromColInfo(&pCol->info));
} else if (IS_DECIMAL_TYPE(pCol->info.type)) {
// SKIP for decimal types
} else {
GET_TYPED_DATA(typeValue, int64_t, pCol->info.type, colDataGetNumData(pCol, r));
GET_TYPED_DATA(typeValue, int64_t, pCol->info.type, colDataGetNumData(pCol, r), typeGetTypeModFromColInfo(&pCol->info));
}
}
}

264
source/common/src/tdataformat.c
View File

@ -18,6 +18,7 @@
#include "tRealloc.h"
#include "tdatablock.h"
#include "tlog.h"
#include "decimal.h"
static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData);
static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward);
@ -277,7 +278,7 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
(*ppRow)->numOfPKs = sinfo->numOfPKs;
(*ppRow)->sver = schema->version;
(*ppRow)->len = sinfo->tupleRowSize;
(*ppRow)->ts = colValArray[0].value.val;
(*ppRow)->ts = VALUE_GET_TRIVIAL_DATUM(&colValArray[0].value);
if (sinfo->tupleFlag == HAS_NONE || sinfo->tupleFlag == HAS_NULL) {
return 0;
@ -315,7 +316,8 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
varlen += colValArray[colValIndex].value.nData;
}
} else {
(void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val,
(void)memcpy(fixed + schema->columns[i].offset,
VALUE_GET_DATUM(&colValArray[colValIndex].value, schema->columns[i].type),
tDataTypes[schema->columns[i].type].bytes);
}
} else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL
@ -360,7 +362,7 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
(*ppRow)->numOfPKs = sinfo->numOfPKs;
(*ppRow)->sver = schema->version;
(*ppRow)->len = sinfo->kvRowSize;
(*ppRow)->ts = colValArray[0].value.val;
(*ppRow)->ts = VALUE_GET_TRIVIAL_DATUM(&colValArray[0].value);
if (!(sinfo->flag != HAS_NONE && sinfo->flag != HAS_NULL)) {
return TSDB_CODE_INVALID_PARA;
@ -397,7 +399,7 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
payloadSize += colValArray[colValIndex].value.nData;
} else {
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
(void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val,
(void)memcpy(payload + payloadSize, VALUE_GET_DATUM(&colValArray[colValIndex].value, schema->columns[i].type),
tDataTypes[schema->columns[i].type].bytes);
payloadSize += tDataTypes[schema->columns[i].type].bytes;
}
@ -490,7 +492,8 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted,
}
value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow;
} else {
(void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
valueSetDatum(&value, infos[iInfo].type,
(uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
infos[iInfo].bind->buffer_length);
}
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
@ -543,7 +546,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->cid = pTColumn->colId;
pColVal->value.type = pTColumn->type;
pColVal->flag = CV_FLAG_VALUE;
(void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
VALUE_SET_TRIVIAL_DATUM(&pColVal->value, pRow->ts);
return 0;
}
@ -607,7 +610,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = NULL;
}
} else {
(void)memcpy(&pColVal->value.val, pData, pTColumn->bytes);
valueSetDatum(&pColVal->value, pTColumn->type, pData, pTColumn->bytes);
}
}
return 0;
@ -658,7 +661,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset);
pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData);
} else {
(void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
valueSetDatum(&pColVal->value, pTColumn->type, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
}
}
@ -902,7 +905,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.cid = pTColumn->colId;
pIter->cv.value.type = pTColumn->type;
pIter->cv.flag = CV_FLAG_VALUE;
(void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY));
VALUE_SET_TRIVIAL_DATUM(&pIter->cv.value, pIter->pRow->ts);
goto _exit;
}
@ -947,7 +950,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL;
}
} else {
(void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes);
valueSetDatum(&pIter->cv.value, pTColumn->type, pData, pTColumn->bytes);
}
}
@ -1006,7 +1009,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL;
}
} else {
(void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
valueSetDatum(&pIter->cv.value, pTColumn->type, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
}
goto _exit;
}
@ -1326,7 +1329,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
key->pks[i].pData = tdata;
key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData);
} else {
(void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes);
valueSetDatum(key->pks + i, indices[i].type, tdata, tDataTypes[indices[i].type].bytes);
}
}
}
@ -1346,26 +1349,26 @@ int32_t tValueCompare(const SValue *tv1, const SValue *tv2) {
switch (tv1->type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
T_COMPARE_SCALAR_VALUE(int8_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(int8_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_SMALLINT:
T_COMPARE_SCALAR_VALUE(int16_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(int16_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_INT:
T_COMPARE_SCALAR_VALUE(int32_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(int32_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
T_COMPARE_SCALAR_VALUE(int64_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(int64_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_FLOAT:
T_COMPARE_SCALAR_VALUE(float, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(float, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_DOUBLE:
T_COMPARE_SCALAR_VALUE(double, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(double, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_UTINYINT:
T_COMPARE_SCALAR_VALUE(uint8_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(uint8_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_USMALLINT:
T_COMPARE_SCALAR_VALUE(uint16_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(uint16_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_UINT:
T_COMPARE_SCALAR_VALUE(uint32_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(uint32_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_UBIGINT:
T_COMPARE_SCALAR_VALUE(uint64_t, &tv1->val, &tv2->val);
T_COMPARE_SCALAR_VALUE(uint64_t, &VALUE_GET_TRIVIAL_DATUM(tv1), &VALUE_GET_TRIVIAL_DATUM(tv2));
case TSDB_DATA_TYPE_GEOMETRY:
case TSDB_DATA_TYPE_BINARY: {
int32_t ret = strncmp((const char *)tv1->pData, (const char *)tv2->pData, TMIN(tv1->nData, tv2->nData));
@ -1420,12 +1423,7 @@ void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
SValue *pVal = &pDst->pks[i];
pVal->type = pSrc->pks[i].type;
if (IS_NUMERIC_TYPE(pVal->type)) {
pVal->val = pSrc->pks[i].val;
} else {
pVal->nData = pSrc->pks[i].nData;
(void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
}
valueCloneDatum(pVal, pSrc->pks + i, pVal->type);
}
}
}
@ -2267,8 +2265,7 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
return TSDB_CODE_INVALID_PARA;
}
return tColDataAppendValueImpl[pColData->flag][pColVal->flag](
pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val,
pColVal->value.nData);
pColData, VALUE_GET_DATUM(&pColVal->value, pColData->type), pColVal->value.nData);
}
static FORCE_INLINE int32_t tColDataUpdateValue10(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
@ -2581,8 +2578,7 @@ int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward)
if (tColDataUpdateValueImpl[pColData->flag][pColVal->flag] == NULL) return 0;
return tColDataUpdateValueImpl[pColData->flag][pColVal->flag](
pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val,
pColVal->value.nData, forward);
pColData, VALUE_GET_DATUM(&pColVal->value, pColData->type), pColVal->value.nData, forward);
}
static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE
@ -2614,7 +2610,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo
}
value.pData = pColData->pData + pColData->aOffset[iVal];
} else {
(void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal,
valueSetDatum(&value, pColData->type, pColData->pData + tDataTypes[pColData->type].bytes * iVal,
tDataTypes[pColData->type].bytes);
}
*pColVal = COL_VAL_VALUE(pColData->cid, value);
@ -3323,9 +3319,7 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte
if (TSDB_DATA_TYPE_BOOL == value.type && *val > 1) {
*val = 1;
}
(void)memcpy(&value.val, val,
/*(uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,*/
infos[iInfo].bytes /*bind->buffer_length*/);
valueSetDatum(&value, infos[iInfo].type, val, infos[iInfo].bytes);
}
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
}
@ -3839,8 +3833,9 @@ int32_t tDecodeRow(SDecoder *pDecoder, SRow **ppRow) {
if ((MIN) > (VAL)) (MIN) = (VAL); \
} while (0)
static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t*numOfNull = &pAggs->numOfNull;
*sum = 0;
*max = 0;
*min = 1;
@ -3870,8 +3865,9 @@ static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, i
}
}
static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*sum = 0;
*max = INT8_MIN;
*min = INT8_MAX;
@ -3901,8 +3897,9 @@ static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum
}
}
static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*sum = 0;
*max = INT16_MIN;
*min = INT16_MAX;
@ -3932,8 +3929,9 @@ static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t
}
}
static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*sum = 0;
*max = INT32_MIN;
*min = INT32_MAX;
@ -3963,8 +3961,9 @@ static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, in
}
}
static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*sum = 0;
*max = INT64_MIN;
*min = INT64_MAX;
@ -3994,8 +3993,9 @@ static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum,
}
}
static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*(double *)sum = 0;
*(double *)max = -FLT_MAX;
*(double *)min = FLT_MAX;
@ -4025,8 +4025,9 @@ static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum,
}
}
static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*(double *)sum = 0;
*(double *)max = -DBL_MAX;
*(double *)min = DBL_MAX;
@ -4056,8 +4057,9 @@ static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum,
}
}
static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT8_MAX;
@ -4087,8 +4089,9 @@ static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *su
}
}
static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT16_MAX;
@ -4118,8 +4121,9 @@ static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_
}
}
static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT32_MAX;
@ -4149,8 +4153,9 @@ static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, i
}
}
static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = UINT64_MAX;
@ -4180,8 +4185,9 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum
}
}
static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
int16_t *numOfNull) {
static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, SColumnDataAgg* pAggs) {
int64_t *sum = &pAggs->sum, *max = &pAggs->max, *min = &pAggs->min;
int16_t *numOfNull = &pAggs->numOfNull;
*(uint64_t *)sum = 0;
*(uint64_t *)max = 0;
*(uint64_t *)min = 0;
@ -4216,7 +4222,90 @@ static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum
}
}
void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = {
#define CALC_DECIMAL_SUM_MAX_MIN(TYPE, pSumOp, pCompOp, pColData, pSum, pMax, pMin) \
do { \
if (decimal128AddCheckOverflow((Decimal *)pSum, pVal, DECIMAL_WORD_NUM(TYPE))) *pOverflow = true; \
pSumOp->add(pSum, pVal, DECIMAL_WORD_NUM(TYPE)); \
if (pCompOp->gt(pVal, pMax, DECIMAL_WORD_NUM(TYPE))) { \
*(pMax) = *pVal; \
} \
if (pCompOp->lt(pVal, pMin, DECIMAL_WORD_NUM(TYPE))) { \
*(pMin) = *pVal; \
} \
} while (0)
static FORCE_INLINE void tColDataCalcSMADecimal64Type(SColData* pColData, SColumnDataAgg* pAggs) {
Decimal128 *pSum = (Decimal128 *)pAggs->decimal128Sum;
Decimal64 *pMax = (Decimal64 *)pAggs->decimal128Max, *pMin = (Decimal64 *)pAggs->decimal128Min;
uint8_t *pOverflow = &pAggs->overflow;
*pSum = DECIMAL128_ZERO;
*pMax = DECIMAL64_MIN;
*pMin = DECIMAL64_MAX;
pAggs->numOfNull = 0;
pAggs->colId |= DECIMAL_AGG_FLAG;
Decimal64 *pVal = NULL;
const SDecimalOps *pSumOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL);
const SDecimalOps *pCompOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL64);
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
pVal = ((Decimal64*)pColData->pData) + iVal;
CALC_DECIMAL_SUM_MAX_MIN(Decimal64, pSumOps, pCompOps, pColData, pSum, pMax, pMin);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
pAggs->numOfNull++;
break;
case 2:
pVal = ((Decimal64 *)pColData->pData) + iVal;
CALC_DECIMAL_SUM_MAX_MIN(Decimal64, pSumOps, pCompOps, pColData, pSum, pMax, pMin);
break;
default:
break;
}
}
}
}
static FORCE_INLINE void tColDataCalcSMADecimal128Type(SColData* pColData, SColumnDataAgg* pAggs) {
Decimal128 *pSum = (Decimal128 *)pAggs->decimal128Sum, *pMax = (Decimal128 *)pAggs->decimal128Max,
*pMin = (Decimal128 *)pAggs->decimal128Min;
uint8_t *pOverflow = &pAggs->overflow;
*pSum = DECIMAL128_ZERO;
*pMax = DECIMAL128_MIN;
*pMin = DECIMAL128_MAX;
pAggs->numOfNull = 0;
pAggs->colId |= DECIMAL_AGG_FLAG;
Decimal128 *pVal = NULL;
const SDecimalOps *pOps = getDecimalOps(TSDB_DATA_TYPE_DECIMAL);
if (HAS_VALUE == pColData->flag) {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
pVal = ((Decimal128*)pColData->pData) + iVal;
CALC_DECIMAL_SUM_MAX_MIN(Decimal128, pOps, pOps, pColData, pSum, pMax, pMin);
}
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
switch (tColDataGetBitValue(pColData, iVal)) {
case 0:
case 1:
pAggs->numOfNull++;
break;
case 2:
pVal = ((Decimal128*)pColData->pData) + iVal;
CALC_DECIMAL_SUM_MAX_MIN(Decimal128, pOps, pOps, pColData, pSum, pMax, pMin);
break;
default:
break;
}
}
}
}
void (*tColDataCalcSMA[])(SColData *pColData, SColumnDataAgg* pAggs) = {
NULL,
tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL
tColDataCalcSMATinyInt, // TSDB_DATA_TYPE_TINYINT
@ -4234,10 +4323,11 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_
tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARBINARY
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_DECIMAL
tColDataCalcSMADecimal128Type, // TSDB_DATA_TYPE_DECIMAL
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_BLOB
NULL, // TSDB_DATA_TYPE_MEDIUMBLOB
tColDataCalcSMAVarType // TSDB_DATA_TYPE_GEOMETRY
tColDataCalcSMAVarType, // TSDB_DATA_TYPE_GEOMETRY
tColDataCalcSMADecimal64Type, // TSDB_DATA_TYPE_DECIMAL64
};
// SValueColumn ================================
@ -4284,7 +4374,7 @@ int32_t tValueColumnAppend(SValueColumn *valCol, const SValue *value) {
return code;
}
} else {
code = tBufferPut(&valCol->data, &value->val, tDataTypes[value->type].bytes);
code = tBufferPut(&valCol->data, VALUE_GET_DATUM(value, value->type), tDataTypes[value->type].bytes);
if (code) return code;
}
valCol->numOfValues++;
@ -4317,7 +4407,7 @@ int32_t tValueColumnUpdate(SValueColumn *valCol, int32_t idx, const SValue *valu
}
return tBufferPutAt(&valCol->data, offsets[idx], value->pData, value->nData);
} else {
return tBufferPutAt(&valCol->data, idx * tDataTypes[valCol->type].bytes, &value->val,
return tBufferPutAt(&valCol->data, idx * tDataTypes[valCol->type].bytes, VALUE_GET_DATUM(value, valCol->type),
tDataTypes[valCol->type].bytes);
}
return 0;
@ -4343,7 +4433,7 @@ int32_t tValueColumnGet(SValueColumn *valCol, int32_t idx, SValue *value) {
value->pData = (uint8_t *)tBufferGetDataAt(&valCol->data, offset);
} else {
SBufferReader reader = BUFFER_READER_INITIALIZER(idx * tDataTypes[value->type].bytes, &valCol->data);
TAOS_CHECK_RETURN(tBufferGet(&reader, tDataTypes[value->type].bytes, &value->val));
TAOS_CHECK_RETURN(tBufferGet(&reader, tDataTypes[value->type].bytes, VALUE_GET_DATUM(value, value->type)));
}
return 0;
}
@ -4677,3 +4767,45 @@ int32_t tDecompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *outpu
output->size += info->originalSize;
return 0;
}
// handle all types, including var data
void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len) {
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) {
pVal->pData = pDatum;
pVal->nData = len;
} else {
switch (len) {
case sizeof(uint8_t):
pVal->val = *(uint8_t *)pDatum;
break;
case sizeof(uint16_t):
pVal->val = *(uint16_t *)pDatum;
break;
case sizeof(uint32_t):
pVal->val = *(uint32_t *)pDatum;
break;
case sizeof(uint64_t):
pVal->val = *(uint64_t *)pDatum;
break;
default:
break;
}
}
}
void valueCloneDatum(SValue *pDst, const SValue *pSrc, int8_t type) {
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) {
memcpy(pDst->pData, pSrc->pData, pSrc->nData);
pDst->nData = pSrc->nData;
} else {
pDst->val = pSrc->val;
}
}
void valueClearDatum(SValue *pVal, int8_t type) {
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) {
taosMemoryFreeClear(pVal->pData);
pVal->nData = 0;
} else {
pVal->val = 0;
}
}

19
source/common/src/tglobal.c
View File

@ -353,6 +353,10 @@ int tsStreamAggCnt = 100000;
bool tsStreamCoverage = false;
bool tsStreamRunHistoryAsync = false;
char tsAdapterFqdn[TSDB_FQDN_LEN] = "localhost";
uint16_t tsAdapterPort = 6041;
char tsAdapterToken[512] = "cm9vdDp0YW9zZGF0YQ==";
bool tsUpdateCacheBatch = true;
int8_t tsS3EpNum = 0;
@ -1003,6 +1007,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyMessageSize", tsStreamNotifyMessageSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyFrameSize", tsStreamNotifyFrameSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "adapterFqdn", tsAdapterFqdn, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "adapterPort", tsAdapterPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddString(pCfg, "adapterToken", tsAdapterToken, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
// clang-format on
// GRANT_CFG_ADD;
@ -1909,6 +1917,17 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamNotifyFrameSize");
tsStreamNotifyFrameSize = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "adapterFqdn");
TAOS_CHECK_RETURN(taosCheckCfgStrValueLen(pItem->name, pItem->str, TSDB_FQDN_LEN));
tstrncpy(tsAdapterFqdn, pItem->str, TSDB_FQDN_LEN);
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "adapterPort");
tsAdapterPort = (uint16_t)pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "adapterToken");
TAOS_CHECK_RETURN(taosCheckCfgStrValueLen(pItem->name, pItem->str, tListLen(tsAdapterToken)));
tstrncpy(tsAdapterToken, pItem->str, tListLen(tsAdapterToken));
// GRANT_CFG_GET;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}

4
source/common/src/trow.c
View File

@ -517,7 +517,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow, int8_t r
val = varBuf;
++iBound;
} else {
val = (const void *)&pColVal->value.val;
val = VALUE_GET_DATUM(&pColVal->value, pTColumn->type);
++iBound;
}
} else {
@ -972,7 +972,7 @@ int32_t tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pC
pColVal->value.nData = varDataLen(cv.val);
pColVal->value.pData = varDataVal(cv.val);
} else {
(void)memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes);
valueSetDatum(&pColVal->value, pTColumn->type, cv.val, tDataTypes[pTColumn->type].bytes);
}
}
return 0;

91
source/common/src/ttypes.c
View File

@ -17,7 +17,7 @@
#include "ttypes.h"
#include "tcompression.h"
const int32_t TYPE_BYTES[21] = {
const int32_t TYPE_BYTES[TSDB_DATA_TYPE_MAX] = {
2, // TSDB_DATA_TYPE_NULL
CHAR_BYTES, // TSDB_DATA_TYPE_BOOL
CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT
@ -35,10 +35,11 @@ const int32_t TYPE_BYTES[21] = {
sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT
TSDB_MAX_JSON_TAG_LEN, // TSDB_DATA_TYPE_JSON
sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_VARBINARY
TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented
DECIMAL128_BYTES, // TSDB_DATA_TYPE_DECIMAL: placeholder, not implemented
TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_BLOB: placeholder, not implemented
TSDB_MAX_TAGS_LEN, // TSDB_DATA_TYPE_MEDIUMBLOB: placeholder, not implemented
sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_GEOMETRY
DECIMAL64_BYTES, // TSDB_DATA_TYPE_DECIMAL64
};
tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
@ -63,10 +64,11 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString},
{TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString,
tsDecompressString}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, DECIMAL128_BYTES, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString, tsDecompressString},
{TSDB_DATA_TYPE_DECIMAL64, 7, DECIMAL64_BYTES, "DECIMAL", 0, 0, NULL, NULL},
};
tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = {
@ -92,10 +94,11 @@ tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString2, tsDecompressString2},
{TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString2,
tsDecompressString2}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, DECIMAL128_BYTES, "DECIMAL", 0, 0, tsCompressDecimal128, tsDecompressDecimal128},
{TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString2, tsDecompressString2},
{TSDB_DATA_TYPE_DECIMAL64, 9, DECIMAL64_BYTES, "DECIMAL64", 0, 0, tsCompressDecimal64, tsDecompressDecimal64},
};
@ -229,3 +232,83 @@ int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
return 0;
}
uint8_t decimalTypeFromPrecision(uint8_t precision) {
return precision > TSDB_DECIMAL64_MAX_PRECISION ? TSDB_DATA_TYPE_DECIMAL : TSDB_DATA_TYPE_DECIMAL64;
}
STypeMod decimalCalcTypeMod(uint8_t prec, uint8_t scale) {
return ((STypeMod)prec << 8) + scale;
}
void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale) {
if (precision) *precision = (uint8_t)((typeMod >> 8) & 0xFF);
if (scale) *scale = (uint8_t)(typeMod & 0xFF);
}
STypeMod typeGetTypeModFromDataType(const SDataType* pDataType) {
if (IS_DECIMAL_TYPE(pDataType->type)) return decimalCalcTypeMod(pDataType->precision, pDataType->scale);
return 0;
}
STypeMod typeGetTypeMod(uint8_t type, uint8_t prec, uint8_t scale, int32_t bytes) {
if (IS_DECIMAL_TYPE(type)) {
return decimalCalcTypeMod(prec, scale);
}
return 0;
}
void fillTypeFromTypeMod(SDataType* pType, STypeMod mod) {
if (IS_DECIMAL_TYPE(pType->type)) {
decimalFromTypeMod(mod, &pType->precision, &pType->scale);
}
}
void extractTypeFromTypeMod(uint8_t type, STypeMod typeMod, uint8_t *prec, uint8_t *scale, int32_t *bytes) {
if (IS_DECIMAL_TYPE(type)) {
decimalFromTypeMod(typeMod, prec, scale);
} else {
if (prec) *prec = 0;
if (scale) *scale = 0;
}
if (bytes) *bytes = tDataTypes[type].bytes;
}
uint8_t getScaleFromTypeMod(int32_t type, STypeMod mod) {
if (IS_DECIMAL_TYPE(type)) return (uint8_t)(mod & 0xFF);
return 0;
}
// bytes, 0, prec, scale
void fillBytesForDecimalType(int32_t *pBytes, int32_t type, uint8_t precision, uint8_t scale) {
*pBytes = 0;
*pBytes = tDataTypes[type].bytes << 24;
*pBytes |= (uint32_t)precision << 8;
*pBytes |= scale;
}
void extractDecimalTypeInfoFromBytes(int32_t *pBytes, uint8_t *precision, uint8_t *scale) {
*precision = (uint8_t)((*pBytes >> 8) & 0xFF);
*scale = (uint8_t)(*pBytes & 0xFF);
*pBytes >>= 24;
}
int32_t calcTypeBytesFromSchemaBytes(int32_t type, int32_t schemaBytes, bool isStmt) {
if (isStmt) return schemaBytes;
if (type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) {
return schemaBytes - VARSTR_HEADER_SIZE;
} else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) {
return (schemaBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
}
return schemaBytes;
}
int32_t calcSchemaBytesFromTypeBytes(int32_t type, int32_t varTypeBytes, bool isStmt) {
if (isStmt) return varTypeBytes;
if (type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) {
return varTypeBytes + VARSTR_HEADER_SIZE;
} else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) {
return varTypeBytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
}
return varTypeBytes;
}

6
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -561,6 +561,7 @@ typedef struct {
col_id_t colId;
int32_t cmprAlg;
} SCmprObj;
typedef struct {
char name[TSDB_TABLE_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
@ -591,6 +592,7 @@ typedef struct {
int8_t source;
SColCmpr* pCmpr;
int64_t keep;
SExtSchema* pExtSchemas;
} SStbObj;
typedef struct {
@ -815,9 +817,9 @@ typedef struct {
char* sql;
char* ast;
char* physicalPlan;
SArray* tasks; // SArray<SArray<SStreamTask>>
SArray* pHTasksList; // generate the results for already stored ts data
SArray* pTaskList; // SArray<SArray<SStreamTask>>
SArray* pHTaskList; // generate the results for already stored ts data
int64_t hTaskUid; // stream task for history ts data
SSchemaWrapper outputSchema;

18
source/dnode/mnode/impl/src/mndDef.c
View File

@ -66,10 +66,10 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
}
int32_t sz = taosArrayGetSize(pObj->tasks);
int32_t sz = taosArrayGetSize(pObj->pTaskList);
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, sz));
for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGetP(pObj->tasks, i);
SArray *pArray = taosArrayGetP(pObj->pTaskList, i);
int32_t innerSz = taosArrayGetSize(pArray);
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz));
for (int32_t j = 0; j < innerSz; j++) {
@ -129,16 +129,16 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->ast));
TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan));
if (pObj->tasks != NULL) {
pObj->tasks = freeStreamTasks(pObj->tasks);
if (pObj->pTaskList != NULL) {
pObj->pTaskList = freeStreamTasks(pObj->pTaskList);
}
int32_t sz;
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &sz));
if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(void *));
if (pObj->tasks == NULL) {
pObj->pTaskList = taosArrayInit(sz, sizeof(void *));
if (pObj->pTaskList == NULL) {
code = terrno;
TAOS_RETURN(code);
}
@ -168,7 +168,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
}
}
}
if (taosArrayPush(pObj->tasks, &pArray) == NULL) {
if (taosArrayPush(pObj->pTaskList, &pArray) == NULL) {
taosArrayDestroy(pArray);
code = terrno;
TAOS_RETURN(code);
@ -227,8 +227,8 @@ void tFreeStreamObj(SStreamObj *pStream) {
taosMemoryFree(pStream->outputSchema.pSchema);
}
pStream->tasks = freeStreamTasks(pStream->tasks);
pStream->pHTasksList = freeStreamTasks(pStream->pHTasksList);
pStream->pTaskList = freeStreamTasks(pStream->pTaskList);
pStream->pHTaskList = freeStreamTasks(pStream->pHTaskList);
// tagSchema.pSchema
if (pStream->tagSchema.nCols > 0) {

3
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -664,6 +664,7 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->pColumns = NULL;
pNew->pCmpr = NULL;
pNew->pTags = NULL;
pNew->pExtSchemas = NULL;
pNew->updateTime = taosGetTimestampMs();
pNew->lock = 0;
@ -733,6 +734,7 @@ _OVER:
taosMemoryFree(newStb.pTags);
taosMemoryFree(newStb.pColumns);
taosMemoryFree(newStb.pCmpr);
taosMemoryFreeClear(newStb.pExtSchemas);
}
mndTransDrop(pTrans);
TAOS_RETURN(code);
@ -847,6 +849,7 @@ _OVER:
taosMemoryFree(newObj.pTags);
taosMemoryFree(newObj.pColumns);
taosMemoryFree(newObj.pCmpr);
taosMemoryFreeClear(newObj.pExtSchemas);
mndTransDrop(pTrans);
mndReleaseStb(pMnode, pStb);

174
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -237,31 +237,49 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, SStreamObj* pStream) {
return pVgroup;
}
static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgroup, SEpSet* pEpset, bool isFillhistory) {
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
static void streamGetUidTaskList(SStreamObj* pStream, EStreamTaskType type, uint64_t* pUid, SArray*** pTaskList) {
if (type == STREAM_NORMAL_TASK) {
*pUid = pStream->uid;
*pTaskList = taosArrayGetLast(pStream->pTaskList);
} else if (type == STREAM_HISTORY_TASK || type == STREAM_RECALCUL_TASK) {
*pUid = pStream->hTaskUid;
*pTaskList = taosArrayGetLast(pStream->pHTaskList);
}
}
static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgroup, SEpSet* pEpset, EStreamTaskType type) {
uint64_t uid = 0;
SArray** pTaskList = NULL;
streamGetUidTaskList(pStream, type, &uid, &pTaskList);
SStreamTask* pTask = NULL;
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, &pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, type, pStream->conf.trigger, 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, 1, &pTask);
if (code != 0) {
return code;
}
mDebug("doAddSinkTask taskId:%s, %p vgId:%d, isFillHistory:%d", pTask->id.idStr, pTask, pVgroup->vgId, isFillhistory);
mDebug("doAddSinkTask taskId:%s, %p vgId:%d, isFillHistory:%d", pTask->id.idStr, pTask, pVgroup->vgId, (type == STREAM_HISTORY_TASK));
pTask->info.nodeId = pVgroup->vgId;
pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup);
return mndSetSinkTaskInfo(pStream, pTask);
}
bool needHistoryTask(SStreamObj* pStream) {
return (pStream->conf.fillHistory) || (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE);
}
static int32_t doAddSinkTaskToVg(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset, SVgObj* vgObj) {
int32_t code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, false);
int32_t code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, STREAM_NORMAL_TASK);
if (code != 0) {
return code;
}
if (pStream->conf.fillHistory) {
code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, true);
if (needHistoryTask(pStream)) {
EStreamTaskType type = (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) ? STREAM_RECALCUL_TASK
: STREAM_HISTORY_TASK;
code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, type);
if (code != 0) {
return code;
}
@ -321,7 +339,7 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe
SDataRange* pRange = &pTask->dataRange;
STimeWindow* pWindow = &pRange->window;
if (pTask->info.fillHistory) {
if (pTask->info.fillHistory == STREAM_HISTORY_TASK) {
pWindow->skey = INT64_MIN;
pWindow->ekey = skey - 1;
@ -352,13 +370,25 @@ static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFi
}
}
static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam, SStreamTask** pTask) {
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, EStreamTaskType type, bool useTriggerParam,
int8_t hasAggTasks, SStreamTask** pTask) {
uint64_t uid = 0;
SArray** pTaskList = NULL;
streamGetUidTaskList(pStream, type, &uid, &pTaskList);
int32_t trigger = 0;
if (type == STREAM_RECALCUL_TASK) {
trigger = STREAM_TRIGGER_WINDOW_CLOSE;
} else {
trigger = pStream->conf.trigger;
}
int32_t triggerParam = useTriggerParam ? pStream->conf.triggerParam : 0;
int32_t code =
tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, type, trigger, triggerParam,
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5,
hasAggTasks, pTask);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, pStream->conf.trigger,
useTriggerParam ? pStream->conf.triggerParam : 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, pTask);
return code;
}
@ -369,19 +399,19 @@ static int32_t addNewTaskList(SStreamObj* pStream) {
return terrno;
}
if (taosArrayPush(pStream->tasks, &pTaskList) == NULL) {
if (taosArrayPush(pStream->pTaskList, &pTaskList) == NULL) {
mError("failed to put into array, code:%s", tstrerror(terrno));
return terrno;
}
if (pStream->conf.fillHistory) {
if (needHistoryTask(pStream)) {
pTaskList = taosArrayInit(0, POINTER_BYTES);
if (pTaskList == NULL) {
mError("failed init task list, code:%s", tstrerror(terrno));
mError("failed init history task list, code:%s", tstrerror(terrno));
return terrno;
}
if (taosArrayPush(pStream->pHTasksList, &pTaskList) == NULL) {
if (taosArrayPush(pStream->pHTaskList, &pTaskList) == NULL) {
mError("failed to put into array, code:%s", tstrerror(terrno));
return terrno;
}
@ -392,8 +422,8 @@ static int32_t addNewTaskList(SStreamObj* pStream) {
// set the history task id
static void setHTasksId(SStreamObj* pStream) {
SArray* pTaskList = *(SArray**)taosArrayGetLast(pStream->tasks);
SArray* pHTaskList = *(SArray**)taosArrayGetLast(pStream->pHTasksList);
SArray* pTaskList = *(SArray**)taosArrayGetLast(pStream->pTaskList);
SArray* pHTaskList = *(SArray**)taosArrayGetLast(pStream->pHTaskList);
for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) {
SStreamTask** pStreamTask = taosArrayGet(pTaskList, i);
@ -411,17 +441,17 @@ static void setHTasksId(SStreamObj* pStream) {
}
static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
SArray* pVerList, SVgObj* pVgroup, bool isHistoryTask, bool useTriggerParam) {
SArray* pVerList, SVgObj* pVgroup, EStreamTaskType type, bool useTriggerParam, int8_t hasAggTasks) {
SStreamTask* pTask = NULL;
int32_t code = buildSourceTask(pStream, pEpset, isHistoryTask, useTriggerParam, &pTask);
int32_t code = buildSourceTask(pStream, pEpset, type, useTriggerParam, hasAggTasks, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
mDebug("doAddSourceTask taskId:%s, %p vgId:%d, historyTask:%d", pTask->id.idStr, pTask, pVgroup->vgId, isHistoryTask);
mDebug("doAddSourceTask taskId:%s, %p vgId:%d, historyTask:%d", pTask->id.idStr, pTask, pVgroup->vgId, (type == STREAM_HISTORY_TASK));
if (pStream->conf.fillHistory) {
haltInitialTaskStatus(pTask, plan, isHistoryTask);
if (needHistoryTask(pStream)) {
haltInitialTaskStatus(pTask, plan, (type == STREAM_HISTORY_TASK));
}
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
@ -466,7 +496,7 @@ static SSubplan* getAggSubPlan(const SQueryPlan* pPlan, int index) {
}
static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset,
int64_t nextWindowSkey, SArray* pVerList, bool useTriggerParam) {
int64_t nextWindowSkey, SArray* pVerList, bool useTriggerParam, bool hasAggTasks) {
void* pIter = NULL;
SSdb* pSdb = pMnode->pSdb;
int32_t code = addNewTaskList(pStream);
@ -486,7 +516,8 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
continue;
}
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam);
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, STREAM_NORMAL_TASK,
useTriggerParam, hasAggTasks);
if (code != 0) {
mError("failed to create stream task, code:%s", tstrerror(code));
@ -495,8 +526,16 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
return code;
}
if (pStream->conf.fillHistory) {
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, true, useTriggerParam);
if (needHistoryTask(pStream)) {
EStreamTaskType type = 0;
if (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE && (pStream->conf.fillHistory == 0)) {
type = STREAM_RECALCUL_TASK; // only the recalculating task
} else {
type = STREAM_HISTORY_TASK; // set the fill-history option
}
code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, type,
useTriggerParam, hasAggTasks);
if (code != 0) {
sdbRelease(pSdb, pVgroup);
return code;
@ -506,33 +545,35 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
sdbRelease(pSdb, pVgroup);
}
if (pStream->conf.fillHistory) {
if (needHistoryTask(pStream)) {
setHTasksId(pStream);
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam,
static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, EStreamTaskType type, bool useTriggerParam,
SStreamTask** pAggTask) {
*pAggTask = NULL;
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
uint64_t uid = 0;
SArray** pTaskList = NULL;
streamGetUidTaskList(pStream, type, &uid, &pTaskList);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, pStream->conf.trigger,
useTriggerParam ? pStream->conf.triggerParam : 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, pAggTask);
int64_t triggerParam = useTriggerParam? pStream->conf.triggerParam:0;
int32_t code = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, type, pStream->conf.trigger,
triggerParam, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, 1, pAggTask);
return code;
}
static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup,
SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) {
SSnodeObj* pSnode, EStreamTaskType type, bool useTriggerParam) {
int32_t code = 0;
SStreamTask* pTask = NULL;
const char* id = NULL;
code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
code = buildAggTask(pStream, pEpset, type, useTriggerParam, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -540,10 +581,10 @@ static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan,
id = pTask->id.idStr;
if (pSnode != NULL) {
code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode);
mDebug("doAddAggTask taskId:%s, %p snode id:%d, isFillHistory:%d", id, pTask, pSnode->id, isFillhistory);
mDebug("doAddAggTask taskId:%s, %p snode id:%d, isFillHistory:%d", id, pTask, pSnode->id, (type == STREAM_HISTORY_TASK));
} else {
code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
mDebug("doAddAggTask taskId:%s, %p vgId:%d, isFillHistory:%d", id, pTask, pVgroup->vgId, isFillhistory);
mDebug("doAddAggTask taskId:%s, %p vgId:%d, isFillHistory:%d", id, pTask, pVgroup->vgId, (type == STREAM_HISTORY_TASK));
}
return code;
}
@ -561,13 +602,15 @@ static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, S
pVgroup = mndSchedFetchOneVg(pMnode, pStream);
}
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, false, useTriggerParam);
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, STREAM_NORMAL_TASK, useTriggerParam);
if (code != 0) {
goto END;
}
if (pStream->conf.fillHistory) {
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, true, useTriggerParam);
if (needHistoryTask(pStream)) {
EStreamTaskType type = (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) ? STREAM_RECALCUL_TASK
: STREAM_HISTORY_TASK;
code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, type, useTriggerParam);
if (code != 0) {
goto END;
}
@ -602,9 +645,10 @@ static int32_t addSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset)
}
}
if (pStream->conf.fillHistory) {
if (needHistoryTask(pStream)) {
setHTasksId(pStream);
}
return TDB_CODE_SUCCESS;
}
@ -683,6 +727,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
bool hasExtraSink = false;
bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
if (pDbObj == NULL) {
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
@ -694,9 +739,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
mDebug("doScheduleStream numOfPlanLevel:%d, exDb:%d, multiTarget:%d, fix vgId:%d, physicalPlan:%s", numOfPlanLevel,
externalTargetDB, multiTarget, pStream->fixedSinkVgId, pStream->physicalPlan);
pStream->tasks = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
pStream->pHTasksList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
if (pStream->tasks == NULL || pStream->pHTasksList == NULL) {
pStream->pTaskList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
pStream->pHTaskList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
if (pStream->pTaskList == NULL || pStream->pHTaskList == NULL) {
mError("failed to create stream obj, code:%s", tstrerror(terrno));
return terrno;
}
@ -719,15 +764,16 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
TAOS_RETURN(code);
}
code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, (numOfPlanLevel == 1));
int8_t hasAggTasks = (numOfPlanLevel > 1) ? 1 : 0; // task level is greater than 1, which means agg existing
code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, (numOfPlanLevel == 1), hasAggTasks);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (numOfPlanLevel == 1) {
bindSourceSink(pStream, pMnode, pStream->tasks, hasExtraSink);
if (pStream->conf.fillHistory) {
bindSourceSink(pStream, pMnode, pStream->pHTasksList, hasExtraSink);
bindSourceSink(pStream, pMnode, pStream->pTaskList, hasExtraSink);
if (needHistoryTask(pStream)) {
bindSourceSink(pStream, pMnode, pStream->pHTaskList, hasExtraSink);
}
return TDB_CODE_SUCCESS;
}
@ -741,7 +787,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
}
do {
SArray** list = taosArrayGetLast(pStream->tasks);
SArray** list = taosArrayGetLast(pStream->pTaskList);
float size = (float)taosArrayGetSize(*list);
size_t cnt = (size_t)ceil(size / tsStreamAggCnt);
if (cnt <= 1) break;
@ -758,9 +804,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
return code;
}
bindTwoLevel(pStream->tasks, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
if (pStream->conf.fillHistory) {
bindTwoLevel(pStream->pHTasksList, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
bindTwoLevel(pStream->pTaskList, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
if (needHistoryTask(pStream)) {
bindTwoLevel(pStream->pHTaskList, j * tsStreamAggCnt, (j + 1) * tsStreamAggCnt);
}
}
} while (1);
@ -774,7 +820,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
}
mDebug("doScheduleStream add final agg");
SArray** list = taosArrayGetLast(pStream->tasks);
SArray** list = taosArrayGetLast(pStream->pTaskList);
size_t size = taosArrayGetSize(*list);
code = addNewTaskList(pStream);
@ -786,14 +832,14 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (code != TSDB_CODE_SUCCESS) {
TAOS_RETURN(code);
}
bindTwoLevel(pStream->tasks, 0, size);
if (pStream->conf.fillHistory) {
bindTwoLevel(pStream->pHTasksList, 0, size);
bindTwoLevel(pStream->pTaskList, 0, size);
if (needHistoryTask(pStream)) {
bindTwoLevel(pStream->pHTaskList, 0, size);
}
bindAggSink(pStream, pMnode, pStream->tasks);
if (pStream->conf.fillHistory) {
bindAggSink(pStream, pMnode, pStream->pHTasksList);
bindAggSink(pStream, pMnode, pStream->pTaskList);
if (needHistoryTask(pStream)) {
bindAggSink(pStream, pMnode, pStream->pHTaskList);
}
TAOS_RETURN(code);
}

8
source/dnode/mnode/impl/src/mndSma.c
View File

@ -1595,7 +1595,7 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) {
if (!pCxt->pCreateStreamReq->pTags) {
return terrno;
}
SField f = {0};
SFieldWithOptions f = {0};
int32_t code = 0;
if (pCxt->pSrcStb) {
for (int32_t idx = 0; idx < pCxt->pCreateStreamReq->numOfTags - 1; ++idx) {
@ -1630,6 +1630,10 @@ static int32_t mndCreateTSMABuildCreateStreamReq(SCreateTSMACxt *pCxt) {
f.type = pExprNode->resType.type;
f.flags = COL_SMA_ON;
tstrncpy(f.name, pExprNode->userAlias, TSDB_COL_NAME_LEN);
if (IS_DECIMAL_TYPE(f.type)) {
f.typeMod = decimalCalcTypeMod(pExprNode->resType.precision, pExprNode->resType.scale);
f.flags |= COL_HAS_TYPE_MOD;
}
if (NULL == taosArrayPush(pCxt->pCreateStreamReq->pCols, &f)) {
code = terrno;
break;
@ -1797,7 +1801,7 @@ static int32_t mndCreateTSMA(SCreateTSMACxt *pCxt) {
}
}
if (LIST_LENGTH(pProjects) > 0) {
createStreamReq.pCols = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SField));
createStreamReq.pCols = taosArrayInit(LIST_LENGTH(pProjects), sizeof(SFieldWithOptions));
if (!createStreamReq.pCols) {
code = terrno;
goto _OVER;

163
source/dnode/mnode/impl/src/mndStb.c
View File

@ -117,11 +117,12 @@ void mndCleanupStb(SMnode *pMnode) {}
SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
int32_t code = 0;
int32_t lino = 0;
bool hasTypeMod = false;
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + pStb->commentLen +
pStb->ast1Len + pStb->ast2Len + pStb->numOfColumns * sizeof(SColCmpr) + STB_RESERVE_SIZE +
taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN;
taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN + sizeof(int32_t) * pStb->numOfColumns;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER;
@ -155,6 +156,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER)
SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
hasTypeMod = hasTypeMod || HAS_TYPE_MOD(pSchema);
}
for (int32_t i = 0; i < pStb->numOfTags; ++i) {
@ -192,6 +194,13 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
}
SDB_SET_INT64(pRaw, dataPos, pStb->keep, _OVER)
if (hasTypeMod) {
for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
SDB_SET_INT32(pRaw, dataPos, pStb->pExtSchemas[i].typeMod, _OVER);
}
}
SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -214,6 +223,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow *pRow = NULL;
SStbObj *pStb = NULL;
bool hasExtSchemas = false;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -266,6 +276,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
hasExtSchemas = hasExtSchemas || HAS_TYPE_MOD(pSchema);
}
for (int32_t i = 0; i < pStb->numOfTags; ++i) {
@ -320,6 +331,16 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
}
SDB_GET_INT64(pRaw, dataPos, &pStb->keep, _OVER)
// type mod
if (hasExtSchemas) {
pStb->pExtSchemas = taosMemoryCalloc(pStb->numOfColumns, sizeof(SExtSchema));
if (!pStb->pExtSchemas) goto _OVER;
for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
SSchema *pSchema = &pStb->pColumns[i];
SDB_GET_INT32(pRaw, dataPos, &pStb->pExtSchemas[i].typeMod, _OVER)
}
}
SDB_GET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER)
terrno = 0;
@ -332,6 +353,7 @@ _OVER:
taosMemoryFreeClear(pStb->pTags);
taosMemoryFreeClear(pStb->comment);
taosMemoryFree(pStb->pCmpr);
taosMemoryFreeClear(pStb->pExtSchemas);
}
taosMemoryFreeClear(pRow);
return NULL;
@ -349,6 +371,7 @@ void mndFreeStb(SStbObj *pStb) {
taosMemoryFreeClear(pStb->pAst1);
taosMemoryFreeClear(pStb->pAst2);
taosMemoryFreeClear(pStb->pCmpr);
taosMemoryFreeClear(pStb->pExtSchemas);
}
static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) {
@ -363,69 +386,55 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) {
}
static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
terrno = 0;
mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew);
taosWLockLatch(&pOld->lock);
int32_t numOfColumns = pOld->numOfColumns;
if (pOld->numOfColumns < pNew->numOfColumns) {
void *pColumns = taosMemoryMalloc(pNew->numOfColumns * sizeof(SSchema));
if (pColumns != NULL) {
if (pColumns == NULL) {
goto END;
}
taosMemoryFree(pOld->pColumns);
pOld->pColumns = pColumns;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
taosWUnLockLatch(&pOld->lock);
}
}
if (pOld->numOfTags < pNew->numOfTags) {
void *pTags = taosMemoryMalloc(pNew->numOfTags * sizeof(SSchema));
if (pTags != NULL) {
if (pTags == NULL) {
goto END;
}
taosMemoryFree(pOld->pTags);
pOld->pTags = pTags;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
taosWUnLockLatch(&pOld->lock);
}
}
if (pOld->commentLen < pNew->commentLen && pNew->commentLen > 0) {
void *comment = taosMemoryMalloc(pNew->commentLen + 1);
if (comment != NULL) {
if (comment == NULL) {
goto END;
}
taosMemoryFree(pOld->comment);
pOld->comment = comment;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
taosWUnLockLatch(&pOld->lock);
}
}
pOld->commentLen = pNew->commentLen;
if (pOld->ast1Len < pNew->ast1Len) {
void *pAst1 = taosMemoryMalloc(pNew->ast1Len + 1);
if (pAst1 != NULL) {
if (pAst1 == NULL) {
goto END;
}
taosMemoryFree(pOld->pAst1);
pOld->pAst1 = pAst1;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
taosWUnLockLatch(&pOld->lock);
}
}
if (pOld->ast2Len < pNew->ast2Len) {
void *pAst2 = taosMemoryMalloc(pNew->ast2Len + 1);
if (pAst2 != NULL) {
if (pAst2 == NULL) {
goto END;
}
taosMemoryFree(pOld->pAst2);
pOld->pAst2 = pAst2;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
taosWUnLockLatch(&pOld->lock);
}
}
pOld->updateTime = pNew->updateTime;
@ -459,13 +468,26 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
if (numOfColumns < pNew->numOfColumns) {
taosMemoryFree(pOld->pCmpr);
pOld->pCmpr = taosMemoryCalloc(pNew->numOfColumns, sizeof(SColCmpr));
if (pOld->pCmpr == NULL){
goto END;
}
memcpy(pOld->pCmpr, pNew->pCmpr, pNew->numOfColumns * sizeof(SColCmpr));
} else {
memcpy(pOld->pCmpr, pNew->pCmpr, pNew->numOfColumns * sizeof(SColCmpr));
}
if (pNew->pExtSchemas) {
taosMemoryFreeClear(pOld->pExtSchemas);
pOld->pExtSchemas = taosMemoryCalloc(pNew->numOfColumns, sizeof(SExtSchema));
if (pOld->pExtSchemas == NULL){
goto END;
}
memcpy(pOld->pExtSchemas, pNew->pExtSchemas, pNew->numOfColumns * sizeof(SExtSchema));
}
END:
taosWUnLockLatch(&pOld->lock);
return 0;
return terrno;
}
SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) {
@ -563,6 +585,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3
}
}
}
req.pExtSchemas = pStb->pExtSchemas; // only reference to it.
// get length
int32_t ret = 0;
tEncodeSize(tEncodeSVCreateStbReq, &req, contLen, ret);
@ -861,6 +884,7 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) {
int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) {
int32_t code = 0;
bool hasTypeMods = false;
memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN);
memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
pDst->createdTime = taosGetTimestampMs();
@ -937,6 +961,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
pSchema->colId = pDst->nextColId;
pDst->nextColId++;
hasTypeMods = hasTypeMods || HAS_TYPE_MOD(pSchema);
}
for (int32_t i = 0; i < pDst->numOfTags; ++i) {
@ -961,6 +986,18 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
pColCmpr->id = pSchema->colId;
pColCmpr->alg = pField->compress;
}
if (hasTypeMods) {
pDst->pExtSchemas = taosMemoryCalloc(pDst->numOfColumns, sizeof(SExtSchema));
if (!pDst->pExtSchemas) {
code = terrno;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
SFieldWithOptions * pField = taosArrayGet(pCreate->pColumns, i);
pDst->pExtSchemas[i].typeMod = pField->typeMod;
}
}
TAOS_RETURN(code);
}
static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) {
@ -1204,8 +1241,9 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema));
pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema));
pDst->pCmpr = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SColCmpr));
pDst->pExtSchemas = taosMemoryCalloc(pDst->numOfColumns, sizeof(SExtSchema));
if (pDst->pColumns == NULL || pDst->pTags == NULL || pDst->pCmpr == NULL) {
if (pDst->pColumns == NULL || pDst->pTags == NULL || pDst->pCmpr == NULL || pDst->pExtSchemas == NULL) {
code = terrno;
TAOS_RETURN(code);
}
@ -1253,6 +1291,9 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
} else {
p->alg = pField->compress;
}
if (pField->flags & COL_HAS_TYPE_MOD) {
pDst->pExtSchemas[i].typeMod = pField->typeMod;
}
}
pDst->tagVer = createReq->tagVer;
pDst->colVer = createReq->colVer;
@ -1385,6 +1426,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
taosMemoryFreeClear(pDst.pTags);
taosMemoryFreeClear(pDst.pColumns);
taosMemoryFreeClear(pDst.pCmpr);
taosMemoryFreeClear(pDst.pExtSchemas);
goto _OVER;
}
@ -1392,6 +1434,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
taosMemoryFreeClear(pDst.pTags);
taosMemoryFreeClear(pDst.pColumns);
taosMemoryFreeClear(pDst.pCmpr);
taosMemoryFreeClear(pDst.pExtSchemas);
} else {
code = mndCreateStb(pMnode, pReq, &createReq, pDb);
}
@ -1458,6 +1501,13 @@ int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) {
memcpy(pNew->pColumns, pOld->pColumns, sizeof(SSchema) * pOld->numOfColumns);
memcpy(pNew->pTags, pOld->pTags, sizeof(SSchema) * pOld->numOfTags);
memcpy(pNew->pCmpr, pOld->pCmpr, sizeof(SColCmpr) * pOld->numOfColumns);
if (pOld->pExtSchemas) {
pNew->pExtSchemas = taosMemoryCalloc(pNew->numOfColumns, sizeof(SExtSchema));
if (pNew->pExtSchemas == NULL) {
TAOS_RETURN(terrno);
}
memcpy(pNew->pExtSchemas, pOld->pExtSchemas, sizeof(SExtSchema) * pOld->numOfColumns);
}
TAOS_RETURN(0);
}
@ -1880,7 +1930,7 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
TAOS_RETURN(code);
}
static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols,
static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const SMAlterStbReq* pReq, int32_t ncols,
int8_t withCompress) {
int32_t code = 0;
if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) {
@ -1892,7 +1942,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
TAOS_RETURN(code);
}
if (!mndValidateSchema(pOld->pColumns, pOld->numOfColumns, pFields, TSDB_MAX_BYTES_PER_ROW)) {
if (!mndValidateSchema(pOld->pColumns, pOld->numOfColumns, pReq->pFields, TSDB_MAX_BYTES_PER_ROW)) {
code = TSDB_CODE_PAR_INVALID_ROW_LENGTH;
TAOS_RETURN(code);
}
@ -1908,7 +1958,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
for (int32_t i = 0; i < ncols; i++) {
if (withCompress) {
SFieldWithOptions *pField = taosArrayGet(pFields, i);
SFieldWithOptions *pField = taosArrayGet(pReq->pFields, i);
if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
TAOS_RETURN(code);
@ -1931,7 +1981,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
pCmpr->alg = pField->compress;
mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
} else {
SField *pField = taosArrayGet(pFields, i);
SField *pField = taosArrayGet(pReq->pFields, i);
if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
code = TSDB_CODE_MND_COLUMN_ALREADY_EXIST;
TAOS_RETURN(code);
@ -1955,6 +2005,25 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
}
}
// 1. old schema already has extschemas
// 2. new schema has extschemas
if (pReq->pTypeMods || pOld->pExtSchemas) {
if (!pNew->pExtSchemas) {
// all ext schemas reset to zero
pNew->pExtSchemas = taosMemoryCalloc(pNew->numOfColumns, sizeof(SExtSchema));
if (!pNew->pExtSchemas) TAOS_RETURN(terrno);
}
if (pOld->pExtSchemas) {
memcpy(pNew->pExtSchemas, pOld->pExtSchemas, pOld->numOfColumns * sizeof(SExtSchema));
}
if (taosArrayGetSize(pReq->pTypeMods) > 0) {
// copy added column ext schema
for (int32_t i = 0; i < ncols; ++i) {
pNew->pColumns[pOld->numOfColumns + i].flags |= COL_HAS_TYPE_MOD;
pNew->pExtSchemas[pOld->numOfColumns + i].typeMod = *(STypeMod *)taosArrayGet(pReq->pTypeMods, i);
}
}
}
pNew->colVer++;
TAOS_RETURN(code);
@ -1986,6 +2055,9 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb
int32_t sz = pNew->numOfColumns - col - 1;
memmove(pNew->pColumns + col, pNew->pColumns + col + 1, sizeof(SSchema) * sz);
memmove(pNew->pCmpr + col, pNew->pCmpr + col + 1, sizeof(SColCmpr) * sz);
if (pOld->pExtSchemas) {
memmove(pNew->pExtSchemas + col, pNew->pExtSchemas + col + 1, sizeof(SExtSchema) * sz);
}
pNew->numOfColumns--;
pNew->colVer++;
@ -2206,6 +2278,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa
SSchemaExt *pSchEx = &pRsp->pSchemaExt[i];
pSchEx->colId = pCmpr->id;
pSchEx->compress = pCmpr->alg;
if (pStb->pExtSchemas) {
pSchEx->typeMod = pStb->pExtSchemas[i].typeMod;
}
}
taosRUnLockLatch(&pStb->lock);
@ -2271,6 +2346,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
SSchemaExt *pSchExt = &pRsp->pSchemaExt[i];
pSchExt->colId = pCmpr->id;
pSchExt->compress = pCmpr->alg;
if (pStb->pExtSchemas) {
pSchExt->typeMod = pStb->pExtSchemas[i].typeMod;
}
}
taosRUnLockLatch(&pStb->lock);
@ -2604,6 +2682,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
stbObj.pTags = NULL;
stbObj.pFuncs = NULL;
stbObj.pCmpr = NULL;
stbObj.pExtSchemas = NULL;
stbObj.updateTime = taosGetTimestampMs();
stbObj.lock = 0;
bool updateTagIndex = false;
@ -2625,7 +2704,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0);
break;
case TSDB_ALTER_TABLE_ADD_COLUMN:
code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 0);
code = mndAddSuperTableColumn(pOld, &stbObj, pAlter, pAlter->numOfFields, 0);
break;
case TSDB_ALTER_TABLE_DROP_COLUMN:
pField0 = taosArrayGet(pAlter->pFields, 0);
@ -2643,7 +2722,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
code = mndUpdateSuperTableColumnCompress(pMnode, pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
break;
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION:
code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 1);
code = mndAddSuperTableColumn(pOld, &stbObj, pAlter, pAlter->numOfFields, 1);
break;
default:
needRsp = false;
@ -2665,6 +2744,7 @@ _OVER:
if (pAlter->commentLen > 0) {
taosMemoryFreeClear(stbObj.comment);
}
taosMemoryFreeClear(stbObj.pExtSchemas);
TAOS_RETURN(code);
}
@ -3646,6 +3726,11 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colTypeLen +=
tsnprintf(varDataVal(colTypeStr) + colTypeLen, sizeof(colTypeStr) - colTypeLen - VARSTR_HEADER_SIZE,
"(%d)", (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
} else if (IS_DECIMAL_TYPE(colType)) {
STypeMod typeMod = pStb->pExtSchemas[i].typeMod;
uint8_t prec = 0, scale = 0;
decimalFromTypeMod(typeMod, &prec, &scale);
colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d,%d)", prec, scale);
}
varDataSetLen(colTypeStr, colTypeLen);
RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (char *)colTypeStr, false), pStb, &lino, _OVER);

47
source/dnode/mnode/impl/src/mndStream.c
View File

@ -250,13 +250,13 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
pOldStream->updateTime = pNewStream->updateTime;
pOldStream->checkpointId = pNewStream->checkpointId;
pOldStream->checkpointFreq = pNewStream->checkpointFreq;
if (pOldStream->tasks == NULL){
pOldStream->tasks = pNewStream->tasks;
pNewStream->tasks = NULL;
if (pOldStream->pTaskList == NULL) {
pOldStream->pTaskList = pNewStream->pTaskList;
pNewStream->pTaskList = NULL;
}
if (pOldStream->pHTasksList == NULL){
pOldStream->pHTasksList = pNewStream->pHTasksList;
pNewStream->pHTasksList = NULL;
if (pOldStream->pHTaskList == NULL) {
pOldStream->pHTaskList = pNewStream->pHTaskList;
pNewStream->pHTaskList = NULL;
}
taosWUnLockLatch(&pOldStream->lock);
return 0;
@ -478,7 +478,17 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
.deleteMark = pObj->deleteMark,
.igCheckUpdate = pObj->igCheckUpdate,
.destHasPrimaryKey = hasDestPrimaryKey(&pObj->outputSchema),
.recalculateInterval = pCreate->recalculateInterval,
};
char *pTargetFStable = strchr(pCreate->targetStbFullName, '.');
if (pTargetFStable != NULL) {
pTargetFStable = pTargetFStable + 1;
}
tstrncpy(cxt.pStbFullName, pTargetFStable, TSDB_TABLE_FNAME_LEN);
tstrncpy(cxt.pWstartName, pCreate->pWstartName, TSDB_COL_NAME_LEN);
tstrncpy(cxt.pWendName, pCreate->pWendName, TSDB_COL_NAME_LEN);
tstrncpy(cxt.pGroupIdName, pCreate->pGroupIdName, TSDB_COL_NAME_LEN);
tstrncpy(cxt.pIsWindowFilledName, pCreate->pIsWindowFilledName, TSDB_COL_NAME_LEN);
// using ast and param to build physical plan
if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) < 0) {
@ -590,11 +600,11 @@ int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) {
destroyStreamTaskIter(pIter);
// persistent stream task for already stored ts data
if (pStream->conf.fillHistory) {
int32_t level = taosArrayGetSize(pStream->pHTasksList);
if (pStream->conf.fillHistory || (pStream->conf.trigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE)) {
int32_t level = taosArrayGetSize(pStream->pHTaskList);
for (int32_t i = 0; i < level; i++) {
SArray *pLevel = taosArrayGetP(pStream->pHTasksList, i);
SArray *pLevel = taosArrayGetP(pStream->pHTaskList, i);
int32_t numOfTasks = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < numOfTasks; j++) {
@ -642,6 +652,11 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
pField->type = pStream->outputSchema.pSchema[i].type;
pField->bytes = pStream->outputSchema.pSchema[i].bytes;
pField->compress = createDefaultColCmprByType(pField->type);
if (IS_DECIMAL_TYPE(pField->type)) {
uint8_t prec = 0, scale = 0;
extractDecimalTypeInfoFromBytes(&pField->bytes, &prec, &scale);
pField->typeMod = decimalCalcTypeMod(prec, scale);
}
}
if (pStream->tagSchema.nCols == 0) {
@ -807,9 +822,9 @@ static int32_t addStreamNotifyInfo(SCMCreateStreamReq *createReq, SStreamObj *pS
goto _end;
}
level = taosArrayGetSize(pStream->tasks);
level = taosArrayGetSize(pStream->pTaskList);
for (int32_t i = 0; i < level; ++i) {
pLevel = taosArrayGetP(pStream->tasks, i);
pLevel = taosArrayGetP(pStream->pTaskList, i);
nTasks = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < nTasks; ++j) {
code = addStreamTaskNotifyInfo(createReq, pStream, taosArrayGetP(pLevel, j));
@ -818,9 +833,9 @@ static int32_t addStreamNotifyInfo(SCMCreateStreamReq *createReq, SStreamObj *pS
}
if (pStream->conf.fillHistory && createReq->notifyHistory) {
level = taosArrayGetSize(pStream->pHTasksList);
level = taosArrayGetSize(pStream->pHTaskList);
for (int32_t i = 0; i < level; ++i) {
pLevel = taosArrayGetP(pStream->pHTasksList, i);
pLevel = taosArrayGetP(pStream->pHTaskList, i);
nTasks = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < nTasks; ++j) {
code = addStreamTaskNotifyInfo(createReq, pStream, taosArrayGetP(pLevel, j));
@ -916,7 +931,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
code = mndAcquireStream(pMnode, createReq.name, &pStream);
if (pStream != NULL && code == 0) {
if (pStream->tasks != NULL){
if (pStream->pTaskList != NULL){
if (createReq.igExists) {
mInfo("stream:%s, already exist, ignore exist is set", createReq.name);
mndReleaseStream(pMnode, pStream);
@ -1245,9 +1260,9 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
pStream->currentTick = 1;
// 1. redo action: broadcast checkpoint source msg for all source vg
int32_t totalLevel = taosArrayGetSize(pStream->tasks);
int32_t totalLevel = taosArrayGetSize(pStream->pTaskList);
for (int32_t i = 0; i < totalLevel; i++) {
SArray *pLevel = taosArrayGetP(pStream->tasks, i);
SArray *pLevel = taosArrayGetP(pStream->pTaskList, i);
SStreamTask *p = taosArrayGetP(pLevel, 0);
if (p->info.taskLevel == TASK_LEVEL__SOURCE) {

10
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -40,7 +40,7 @@ int32_t createStreamTaskIter(SStreamObj *pStream, SStreamTaskIter **pIter) {
(*pIter)->level = -1;
(*pIter)->ordinalIndex = 0;
(*pIter)->pStream = pStream;
(*pIter)->totalLevel = taosArrayGetSize(pStream->tasks);
(*pIter)->totalLevel = taosArrayGetSize(pStream->pTaskList);
(*pIter)->pTask = NULL;
return 0;
@ -57,7 +57,7 @@ bool streamTaskIterNextTask(SStreamTaskIter *pIter) {
}
while (pIter->level < pIter->totalLevel) {
SArray *pList = taosArrayGetP(pIter->pStream->tasks, pIter->level);
SArray *pList = taosArrayGetP(pIter->pStream->pTaskList, pIter->level);
if (pIter->ordinalIndex >= taosArrayGetSize(pList)) {
pIter->level += 1;
pIter->ordinalIndex = 0;
@ -394,8 +394,8 @@ int32_t mndGetStreamTask(STaskId *pId, SStreamObj *pStream, SStreamTask **pTask)
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
int32_t num = 0;
for (int32_t i = 0; i < taosArrayGetSize(pStream->tasks); ++i) {
SArray *pLevel = taosArrayGetP(pStream->tasks, i);
for (int32_t i = 0; i < taosArrayGetSize(pStream->pTaskList); ++i) {
SArray *pLevel = taosArrayGetP(pStream->pTaskList, i);
num += taosArrayGetSize(pLevel);
}
@ -1110,7 +1110,7 @@ int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_
TSDB_CHECK_CODE(code, lino, _end);
int8_t streamStatus = atomic_load_8(&pStream->status);
if (isPaused && pStream->tasks != NULL) {
if (isPaused && pStream->pTaskList != NULL) {
streamStatus = STREAM_STATUS__PAUSE;
}
mndShowStreamStatus(status2, streamStatus);

6
source/dnode/mnode/impl/test/stream/stream.cpp
View File

@ -229,8 +229,8 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
pStream->uid = defStreamId;
pStream->lock = 0;
pStream->tasks = taosArrayInit(1, POINTER_BYTES);
pStream->pHTasksList = taosArrayInit(1, POINTER_BYTES);
pStream->pTaskList = taosArrayInit(1, POINTER_BYTES);
pStream->pHTaskList = taosArrayInit(1, POINTER_BYTES);
SArray* pLevel = taosArrayInit(1, POINTER_BYTES);
SStreamTask* pTask = static_cast<SStreamTask*>(taosMemoryCalloc(1, sizeof(SStreamTask)));
@ -243,7 +243,7 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
void* px = taosArrayPush(pLevel, &pTask);
ASSERT(px != NULL);
px = taosArrayPush(pStream->tasks, &pLevel);
px = taosArrayPush(pStream->pTaskList, &pLevel);
ASSERT(px != NULL);
code = mndCreateStreamResetStatusTrans(pMnode, pStream, 1);

6
source/dnode/snode/src/snode.c
View File

@ -43,17 +43,17 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
tqSetRestoreVersionInfo(pTask);
char *p = streamTaskGetStatus(pTask).name;
if (pTask->info.fillHistory) {
if (pTask->info.fillHistory == STREAM_HISTORY_TASK) {
sndInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
" child id:%d, level:%d, status:%s taskType:%d, related stream task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam);
} else {
sndInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
" child id:%d, level:%d, status:%s taskType:%d, related helper-task:0x%x trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam);

29
source/dnode/snode/src/snodeInitApi.c
View File

@ -32,6 +32,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStatePutParName = streamStatePutParName;
pStore->streamStateGetParName = streamStateGetParName;
pStore->streamStateDeleteParName = streamStateDeleteParName;
pStore->streamStateSetParNameInvalid = streamStateSetParNameInvalid;
pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
pStore->streamStateReleaseBuf = streamStateReleaseBuf;
@ -47,8 +48,11 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateClear = streamStateClear;
pStore->streamStateSaveInfo = streamStateSaveInfo;
pStore->streamStateGetInfo = streamStateGetInfo;
pStore->streamStateGetNumber = streamStateGetNumber;
pStore->streamStateDeleteInfo = streamStateDeleteInfo;
pStore->streamStateSetNumber = streamStateSetNumber;
pStore->streamStateGetPrev = streamStateGetPrev;
pStore->streamStateGetAllPrev = streamStateGetAllPrev;
pStore->streamStateFillPut = streamStateFillPut;
pStore->streamStateFillGet = streamStateFillGet;
@ -70,6 +74,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
pStore->streamStateClearExpiredSessionState = streamStateClearExpiredSessionState;
pStore->streamStateSetRecFlag = streamStateSetRecFlag;
pStore->streamStateGetRecFlag = streamStateGetRecFlag;
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
pStore->streamStateSessionPut = streamStateSessionPut;
@ -82,6 +89,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange;
pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition;
pStore->streamStateSessionSaveToDisk = streamStateSessionSaveToDisk;
pStore->streamStateFlushReaminInfoToDisk = streamStateFlushReaminInfoToDisk;
pStore->streamStateSessionDeleteAll = streamStateSessionDeleteAll;
pStore->updateInfoInit = updateInfoInit;
pStore->updateInfoFillBlockData = updateInfoFillBlockData;
@ -113,7 +123,26 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateClear = streamFileStateClear;
pStore->needClearDiskBuff = needClearDiskBuff;
pStore->streamStateGetAndSetTsData = streamStateGetAndSetTsData;
pStore->streamStateTsDataCommit = streamStateTsDataCommit;
pStore->streamStateInitTsDataState = streamStateInitTsDataState;
pStore->streamStateDestroyTsDataState = streamStateDestroyTsDataState;
pStore->streamStateRecoverTsData = streamStateRecoverTsData;
pStore->streamStateReloadTsDataState = streamStateReloadTsDataState;
pStore->streamStateMergeAndSaveScanRange = streamStateMergeAndSaveScanRange;
pStore->streamStateMergeAllScanRange = streamStateMergeAllScanRange;
pStore->streamStatePopScanRange = streamStatePopScanRange;
pStore->streamStateCheckSessionState = streamStateCheckSessionState;
pStore->streamStateGetLastStateCur = streamStateGetLastStateCur;
pStore->streamStateLastStateCurNext = streamStateLastStateCurNext;
pStore->streamStateNLastStateGetKVByCur = streamStateNLastStateGetKVByCur;
pStore->streamStateGetLastSessionStateCur = streamStateGetLastSessionStateCur;
pStore->streamStateLastSessionStateCurNext = streamStateLastSessionStateCurNext;
pStore->streamStateNLastSessionStateGetKVByCur = streamStateNLastSessionStateGetKVByCur;
pStore->streamStateOpen = streamStateOpen;
pStore->streamStateRecalatedOpen = streamStateRecalatedOpen;
pStore->streamStateClose = streamStateClose;
pStore->streamStateBegin = streamStateBegin;
pStore->streamStateCommit = streamStateCommit;

3
source/dnode/vnode/inc/vnode.h
View File

@ -185,6 +185,8 @@ void tsdbReaderSetCloseFlag(STsdbReader *pReader);
int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
void tsdbSetFilesetDelimited(STsdbReader *pReader);
void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param);
int32_t tsdbReaderGetProgress(const STsdbReader *pReader, void **pBuf, uint64_t *pLen);
int32_t tsdbReaderSetProgress(STsdbReader *pReader, const void *buf, uint64_t len);
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
@ -221,6 +223,7 @@ typedef struct STqReader {
SSDataBlock *pResBlock;
int64_t lastTs;
bool hasPrimaryKey;
SExtSchema *extSchema;
} STqReader;
STqReader *tqReaderOpen(SVnode *pVnode);

3
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -169,8 +169,9 @@ int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tb
int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount);
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, SExtSchema** extSchema);
int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock);
SExtSchema* metaGetSExtSchema(const SMetaEntry *pME);
int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);

92
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -15,6 +15,83 @@
#include "meta.h"
static bool schemasHasTypeMod(const SSchema *pSchema, int32_t nCols) {
for (int32_t i = 0; i < nCols; i++) {
if (HAS_TYPE_MOD(pSchema + i)) {
return true;
}
}
return false;
}
static int32_t metaEncodeExtSchema(SEncoder* pCoder, const SMetaEntry* pME) {
if (pME->pExtSchemas) {
const SSchemaWrapper *pSchWrapper = NULL;
bool hasTypeMods = false;
if (pME->type == TSDB_SUPER_TABLE) {
pSchWrapper = &pME->stbEntry.schemaRow;
} else if (pME->type == TSDB_NORMAL_TABLE) {
pSchWrapper = &pME->ntbEntry.schemaRow;
} else {
return 0;
}
hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols);
for (int32_t i = 0; i < pSchWrapper->nCols && hasTypeMods; ++i) {
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->pExtSchemas[i].typeMod));
}
}
return 0;
}
static int32_t metaDecodeExtSchemas(SDecoder* pDecoder, SMetaEntry* pME) {
bool hasExtSchema = false;
SSchemaWrapper* pSchWrapper = NULL;
if (pME->type == TSDB_SUPER_TABLE) {
pSchWrapper = &pME->stbEntry.schemaRow;
} else if (pME->type == TSDB_NORMAL_TABLE) {
pSchWrapper = &pME->ntbEntry.schemaRow;
} else {
return 0;
}
hasExtSchema = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols);
if (hasExtSchema && pSchWrapper->nCols > 0) {
pME->pExtSchemas = (SExtSchema*)tDecoderMalloc(pDecoder, sizeof(SExtSchema) * pSchWrapper->nCols);
if (pME->pExtSchemas == NULL) {
return terrno;
}
for (int32_t i = 0; i < pSchWrapper->nCols && hasExtSchema; i++) {
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pME->pExtSchemas[i].typeMod));
}
}
return 0;
}
SExtSchema* metaGetSExtSchema(const SMetaEntry *pME) {
const SSchemaWrapper *pSchWrapper = NULL;
bool hasTypeMods = false;
if (pME->type == TSDB_SUPER_TABLE) {
pSchWrapper = &pME->stbEntry.schemaRow;
} else if (pME->type == TSDB_NORMAL_TABLE) {
pSchWrapper = &pME->ntbEntry.schemaRow;
} else {
return NULL;
}
hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols);
if (hasTypeMods) {
SExtSchema* ret = taosMemoryMalloc(sizeof(SExtSchema) * pSchWrapper->nCols);
if (ret != NULL){
memcpy(ret, pME->pExtSchemas, pSchWrapper->nCols * sizeof(SExtSchema));
}
return ret;
}
return NULL;
}
int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) {
const SColCmprWrapper *pw = &pME->colCmpr;
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols));
@ -129,6 +206,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
return TSDB_CODE_INVALID_PARA;
}
TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME));
TAOS_CHECK_RETURN(metaEncodeExtSchema(pCoder, pME));
}
if (pME->type == TSDB_SUPER_TABLE) {
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->stbEntry.keep));
@ -211,6 +289,9 @@ int metaDecodeEntryImpl(SDecoder *pCoder, SMetaEntry *pME, bool headerOnly) {
}
TABLE_SET_COL_COMPRESSED(pME->flags);
}
if (!tDecodeIsEnd(pCoder)) {
TAOS_CHECK_RETURN(metaDecodeExtSchemas(pCoder, pME));
}
}
if (pME->type == TSDB_SUPER_TABLE) {
if (!tDecodeIsEnd(pCoder)) {
@ -218,6 +299,7 @@ int metaDecodeEntryImpl(SDecoder *pCoder, SMetaEntry *pME, bool headerOnly) {
}
}
tEndDecode(pCoder);
return 0;
}
@ -270,6 +352,7 @@ void metaCloneEntryFree(SMetaEntry **ppEntry) {
return;
}
metaCloneColCmprFree(&(*ppEntry)->colCmpr);
taosMemoryFreeClear((*ppEntry)->pExtSchemas);
taosMemoryFreeClear(*ppEntry);
return;
@ -377,6 +460,15 @@ int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry) {
metaCloneEntryFree(ppEntry);
return code;
}
if (pEntry->pExtSchemas && pEntry->colCmpr.nCols > 0) {
(*ppEntry)->pExtSchemas = taosMemoryCalloc(pEntry->colCmpr.nCols, sizeof(SExtSchema));
if (!(*ppEntry)->pExtSchemas) {
code = terrno;
metaCloneEntryFree(ppEntry);
return code;
}
memcpy((*ppEntry)->pExtSchemas, pEntry->pExtSchemas, sizeof(SExtSchema) * pEntry->colCmpr.nCols);
}
return code;
}

1
source/dnode/vnode/src/meta/metaEntry2.c
View File

@ -1080,7 +1080,6 @@ static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pE
const SMetaHandleParam param = {
.pEntry = pEntry,
};
code = metaTableOpFn[op->table][op->op](pMeta, &param);
if (TSDB_CODE_SUCCESS != code) {
metaErr(TD_VID(pMeta->pVnode), code);

8
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -378,7 +378,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
return 0;
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, SExtSchema** extSchema) {
void *pData = NULL;
int nData = 0;
int64_t version;
@ -409,6 +409,7 @@ _query:
if (me.type == TSDB_SUPER_TABLE) {
if (sver == -1 || sver == me.stbEntry.schemaRow.version) {
pSchema = tCloneSSchemaWrapper(&me.stbEntry.schemaRow);
if (extSchema != NULL) *extSchema = metaGetSExtSchema(&me);
tDecoderClear(&dc);
goto _exit;
}
@ -419,10 +420,12 @@ _query:
} else {
if (sver == -1 || sver == me.ntbEntry.schemaRow.version) {
pSchema = tCloneSSchemaWrapper(&me.ntbEntry.schemaRow);
if (extSchema != NULL) *extSchema = metaGetSExtSchema(&me);
tDecoderClear(&dc);
goto _exit;
}
}
if (extSchema != NULL) *extSchema = metaGetSExtSchema(&me);
tDecoderClear(&dc);
// query from skm db
@ -664,7 +667,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
STSchema *pTSchema = NULL;
SSchemaWrapper *pSW = NULL;
pSW = metaGetTableSchema(pMeta, uid, sver, lock);
pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL);
if (!pSW) return NULL;
pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version);
@ -1465,6 +1468,7 @@ END:
if (pCursor->pMeta) metaULock(pCursor->pMeta);
if (pCursor->pCur) tdbTbcClose(pCursor->pCur);
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
taosMemoryFreeClear(oStbEntry.pExtSchemas);
tDecoderClear(&dc);
tdbFree(pData);

2
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -593,7 +593,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}

36
source/dnode/vnode/src/meta/metaTable.c
View File

@ -69,6 +69,40 @@ int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, u
return 0;
}
int32_t addTableExtSchema(SMetaEntry *pEntry, const SSchema *pColumn, int32_t newColNum, SExtSchema *pExtSchema) {
// no need to add ext schema when no column needs ext schemas
if (!HAS_TYPE_MOD(pColumn) && !pEntry->pExtSchemas) return 0;
if (!pEntry->pExtSchemas) {
// add a column which needs ext schema
// set all extschemas to zero for all columns alrady existed
pEntry->pExtSchemas = (SExtSchema *)taosMemoryCalloc(newColNum, sizeof(SExtSchema));
} else {
// already has columns with ext schema
pEntry->pExtSchemas = (SExtSchema *)taosMemoryRealloc(pEntry->pExtSchemas, sizeof(SExtSchema) * newColNum);
}
if (!pEntry->pExtSchemas) return terrno;
pEntry->pExtSchemas[newColNum - 1] = *pExtSchema;
return 0;
}
int32_t dropTableExtSchema(SMetaEntry *pEntry, int32_t dropColId, int32_t newColNum) {
// no ext schema, no need to drop
if (!pEntry->pExtSchemas) return 0;
if (dropColId == newColNum) {
// drop the last column
pEntry->pExtSchemas[dropColId - 1] = (SExtSchema){0};
} else {
// drop a column in the middle
memmove(pEntry->pExtSchemas + dropColId, pEntry->pExtSchemas + dropColId + 1,
(newColNum - dropColId) * sizeof(SExtSchema));
}
for (int32_t i = 0; i < newColNum; i++) {
if (hasExtSchema(pEntry->pExtSchemas + i)) return 0;
}
taosMemoryFreeClear(pEntry->pExtSchemas);
return 0;
}
int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema));
if (NULL == pMetaRsp->pSchemas) {
@ -812,7 +846,7 @@ int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) {
taosHashClear(pColCmprObj);
return rc;
}
if (useCompress(e.type)) {
if (withExtSchema(e.type)) {
SColCmprWrapper *p = &e.colCmpr;
for (int32_t i = 0; i < p->nCols; i++) {
SColCmpr *pCmpr = &p->pColCmpr[i];

26
source/dnode/vnode/src/meta/metaTable2.c
View File

@ -21,6 +21,8 @@ extern int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEnt
extern int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry **ppEntry);
extern void metaFetchEntryFree(SMetaEntry **ppEntry);
extern int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress);
extern int32_t addTableExtSchema(SMetaEntry* pEntry, const SSchema* pColumn, int32_t newColNum, SExtSchema* pExtSchema);
extern int32_t dropTableExtSchema(SMetaEntry* pEntry, int32_t dropColId, int32_t newColNum);
static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
int32_t vgId = TD_VID(pMeta->pVnode);
@ -190,6 +192,7 @@ int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq
TABLE_SET_COL_COMPRESSED(entry.flags);
entry.colCmpr = pReq->colCmpr;
}
entry.pExtSchemas = pReq->pExtSchemas;
code = metaHandleEntry2(pMeta, &entry);
if (TSDB_CODE_SUCCESS == code) {
@ -436,6 +439,9 @@ static int32_t metaBuildCreateNormalTableRsp(SMeta *pMeta, SMetaEntry *pEntry, S
SColCmpr *p = &pEntry->colCmpr.pColCmpr[i];
(*ppRsp)->pSchemaExt[i].colId = p->id;
(*ppRsp)->pSchemaExt[i].compress = p->alg;
if (pEntry->pExtSchemas) {
(*ppRsp)->pSchemaExt[i].typeMod = pEntry->pExtSchemas[i].typeMod;
}
}
return code;
@ -466,6 +472,7 @@ static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbRe
.ntbEntry.schemaRow = pReq->ntb.schemaRow,
.ntbEntry.ncid = pReq->ntb.schemaRow.pSchema[pReq->ntb.schemaRow.nCols - 1].colId + 1,
.colCmpr = pReq->colCmpr,
.pExtSchemas = pReq->pExtSchemas,
};
TABLE_SET_COL_COMPRESSED(entry.flags);
@ -621,6 +628,7 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST
int32_t rowSize = 0;
SSchemaWrapper *pSchema = &pEntry->ntbEntry.schemaRow;
SSchema *pColumn;
SExtSchema extSchema = {0};
pEntry->version = version;
for (int32_t i = 0; i < pSchema->nCols; i++) {
pColumn = &pSchema->pSchema[i];
@ -655,6 +663,7 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST
pColumn->type = pReq->type;
pColumn->flags = pReq->flags;
pColumn->colId = pEntry->ntbEntry.ncid++;
extSchema.typeMod = pReq->typeMod;
tstrncpy(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN);
uint32_t compress;
if (TSDB_ALTER_TABLE_ADD_COLUMN == pReq->action) {
@ -669,6 +678,13 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST
metaFetchEntryFree(&pEntry);
TAOS_RETURN(code);
}
code = addTableExtSchema(pEntry, pColumn, pSchema->nCols, &extSchema);
if (code) {
metaError("vgId:%d, %s failed to add ext schema at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode),
__func__, __FILE__, __LINE__, tstrerror(code), version);
metaFetchEntryFree(&pEntry);
TAOS_RETURN(code);
}
// do handle entry
code = metaHandleEntry2(pMeta, pEntry);
@ -777,6 +793,15 @@ int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, S
TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION);
}
// update column extschema
code = dropTableExtSchema(pEntry, iColumn, pSchema->nCols);
if (code) {
metaError("vgId:%d, %s failed to remove extschema at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode),
__func__, __FILE__, __LINE__, tstrerror(code), version);
metaFetchEntryFree(&pEntry);
TAOS_RETURN(code);
}
// do handle entry
code = metaHandleEntry2(pMeta, pEntry);
if (code) {
@ -1763,6 +1788,7 @@ int32_t metaAlterSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq)
.stbEntry.schemaTag = pReq->schemaTag,
.stbEntry.keep = pReq->keep,
.colCmpr = pReq->colCmpr,
.pExtSchemas = pReq->pExtSchemas,
};
TABLE_SET_COL_COMPRESSED(entry.flags);

34
source/dnode/vnode/src/tq/tq.c
View File

@ -849,7 +849,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
if (pTask->info.fillHistory) {
tqInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x "
" child id:%d, level:%d, cur-status:%s, next-status:%s taskType:%d, related stream task:0x%x "
"delaySched:%" PRId64 " ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
@ -857,7 +857,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
} else {
tqInfo("vgId:%d build stream task, s-task:%s, %p checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x "
" child id:%d, level:%d, cur-status:%s next-status:%s taskType:%d, related helper-task:0x%x "
"delaySched:%" PRId64 " ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pTask, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
@ -949,7 +949,7 @@ int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) {
return TSDB_CODE_SUCCESS;
}
// this function should be executed by only one thread, so we set an sentinel to protect this function
// this function should be executed by only one thread, so we set a sentinel to protect this function
int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont;
SStreamMeta* pMeta = pTq->pStreamMeta;
@ -957,6 +957,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
SStreamTask* pStreamTask = NULL;
char* pStatus = NULL;
int32_t taskType = 0;
code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) {
@ -971,8 +972,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamTaskState s = streamTaskGetStatus(pTask);
pStatus = s.name;
taskType = pTask->info.fillHistory;
if ((s.state != TASK_STATUS__SCAN_HISTORY) || (pTask->status.downstreamReady == 0)) {
if ((s.state != TASK_STATUS__SCAN_HISTORY && taskType == STREAM_HISTORY_TASK) ||
(s.state != TASK_STATUS__READY && taskType == STREAM_RECALCUL_TASK) ||
(pTask->status.downstreamReady == 0)) {
tqError("s-task:%s vgId:%d status:%s downstreamReady:%d not allowed/ready for scan-history data, quit", id,
pMeta->vgId, s.name, pTask->status.downstreamReady);
@ -1046,12 +1050,12 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
streamExecScanHistoryInFuture(pTask, retInfo.idleTime);
} else {
SStreamTaskState p = streamTaskGetStatus(pTask);
ETaskStatus s = p.state;
ETaskStatus localStatus = p.state;
if (s == TASK_STATUS__PAUSE) {
if (localStatus == TASK_STATUS__PAUSE) {
tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", id, el,
pTask->execInfo.step1El, status);
} else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) {
} else if (localStatus == TASK_STATUS__STOP || localStatus == TASK_STATUS__DROPPING) {
tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", id, p.name,
pTask->execInfo.step1El);
}
@ -1062,9 +1066,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
}
// the following procedure should be executed, no matter status is stop/pause or not
if (taskType == STREAM_HISTORY_TASK) {
tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El);
if (pTask->info.fillHistory != 1) {
} else if (taskType == STREAM_RECALCUL_TASK) {
tqDebug("s-task:%s recalculate ended, elapsed time:%.2fs", id, pTask->execInfo.step1El);
} else {
tqError("s-task:%s fill-history is disabled, unexpected", id);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
@ -1088,7 +1094,17 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
return TSDB_CODE_STREAM_INTERNAL_ERROR;
}
if (taskType == STREAM_HISTORY_TASK) {
code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, handleStep2Async, pTq);
} else if (taskType == STREAM_RECALCUL_TASK) {
// send recalculate end block
code = streamCreateAddRecalculateEndBlock(pStreamTask);
if (code) {
tqError("s-task:%s failed to create-add recalculate end block, code:%s", id, tstrerror(code));
}
streamTaskSetSchedStatusInactive(pTask);
}
streamMetaReleaseTask(pMeta, pStreamTask);
atomic_store_32(&pTask->status.inScanHistorySentinel, 0);

32
source/dnode/vnode/src/tq/tqRead.c
View File

@ -283,7 +283,7 @@ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
return;
}
bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}
@ -336,6 +336,7 @@ void tqReaderClose(STqReader* pReader) {
tDeleteSchemaWrapper(pReader->pSchemaWrapper);
}
taosMemoryFree(pReader->extSchema);
if (pReader->pColIdList) {
taosArrayDestroy(pReader->pColIdList);
}
@ -597,7 +598,7 @@ END:
return code;
}
int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) {
int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask, SExtSchema* extSrc) {
if (pDst == NULL || pBlock == NULL || pSrc == NULL || mask == NULL) {
return TSDB_CODE_INVALID_PARA;
}
@ -620,6 +621,9 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap
pDst->pSchema[j++] = pSrc->pSchema[i];
SColumnInfoData colInfo =
createColumnInfoData(pSrc->pSchema[i].type, pSrc->pSchema[i].bytes, pSrc->pSchema[i].colId);
if (extSrc != NULL) {
decimalFromTypeMod(extSrc[i].typeMod, &colInfo.info.precision, &colInfo.info.scale);
}
code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != 0) {
return code;
@ -653,6 +657,9 @@ static int32_t buildResSDataBlock(STqReader* pReader, SSchemaWrapper* pSchema, c
SSchema* pColSchema = &pSchema->pSchema[i];
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
if (IS_DECIMAL_TYPE(pColSchema->type) && pReader->extSchema != NULL) {
decimalFromTypeMod(pReader->extSchema[i].typeMod, &colInfo.info.precision, &colInfo.info.scale);
}
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
blockDataFreeRes(pBlock);
@ -680,6 +687,9 @@ static int32_t buildResSDataBlock(STqReader* pReader, SSchemaWrapper* pSchema, c
j++;
} else {
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
if (IS_DECIMAL_TYPE(pColSchema->type) && pReader->extSchema != NULL) {
decimalFromTypeMod(pReader->extSchema[i].typeMod, &colInfo.info.precision, &colInfo.info.scale);
}
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
return -1;
@ -708,7 +718,7 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
colDataSetNULL(pColumnInfoData, rowIndex);
}
} else {
code = colDataSetVal(pColumnInfoData, rowIndex, (void*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal));
code = colDataSetVal(pColumnInfoData, rowIndex, VALUE_GET_DATUM(&pColVal->value, pColVal->value.type), !COL_VAL_IS_VALUE(pColVal));
}
return code;
@ -741,8 +751,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
if ((suid != 0 && pReader->cachedSchemaSuid != suid) || (suid == 0 && pReader->cachedSchemaUid != uid) ||
(pReader->cachedSchemaVer != sversion)) {
tDeleteSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
taosMemoryFree(pReader->extSchema);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, &pReader->extSchema);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64
"version %d, possibly dropped table",
@ -883,7 +893,7 @@ END:
}
static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas,
SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow,
char* assigned, int32_t numOfRows, int32_t curRow,
int32_t* lastRow) {
int32_t code = 0;
SSchemaWrapper* pSW = NULL;
@ -901,7 +911,7 @@ static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData,
pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
TQ_NULL_GO_TO_END(pSW);
TQ_ERR_GO_TO_END(tqMaskBlock(pSW, block, pSchemaWrapper, assigned));
TQ_ERR_GO_TO_END(tqMaskBlock(pSW, block, pReader->pSchemaWrapper, assigned, pReader->extSchema));
tqTrace("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId,
(int32_t)taosArrayGetSize(block->pDataBlock));
@ -911,6 +921,7 @@ static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData,
TQ_NULL_GO_TO_END(taosArrayPush(blocks, block));
TQ_NULL_GO_TO_END(taosArrayPush(schemas, &pSW));
pSW = NULL;
taosMemoryFreeClear(block);
END:
@ -949,7 +960,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
}
if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, assigned, numOfRows,
curRow, &lastRow));
}
@ -1013,7 +1024,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
}
if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, assigned, numOfRows,
curRow, &lastRow));
}
@ -1119,7 +1130,8 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SMqDataRsp* pRsp, SArray* block
pReader->lastBlkUid = uid;
tDeleteSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
taosMemoryFree(pReader->extSchema);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, &pReader->extSchema);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);

2
source/dnode/vnode/src/tq/tqScan.c
View File

@ -313,7 +313,7 @@ END:
if (code != 0){
tqError("%s failed at %d, vgId:%d, task exec error since %s", __FUNCTION__ , lino, pTq->pVnode->config.vgId, tstrerror(code));
}
taosMemoryFree(pSW);
tDeleteSchemaWrapper(pSW);
taosMemoryFree(tbName);
return code;
}

2
source/dnode/vnode/src/tq/tqSink.c
View File

@ -823,7 +823,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
}
} else {
SValue sv = {.type = pCol->type};
memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes);
valueSetDatum(&sv, pCol->type, colData, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {

10
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -300,7 +300,11 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
return false;
}
if (pInfo->taskLevel == TASK_LEVEL__SOURCE && pInfo->trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
if (pInfo->trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
return false;
}
if (pInfo->fillHistory == STREAM_RECALCUL_TASK) {
return false;
}
@ -311,8 +315,8 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
return false;
}
// fill-history task has entered into the last phase, no need to anything
if ((pInfo->fillHistory == 1) && pTask->status.appendTranstateBlock) {
// fill-history task has entered into the last phase, no need to do anything
if ((pInfo->fillHistory == STREAM_HISTORY_TASK) && pTask->status.appendTranstateBlock) {
// the maximum version of data in the WAL has reached already, the step2 is done
tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
pTask->dataRange.range.maxVer);

1
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -75,7 +75,6 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
TSDB_CHECK_NULL(pRsp->blockSchema, code, lino, END, terrno);
END:
if (code != 0){
tqError("%s failed at:%d, code:%s", __FUNCTION__ , lino, tstrerror(code));

27
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -40,7 +40,7 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId);
if (pTask->info.fillHistory) {
if (pTask->info.fillHistory != STREAM_NORMAL_TASK) {
streamId = pTask->streamTaskId.streamId;
taskId = pTask->streamTaskId.taskId;
} else {
@ -50,20 +50,31 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
// sink task does not need the pState
if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
if (pTask->info.fillHistory == STREAM_RECALCUL_TASK) {
pTask->pRecalState = streamStateRecalatedOpen(pMeta->path, pTask, pTask->id.streamId, pTask->id.taskId);
if (pTask->pRecalState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
return terrno;
} else {
tqDebug("s-task:%s recal state:%p", pTask->id.idStr, pTask->pRecalState);
}
}
pTask->pState = streamStateOpen(pMeta->path, pTask, streamId, taskId);
if (pTask->pState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
return terrno;
} else {
tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
tqDebug("s-task:%s stream state:%p", pTask->id.idStr, pTask->pState);
}
}
SReadHandle handle = {
.checkpointId = pTask->chkInfo.checkpointId,
.pStateBackend = pTask->pState,
.pStateBackend = NULL,
.fillHistory = pTask->info.fillHistory,
.winRange = pTask->dataRange.window,
.pOtherBackend = NULL,
};
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -76,6 +87,14 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
initStorageAPI(&handle.api);
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG) {
if (pTask->info.fillHistory == STREAM_RECALCUL_TASK) {
handle.pStateBackend = pTask->pRecalState;
handle.pOtherBackend = pTask->pState;
} else {
handle.pStateBackend = pTask->pState;
handle.pOtherBackend = NULL;
}
code = qCreateStreamExecTaskInfo(&pTask->exec.pExecutor, pTask->exec.qmsg, &handle, vgId, pTask->id.taskId);
if (code) {
tqError("s-task:%s failed to expand task, code:%s", pTask->id.idStr, tstrerror(code));
@ -808,13 +827,11 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
}
pMeta->startInfo.startAllTasks = 1;
streamMetaWUnLock(pMeta);
terrno = 0;
tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d, ts:%" PRId64, vgId,
pMeta->updateInfo.completeTransId, pMeta->updateInfo.completeTs);
streamMetaWLock(pMeta);
streamMetaClear(pMeta);
int64_t el = taosGetTimestampMs() - st;

46
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -334,6 +334,10 @@ static int32_t tsdbCacheDeserializeV0(char const *value, SLastCol *pLastCol) {
pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]);
}
return sizeof(SLastColV0) + pLastColV0->colVal.value.nData;
} else if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
pLastCol->colVal.value.nData = pLastColV0->colVal.value.nData;
pLastCol->colVal.value.pData = (uint8_t*)(&pLastColV0[1]);
return sizeof(SLastColV0) + pLastColV0->colVal.value.nData;
} else {
pLastCol->colVal.value.val = pLastColV0->colVal.value.val;
return sizeof(SLastColV0);
@ -426,6 +430,12 @@ static int32_t tsdbCacheSerializeV0(char const *value, SLastCol *pLastCol) {
memcpy(&pLastColV0[1], pLastCol->colVal.value.pData, pLastCol->colVal.value.nData);
}
return sizeof(SLastColV0) + pLastCol->colVal.value.nData;
} else if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
pLastColV0->colVal.value.nData = pLastCol->colVal.value.nData;
if (pLastCol->colVal.value.nData > 0) {
memcpy(&pLastColV0[1], pLastCol->colVal.value.pData, pLastCol->colVal.value.nData);
}
return sizeof(SLastColV0) + pLastCol->colVal.value.nData;
} else {
pLastColV0->colVal.value.val = pLastCol->colVal.value.val;
return sizeof(SLastColV0);
@ -439,6 +449,9 @@ static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
*size += pLastCol->colVal.value.nData;
}
if (pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
*size += DECIMAL128_BYTES;
}
*size += sizeof(uint8_t) + sizeof(uint8_t) + sizeof(uint8_t); // version + numOfPKs + cacheStatus
for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) {
@ -840,6 +853,16 @@ static int32_t tsdbCacheReallocSLastCol(SLastCol *pCol, size_t *pCharge) {
charge += pCol->colVal.value.nData;
}
if (pCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) {
if (pCol->colVal.value.nData > 0) {
void *p = taosMemoryMalloc(pCol->colVal.value.nData);
if (!p) TAOS_CHECK_EXIT(terrno);
(void)memcpy(p, pCol->colVal.value.pData, pCol->colVal.value.nData);
pCol->colVal.value.pData = p;
}
charge += pCol->colVal.value.nData;
}
if (pCharge) {
*pCharge = charge;
}
@ -866,7 +889,8 @@ void tsdbCacheFreeSLastColItem(void *pItem) {
}
}
if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) {
if ((IS_VAR_DATA_TYPE(pCol->colVal.value.type) || pCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL) &&
pCol->colVal.value.pData) {
taosMemoryFree(pCol->colVal.value.pData);
}
}
@ -888,7 +912,8 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud
}
}
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) /* && pLastCol->colVal.value.nData > 0*/) {
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) ||
pLastCol->colVal.value.type == TSDB_DATA_TYPE_DECIMAL /* && pLastCol->colVal.value.nData > 0*/) {
taosMemoryFree(pLastCol->colVal.value.pData);
}
@ -1341,7 +1366,7 @@ static void tsdbCacheUpdateLastColToNone(SLastCol *pLastCol, ELastCacheStatus ca
taosMemoryFreeClear(pPKValue->pData);
pPKValue->nData = 0;
} else {
pPKValue->val = 0;
valueClearDatum(pPKValue, pPKValue->type);
}
}
pLastCol->rowKey.numOfPKs = 0;
@ -1351,7 +1376,7 @@ static void tsdbCacheUpdateLastColToNone(SLastCol *pLastCol, ELastCacheStatus ca
taosMemoryFreeClear(pLastCol->colVal.value.pData);
pLastCol->colVal.value.nData = 0;
} else {
pLastCol->colVal.value.val = 0;
valueClearDatum(&pLastCol->colVal.value, pLastCol->colVal.value.type);
}
pLastCol->colVal = COL_VAL_NONE(pLastCol->colVal.cid, pLastCol->colVal.value.type);
@ -1708,11 +1733,12 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo
tsdbRowGetKey(&lRow, &tsdbRowKey);
{
SValue tsVal = {.type = TSDB_DATA_TYPE_TIMESTAMP};
VALUE_SET_TRIVIAL_DATUM(&tsVal, lRow.pBlockData->aTSKEY[lRow.iRow]);
SLastUpdateCtx updateCtx = {
.lflag = LFLAG_LAST,
.tsdbRowKey = tsdbRowKey,
.colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP,
.val = lRow.pBlockData->aTSKEY[lRow.iRow]}))};
.colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, tsVal)};
if (!taosArrayPush(ctxArray, &updateCtx)) {
TAOS_CHECK_GOTO(terrno, &lino, _exit);
}
@ -3857,7 +3883,9 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
}
if (slotIds[iCol] == 0) {
STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowKey.key.ts}));
SValue val = {.type = pTColumn->type};
VALUE_SET_TRIVIAL_DATUM(&val, rowKey.key.ts);
*pColVal = COL_VAL_VALUE(pTColumn->colId, val);
SLastCol colTmp = {.rowKey = rowKey.key, .colVal = *pColVal, .cacheStatus = TSDB_LAST_CACHE_VALID};
TAOS_CHECK_GOTO(tsdbCacheReallocSLastCol(&colTmp, NULL), &lino, _err);
@ -4025,7 +4053,9 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
}
if (slotIds[iCol] == 0) {
STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowKey.key.ts}));
SValue val = {.type = pTColumn->type};
VALUE_SET_TRIVIAL_DATUM(&val, rowKey.key.ts);
*pColVal = COL_VAL_VALUE(pTColumn->colId, val);
SLastCol colTmp = {.rowKey = rowKey.key, .colVal = *pColVal, .cacheStatus = TSDB_LAST_CACHE_VALID};
TAOS_CHECK_GOTO(tsdbCacheReallocSLastCol(&colTmp, NULL), &lino, _err);

12
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -77,7 +77,8 @@ static int32_t saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader
TSDB_CHECK_CODE(code, lino, _end);
}
} else {
code = colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal));
code = colDataSetVal(pColInfoData, numOfRows, VALUE_GET_DATUM(&pVal->value, pColVal->colVal.value.type),
!COL_VAL_IS_VALUE(pVal));
TSDB_CHECK_CODE(code, lino, _end);
}
@ -166,7 +167,8 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
memcpy(varDataVal(p->buf), pColVal->colVal.value.pData, pColVal->colVal.value.nData);
p->bytes = pColVal->colVal.value.nData + VARSTR_HEADER_SIZE; // binary needs to plus the header size
} else {
memcpy(p->buf, &pColVal->colVal.value.val, pReader->pSchema->columns[slotId].bytes);
memcpy(p->buf, VALUE_GET_DATUM(&pColVal->colVal.value, pColVal->colVal.value.type),
pReader->pSchema->columns[slotId].bytes);
p->bytes = pReader->pSchema->columns[slotId].bytes;
}
}
@ -543,7 +545,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
}
}
if (IS_VAR_DATA_TYPE(pCol->type)) {
if (IS_VAR_DATA_TYPE(pCol->type) || pCol->type == TSDB_DATA_TYPE_DECIMAL) {
p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));
TSDB_CHECK_NULL(p.colVal.value.pData, code, lino, _end, terrno);
}
@ -601,7 +603,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
memcpy(p->rowKey.pks[j].pData, pColVal->rowKey.pks[j].pData, pColVal->rowKey.pks[j].nData);
p->rowKey.pks[j].nData = pColVal->rowKey.pks[j].nData;
} else {
p->rowKey.pks[j].val = pColVal->rowKey.pks[j].val;
valueCloneDatum(p->rowKey.pks + j, pColVal->rowKey.pks + j, p->rowKey.pks[j].type);
}
}
@ -628,7 +630,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
goto _end;
}
if (!IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) {
if (!IS_VAR_DATA_TYPE(pColVal->colVal.value.type) && pColVal->colVal.value.type != TSDB_DATA_TYPE_DECIMAL) {
p->colVal = pColVal->colVal;
} else {
if (COL_VAL_IS_VALUE(&pColVal->colVal)) {

2
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -1069,7 +1069,7 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
if ((colData->cflag & COL_SMA_ON) == 0 || ((colData->flag & HAS_VALUE) == 0)) continue;
SColumnDataAgg sma[1] = {{.colId = colData->cid}};
tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull);
tColDataCalcSMA[colData->type](colData, sma);
TAOS_CHECK_GOTO(tPutColumnDataAgg(&buffers[0], sma), &lino, _exit);
}

4
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -64,7 +64,7 @@ int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t num
static void freeItem(void *pValue) {
SValue *p = (SValue *)pValue;
if (IS_VAR_DATA_TYPE(p->type)) {
if (IS_VAR_DATA_TYPE(p->type) || p->type == TSDB_DATA_TYPE_DECIMAL) {
taosMemoryFree(p->pData);
}
}
@ -359,7 +359,7 @@ static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray
}
static int32_t tValueDupPayload(SValue *pVal) {
if (IS_VAR_DATA_TYPE(pVal->type)) {
if (IS_VAR_DATA_TYPE(pVal->type) || pVal->type == TSDB_DATA_TYPE_DECIMAL) {
char *p = (char *)pVal->pData;
char *pBuf = taosMemoryMalloc(pVal->nData);
if (pBuf == NULL) {

195
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -130,7 +130,7 @@ static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int
pKey->pks[0].type = cv.value.type;
if (IS_NUMERIC_TYPE(cv.value.type)) {
pKey->pks[0].val = cv.value.val;
valueCloneDatum(pKey->pks, &cv.value, cv.value.type);
} else {
pKey->pks[0].nData = cv.value.nData;
TAOS_MEMCPY(pKey->pks[0].pData, cv.value.pData, cv.value.nData);
@ -182,7 +182,7 @@ static int32_t tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
tdata += tGetU32v(tdata, &pKey->pks[i].nData);
TAOS_MEMCPY(pKey->pks[i].pData, tdata, pKey->pks[i].nData);
} else {
TAOS_MEMCPY(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes);
valueSetDatum(pKey->pks + i, indices[i].type, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes);
}
}
@ -1084,7 +1084,7 @@ static int32_t updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo,
TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
if (IS_NUMERIC_TYPE(pKey->pks[0].type)) {
pKey->pks[0].val = asc ? pBlockInfo->lastPk.val : pBlockInfo->firstPk.val;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, asc ? pBlockInfo->lastPk.val : pBlockInfo->firstPk.val);
} else {
uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData;
pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData);
@ -1127,7 +1127,8 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int
TSDB_CHECK_CODE(code, lino, _end);
}
} else {
code = colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value.val, !COL_VAL_IS_VALUE(pColVal));
code = colDataSetVal(pColInfoData, rowIndex, VALUE_GET_DATUM(&pColVal->value, pColVal->value.type),
!COL_VAL_IS_VALUE(pColVal));
TSDB_CHECK_CODE(code, lino, _end);
}
@ -1353,6 +1354,7 @@ static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpI
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_UBIGINT: {
int32_t mid = dumpedRows >> 1u;
int64_t* pts = (int64_t*)pColData->pData;
@ -1401,6 +1403,20 @@ static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpI
}
break;
}
case TSDB_DATA_TYPE_DECIMAL: {
int32_t mid = dumpedRows >> 1u;
DecimalWord* pDec = (DecimalWord*)pColData->pData;
DecimalWord tmp[2] = {0};
for (int32_t j = 0; j < mid; ++j) {
tmp[0] = pDec[2 * j];
tmp[1] = pDec[2 * j + 1];
pDec[2 * j] = pDec[2 * (dumpedRows - j - 1)];
pDec[2 * j + 1] = pDec[2 * (dumpedRows - j - 1) + 1];
pDec[2 * (dumpedRows - j - 1)] = tmp[0];
pDec[2 * (dumpedRows - j - 1) + 1] = tmp[1];
}
break;
}
}
}
@ -1443,8 +1459,8 @@ static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInf
pLast->pData = (uint8_t*)varDataVal(pBlockInfo->lastPk.pData);
pLast->nData = varDataLen(pBlockInfo->lastPk.pData);
} else {
pFirst->val = pBlockInfo->firstPk.val;
pLast->val = pBlockInfo->lastPk.val;
VALUE_SET_TRIVIAL_DATUM(pFirst, pBlockInfo->firstPk.val);
VALUE_SET_TRIVIAL_DATUM(pLast, pBlockInfo->lastPk.val);
}
}
@ -1899,7 +1915,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
if (IS_VAR_DATA_TYPE(pkType)) {
v1.pData = (uint8_t*)varDataVal(pBlock->lastPk.pData), v1.nData = varDataLen(pBlock->lastPk.pData);
} else {
v1.val = pBlock->lastPk.val;
VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->lastPk.val);
}
return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0);
} else { // no pk
@ -1915,7 +1931,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
if (IS_VAR_DATA_TYPE(pkType)) {
v1.pData = (uint8_t*)varDataVal(pBlock->firstPk.pData), v1.nData = varDataLen(pBlock->firstPk.pData);
} else {
v1.val = pBlock->firstPk.val;
VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->firstPk.val);
}
return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0);
} else { // no pk
@ -2192,7 +2208,7 @@ static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc
pNextProc->ts += step;
if (pSttBlockReader->numOfPks > 0) {
if (IS_NUMERIC_TYPE(pNextProc->pks[0].type)) {
pNextProc->pks[0].val = INT64_MIN;
VALUE_SET_TRIVIAL_DATUM(pNextProc->pks, INT64_MAX);
} else {
memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData);
}
@ -3759,8 +3775,8 @@ static int32_t buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockSca
if (pReader->suppInfo.numOfPks > 0) {
if (IS_NUMERIC_TYPE(pReader->suppInfo.pk.type)) {
pInfo->pks[0].val = pBlockInfo->firstPk.val;
pInfo->pks[1].val = pBlockInfo->lastPk.val;
VALUE_SET_TRIVIAL_DATUM(pInfo->pks, pBlockInfo->firstPk.val);
VALUE_SET_TRIVIAL_DATUM(pInfo->pks + 1, pBlockInfo->lastPk.val);
} else {
(void)memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData));
(void)memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData));
@ -6946,3 +6962,160 @@ void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, v
pReader->notifyFn = notifyFn;
pReader->notifyParam = param;
}
#if 0
static int32_t tsdbEncodeLastProcKeys(const SSHashObj* pTableMap, void** buf) {
void* p = NULL;
int32_t iter = 0;
int32_t len = 0;
len += taosEncodeFixedI32(buf, tSimpleHashGetSize(pTableMap)); // number of tables
while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {
uint64_t uid = *(uint64_t*)tSimpleHashGetKey(p, NULL);
SRowKey* pKey = &(*(STableBlockScanInfo**)p)->lastProcKey;
len += taosEncodeFixedU64(buf, uid); // table uid
len += taosEncodeFixedI64(buf, pKey->ts); // last read ts
len += taosEncodeFixedU8(buf, pKey->numOfPKs); // number of pks
if (pKey->numOfPKs > 0) {
int8_t type = pKey->pks[0].type;
len += taosEncodeFixedI8(buf, type); // pk data type
uint8_t* pData = 0;
uint32_t nData = 0;
if (IS_VAR_DATA_TYPE(type)) {
pData = pKey->pks[0].pData;
nData = pKey->pks[0].nData;
len += taosEncodeFixedU32(buf, nData); // data length
} else {
pData = (uint8_t*)&pKey->pks[0].val;
nData = tDataTypes[pKey->pks[0].type].bytes;
}
len += taosEncodeBinary(buf, pData, nData); // data value
}
}
return len;
}
static int32_t tsdbDecodeLastProcKeys(const void* buf, int32_t len, SSHashObj* pTableMap) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
const void *begin = buf;
TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pTableMap, code, lino, _end, TSDB_CODE_INVALID_PARA);
int32_t numOfTables = 0;
buf = taosDecodeFixedI32(buf, &numOfTables);
TSDB_CHECK_CONDITION(numOfTables == tSimpleHashGetSize(pTableMap), code, lino, _end, TSDB_CODE_INVALID_PARA);
for (int32_t i = 0; i < numOfTables; ++i) {
uint64_t uid = 0;
buf = taosDecodeFixedU64(buf, &uid);
STableBlockScanInfo** pScanInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (pScanInfo == NULL) {
tsdbError("failed to find scan info of uid: %" PRIu64 " when decoding last proc keys", uid);
TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
SRowKey* pKey = &(*pScanInfo)->lastProcKey;
buf = taosDecodeFixedI64(buf, &pKey->ts);
uint8_t numOfPks = 0;
buf = taosDecodeFixedU8(buf, &numOfPks);
if (numOfPks != pKey->numOfPKs) {
tsdbError("numOfPks of uid %" PRIu64 " mismatch: [%u/%u]", uid, numOfPks, pKey->numOfPKs);
TSDB_CHECK_CONDITION(numOfPks == pKey->numOfPKs, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
if (numOfPks > 0) {
int8_t type = 0;
buf = taosDecodeFixedI8(buf, &type);
if (type != pKey->pks[0].type) {
tsdbError("pk type of uid %" PRIu64 " mismatch: [%d/%d]", uid, type, pKey->pks[0].type);
TSDB_CHECK_CONDITION(type == pKey->pks[0].type, code, lino, _end, TSDB_CODE_INVALID_PARA);
}
uint8_t *pData = 0;
uint32_t nData = 0;
if (IS_VAR_DATA_TYPE(type)) {
pData = pKey->pks[0].pData;
buf = taosDecodeFixedU32(buf, &nData);
pKey->pks[0].nData = nData;
} else {
pKey->pks[0].val = 0;
pData = (uint8_t*)&pKey->pks[0].val;
nData = tDataTypes[type].bytes;
}
buf = taosDecodeBinary(buf, (void **)&pData, nData);
}
}
TSDB_CHECK_CONDITION(buf == begin + len , code, lino, _end, TSDB_CODE_INVALID_PARA);
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
#endif
int32_t tsdbReaderGetProgress(const STsdbReader* pReader, void** pBuf, uint64_t* pLen) {
int32_t code = TSDB_CODE_SUCCESS;
#if 0
int32_t lino = 0;
void* buf = NULL;
int32_t len = 0;
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pBuf, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(pLen, code, lino, _end, TSDB_CODE_INVALID_PARA);
*pBuf = NULL;
*pLen = tsdbEncodeLastProcKeys(pReader->status.pTableMap, NULL);
buf = taosMemoryMalloc(*pLen);
TSDB_CHECK_NULL(buf, code, lino, _end, terrno);
*pBuf = buf;
len = tsdbEncodeLastProcKeys(pReader->status.pTableMap, &buf);
TSDB_CHECK_CONDITION(len == *pLen, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
buf = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (buf != NULL) {
taosMemoryFreeClear(buf);
}
#endif
return code;
}
int32_t tsdbReaderSetProgress(STsdbReader* pReader, const void* buf, uint64_t len) {
int32_t code = TSDB_CODE_SUCCESS;
#if 0
int32_t lino = 0;
if (len == 0) {
goto _end;
}
TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);
code = tsdbDecodeLastProcKeys(buf, len, pReader->status.pTableMap);
TSDB_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
#endif
return code;
}

30
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -221,29 +221,29 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
if (asc) {
switch (type) {
case TSDB_DATA_TYPE_BIGINT: {
pKey->pks[0].val = INT64_MIN;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MIN);
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t min = INT32_MIN;
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t min = INT16_MIN;
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_TINYINT: {
int8_t min = INT8_MIN;
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT: {
pKey->pks[0].val = 0;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, 0);
break;
}
default:
@ -253,28 +253,28 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
} else {
switch (type) {
case TSDB_DATA_TYPE_BIGINT:
pKey->pks[0].val = INT64_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MAX);
break;
case TSDB_DATA_TYPE_INT:
pKey->pks[0].val = INT32_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT32_MAX);
break;
case TSDB_DATA_TYPE_SMALLINT:
pKey->pks[0].val = INT16_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT16_MAX);
break;
case TSDB_DATA_TYPE_TINYINT:
pKey->pks[0].val = INT8_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT8_MAX);
break;
case TSDB_DATA_TYPE_UBIGINT:
pKey->pks[0].val = UINT64_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT64_MAX);
break;
case TSDB_DATA_TYPE_UINT:
pKey->pks[0].val = UINT32_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT32_MAX);
break;
case TSDB_DATA_TYPE_USMALLINT:
pKey->pks[0].val = UINT16_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT16_MAX);
break;
case TSDB_DATA_TYPE_UTINYINT:
pKey->pks[0].val = UINT8_MAX;
VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT8_MAX);
break;
default:
code = TSDB_CODE_INVALID_PARA;
@ -722,8 +722,8 @@ int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) {
TSDB_CHECK_CONDITION((pFirstKey->numOfPKs == pLastKey->numOfPKs), code, lino, _end, TSDB_CODE_INVALID_PARA);
if (pFirstKey->numOfPKs > 0) {
if (IS_NUMERIC_TYPE(pFirstKey->pks[0].type)) {
pBlockInfo->firstPk.val = pFirstKey->pks[0].val;
pBlockInfo->lastPk.val = pLastKey->pks[0].val;
pBlockInfo->firstPk.val = VALUE_GET_TRIVIAL_DATUM(pFirstKey->pks);
pBlockInfo->lastPk.val = VALUE_GET_TRIVIAL_DATUM(pLastKey->pks);
} else {
int32_t keyLen = pFirstKey->pks[0].nData;
char* p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);

56
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -615,9 +615,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
}
} else if (pRow->type == TSDBROW_COL_FMT) {
if (iCol == 0) {
*pColVal =
COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID,
((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = pRow->pBlockData->aTSKEY[pRow->iRow]}));
SValue val = {.type = TSDB_DATA_TYPE_TIMESTAMP};
VALUE_SET_TRIVIAL_DATUM(&val, pRow->pBlockData->aTSKEY[pRow->iRow]);
*pColVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, val);
} else {
SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
@ -715,9 +715,9 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) {
return tRowIterNext(pIter->pIter);
} else if (pIter->pRow->type == TSDBROW_COL_FMT) {
if (pIter->iColData == 0) {
pIter->cv = COL_VAL_VALUE(
PRIMARYKEY_TIMESTAMP_COL_ID,
((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]}));
SValue val = {.type = TSDB_DATA_TYPE_TIMESTAMP};
VALUE_SET_TRIVIAL_DATUM(&val, pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]);
pIter->cv = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, val);
++pIter->iColData;
return &pIter->cv;
}
@ -754,8 +754,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
// ts
jCol = 0;
pTColumn = &pTSchema->columns[jCol++];
*pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = key.ts}));
SValue val = {.type = pTColumn->type};
VALUE_SET_TRIVIAL_DATUM(&val, key.ts);
*pColVal = COL_VAL_VALUE(pTColumn->colId, val);
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = terrno;
return code;
@ -776,7 +777,8 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
}
tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->value.type)) {
bool usepData = IS_VAR_DATA_TYPE(pColVal->value.type) || pColVal->value.type == TSDB_DATA_TYPE_DECIMAL;
if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && usepData) {
uint8_t *pVal = pColVal->value.pData;
pColVal->value.pData = NULL;
@ -819,7 +821,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
if (key.version > pMerger->version) {
if (!COL_VAL_IS_NONE(pColVal)) {
if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
if (IS_VAR_DATA_TYPE(pColVal->value.type) || pColVal->value.type == TSDB_DATA_TYPE_DECIMAL) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
if (!pTColVal) return terrno;
if (!COL_VAL_IS_NULL(pColVal)) {
@ -842,7 +844,8 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
} else if (key.version < pMerger->version) {
SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->value.type)) {
bool usepData = IS_VAR_DATA_TYPE(pColVal->value.type) || pColVal->value.type == TSDB_DATA_TYPE_DECIMAL;
if ((!COL_VAL_IS_NULL(pColVal)) && usepData) {
code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
if (code) return code;
@ -878,7 +881,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pSchema) {
void tsdbRowMergerClear(SRowMerger *pMerger) {
for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
if (IS_VAR_DATA_TYPE(pTColVal->value.type)) {
if (IS_VAR_DATA_TYPE(pTColVal->value.type) || pTColVal->value.type == TSDB_DATA_TYPE_DECIMAL) {
tFree(pTColVal->value.pData);
}
}
@ -890,7 +893,7 @@ void tsdbRowMergerCleanup(SRowMerger *pMerger) {
int32_t numOfCols = taosArrayGetSize(pMerger->pArray);
for (int32_t iCol = 1; iCol < numOfCols; iCol++) {
SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
if (IS_VAR_DATA_TYPE(pTColVal->value.type)) {
if (IS_VAR_DATA_TYPE(pTColVal->value.type) || pTColVal->value.type == TSDB_DATA_TYPE_DECIMAL) {
tFree(pTColVal->value.pData);
}
}
@ -1573,11 +1576,23 @@ int32_t tGetDiskDataHdr(SBufferReader *br, SDiskDataHdr *pHdr) {
int32_t tPutColumnDataAgg(SBuffer *buffer, SColumnDataAgg *pColAgg) {
int32_t code;
if ((code = tBufferPutI16v(buffer, pColAgg->colId))) return code;
if (pColAgg->colId & DECIMAL_AGG_FLAG) {
if ((code = tBufferPutI32v(buffer, pColAgg->colId))) return code;
if ((code = tBufferPutI16v(buffer, pColAgg->numOfNull))) return code;
if ((code = tBufferPutU64(buffer, pColAgg->decimal128Sum[0]))) return code;
if ((code = tBufferPutU64(buffer, pColAgg->decimal128Sum[1]))) return code;
if ((code = tBufferPutU64(buffer, pColAgg->decimal128Max[0]))) return code;
if ((code = tBufferPutU64(buffer, pColAgg->decimal128Max[1]))) return code;
if ((code = tBufferPutU64(buffer, pColAgg->decimal128Min[0]))) return code;
if ((code = tBufferPutU64(buffer, pColAgg->decimal128Min[1]))) return code;
if ((code = tBufferPutU8(buffer, pColAgg->overflow))) return code;
} else {
if ((code = tBufferPutI32v(buffer, pColAgg->colId))) return code;
if ((code = tBufferPutI16v(buffer, pColAgg->numOfNull))) return code;
if ((code = tBufferPutI64(buffer, pColAgg->sum))) return code;
if ((code = tBufferPutI64(buffer, pColAgg->max))) return code;
if ((code = tBufferPutI64(buffer, pColAgg->min))) return code;
}
return 0;
}
@ -1585,11 +1600,22 @@ int32_t tPutColumnDataAgg(SBuffer *buffer, SColumnDataAgg *pColAgg) {
int32_t tGetColumnDataAgg(SBufferReader *br, SColumnDataAgg *pColAgg) {
int32_t code;
if ((code = tBufferGetI16v(br, &pColAgg->colId))) return code;
if ((code = tBufferGetI32v(br, &pColAgg->colId))) return code;
if ((code = tBufferGetI16v(br, &pColAgg->numOfNull))) return code;
if (pColAgg->colId & DECIMAL_AGG_FLAG) {
pColAgg->colId &= 0xFFFF;
if ((code = tBufferGetU64(br, &pColAgg->decimal128Sum[0]))) return code;
if ((code = tBufferGetU64(br, &pColAgg->decimal128Sum[1]))) return code;
if ((code = tBufferGetU64(br, &pColAgg->decimal128Max[0]))) return code;
if ((code = tBufferGetU64(br, &pColAgg->decimal128Max[1]))) return code;
if ((code = tBufferGetU64(br, &pColAgg->decimal128Min[0]))) return code;
if ((code = tBufferGetU64(br, &pColAgg->decimal128Min[1]))) return code;
if ((code = tBufferGetU8(br, &pColAgg->overflow))) return code;
} else {
if ((code = tBufferGetI64(br, &pColAgg->sum))) return code;
if ((code = tBufferGetI64(br, &pColAgg->max))) return code;
if ((code = tBufferGetI64(br, &pColAgg->min))) return code;
}
return 0;
}

31
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -69,6 +69,9 @@ void initTsdbReaderAPI(TsdReader* pReader) {
pReader->fileSetReadNext = tsdbFileSetReaderNext;
pReader->fileSetGetEntryField = tsdbFileSetGetEntryField;
pReader->fileSetReaderClose = tsdbFileSetReaderClose;
pReader->getProgress = (int32_t (*)(const void*, void**, uint64_t*))tsdbReaderGetProgress;
pReader->setProgress = (int32_t (*)(void*, const void*, uint64_t))tsdbReaderSetProgress;
}
void initMetadataAPI(SStoreMeta* pMeta) {
@ -158,6 +161,7 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStatePutParName = streamStatePutParName;
pStore->streamStateGetParName = streamStateGetParName;
pStore->streamStateDeleteParName = streamStateDeleteParName;
pStore->streamStateSetParNameInvalid = streamStateSetParNameInvalid;
pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
pStore->streamStateReleaseBuf = streamStateReleaseBuf;
@ -173,8 +177,11 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateClear = streamStateClear;
pStore->streamStateSaveInfo = streamStateSaveInfo;
pStore->streamStateGetInfo = streamStateGetInfo;
pStore->streamStateGetNumber = streamStateGetNumber;
pStore->streamStateDeleteInfo = streamStateDeleteInfo;
pStore->streamStateSetNumber = streamStateSetNumber;
pStore->streamStateGetPrev = streamStateGetPrev;
pStore->streamStateGetAllPrev = streamStateGetAllPrev;
pStore->streamStateFillPut = streamStateFillPut;
pStore->streamStateFillGet = streamStateFillGet;
@ -196,6 +203,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
pStore->streamStateClearExpiredSessionState = streamStateClearExpiredSessionState;
pStore->streamStateSetRecFlag = streamStateSetRecFlag;
pStore->streamStateGetRecFlag = streamStateGetRecFlag;
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
pStore->streamStateSessionPut = streamStateSessionPut;
@ -208,6 +218,9 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange;
pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition;
pStore->streamStateSessionSaveToDisk = streamStateSessionSaveToDisk;
pStore->streamStateFlushReaminInfoToDisk = streamStateFlushReaminInfoToDisk;
pStore->streamStateSessionDeleteAll = streamStateSessionDeleteAll;
pStore->streamStateCountWinAddIfNotExist = streamStateCountWinAddIfNotExist;
pStore->streamStateCountWinAdd = streamStateCountWinAdd;
@ -242,6 +255,24 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateClear = streamFileStateClear;
pStore->needClearDiskBuff = needClearDiskBuff;
pStore->streamStateGetAndSetTsData = streamStateGetAndSetTsData;
pStore->streamStateTsDataCommit = streamStateTsDataCommit;
pStore->streamStateInitTsDataState = streamStateInitTsDataState;
pStore->streamStateDestroyTsDataState = streamStateDestroyTsDataState;
pStore->streamStateRecoverTsData = streamStateRecoverTsData;
pStore->streamStateReloadTsDataState = streamStateReloadTsDataState;
pStore->streamStateMergeAndSaveScanRange = streamStateMergeAndSaveScanRange;
pStore->streamStateMergeAllScanRange = streamStateMergeAllScanRange;
pStore->streamStatePopScanRange = streamStatePopScanRange;
pStore->streamStateCheckSessionState = streamStateCheckSessionState;
pStore->streamStateGetLastStateCur = streamStateGetLastStateCur;
pStore->streamStateLastStateCurNext = streamStateLastStateCurNext;
pStore->streamStateNLastStateGetKVByCur = streamStateNLastStateGetKVByCur;
pStore->streamStateGetLastSessionStateCur = streamStateGetLastSessionStateCur;
pStore->streamStateLastSessionStateCurNext = streamStateLastSessionStateCurNext;
pStore->streamStateNLastSessionStateGetKVByCur = streamStateNLastSessionStateGetKVByCur;
pStore->streamStateOpen = streamStateOpen;
pStore->streamStateClose = streamStateClose;
pStore->streamStateBegin = streamStateBegin;

11
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -34,7 +34,7 @@ void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery);
int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol) {
int8_t tblType = reader->me.type;
if (useCompress(tblType)) {
if (withExtSchema(tblType)) {
SColCmprWrapper *p = &(reader->me.colCmpr);
if (numOfCol != p->nCols) {
vError("fillTableColCmpr table type:%d, col num:%d, col cmpr num:%d mismatch", tblType, numOfCol, p->nCols);
@ -180,6 +180,9 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
if (code < 0) {
goto _exit;
}
for (int32_t i = 0; i < metaRsp.numOfColumns && pReader->me.pExtSchemas; i++) {
metaRsp.pSchemaExt[i].typeMod = pReader->me.pExtSchemas[i].typeMod;
}
} else {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
@ -348,6 +351,10 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i;
pSchExt->colId = pCmpr->id;
pSchExt->compress = pCmpr->alg;
if (pReader->me.pExtSchemas)
pSchExt->typeMod = pReader->me.pExtSchemas[i].typeMod;
else
pSchExt->typeMod = 0;
}
//}
@ -753,7 +760,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
}
int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0);
SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL);
if (pSW) {
*num = pSW->nCols;
tDeleteSchemaWrapper(pSW);

4
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -1800,11 +1800,11 @@ static int32_t vnodeCellValConvertToColVal(STColumn *pCol, SCellVal *pCellVal, S
pColVal->value.pData = (uint8_t *)varDataVal(pCellVal->val);
} else if (TSDB_DATA_TYPE_FLOAT == pCol->type) {
float f = GET_FLOAT_VAL(pCellVal->val);
memcpy(&pColVal->value.val, &f, sizeof(f));
valueSetDatum(&pColVal->value, pCol->type, &f, sizeof(f));
} else if (TSDB_DATA_TYPE_DOUBLE == pCol->type) {
taosSetPInt64Aligned(&pColVal->value.val, (int64_t *)pCellVal->val);
} else {
GET_TYPED_DATA(pColVal->value.val, int64_t, pCol->type, pCellVal->val);
valueSetDatum(&pColVal->value, pCol->type, pCellVal->val, tDataTypes[pCol->type].bytes);
}
pColVal->flag = CV_FLAG_VALUE;

1
source/libs/CMakeLists.txt
View File

@ -25,3 +25,4 @@ add_subdirectory(geometry)
add_subdirectory(command)
add_subdirectory(azure)
add_subdirectory(tcs)
add_subdirectory(decimal)

11
source/libs/catalog/src/ctgCache.c
View File

@ -598,13 +598,22 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt
}
metaSize = CTG_META_SIZE(stbMeta);
*pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize);
int32_t schemaExtSize = 0;
if (stbMeta->schemaExt) {
schemaExtSize = stbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt);
}
*pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize + schemaExtSize);
if (NULL == *pTableMeta) {
CTG_ERR_RET(terrno);
}
TAOS_MEMCPY(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta));
if (stbMeta->schemaExt) {
(*pTableMeta)->schemaExt = (SSchemaExt*)((char*)*pTableMeta + metaSize);
TAOS_MEMCPY((*pTableMeta)->schemaExt, stbMeta->schemaExt, schemaExtSize);
} else {
(*pTableMeta)->schemaExt = NULL;
}
return TSDB_CODE_SUCCESS;
}

4
source/libs/catalog/src/ctgUtil.c
View File

@ -1684,7 +1684,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput)
if (output->tbMeta) {
int32_t metaSize = CTG_META_SIZE(output->tbMeta);
int32_t schemaExtSize = 0;
if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) {
if (withExtSchema(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) {
schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt);
}
@ -1697,7 +1697,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput)
}
TAOS_MEMCPY((*pOutput)->tbMeta, output->tbMeta, metaSize);
if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) {
if (withExtSchema(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) {
(*pOutput)->tbMeta->schemaExt = (SSchemaExt*)((char*)(*pOutput)->tbMeta + metaSize);
TAOS_MEMCPY((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize);
} else {

2
source/libs/command/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
command
PRIVATE os util nodes catalog function transport qcom scheduler
PRIVATE os util nodes catalog function transport qcom scheduler decimal
)
if(${BUILD_TEST})

23
source/libs/command/src/command.c
View File

@ -16,6 +16,7 @@
#include "catalog.h"
#include "command.h"
#include "commandInt.h"
#include "decimal.h"
#include "scheduler.h"
#include "systable.h"
#include "taosdef.h"
@ -150,7 +151,7 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
SColumnInfoData* pCol6 = NULL;
// level
SColumnInfoData* pCol7 = NULL;
if (useCompress(pMeta->tableType)) {
if (withExtSchema(pMeta->tableType)) {
pCol5 = taosArrayGet(pBlock->pDataBlock, 4);
pCol6 = taosArrayGet(pBlock->pDataBlock, 5);
pCol7 = taosArrayGet(pBlock->pDataBlock, 6);
@ -165,7 +166,15 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
STR_TO_VARSTR(buf, pMeta->schema[i].name);
COL_DATA_SET_VAL_AND_CHECK(pCol1, pBlock->info.rows, buf, false);
if (IS_DECIMAL_TYPE(pMeta->schema[i].type) && withExtSchema(pMeta->tableType)) {
uint8_t prec = 0, scale = 0;
decimalFromTypeMod(pMeta->schemaExt[i].typeMod, &prec, &scale);
size_t len = snprintf(buf + VARSTR_HEADER_SIZE, DESCRIBE_RESULT_FIELD_LEN - VARSTR_HEADER_SIZE, "%s(%hhu, %hhu)",
tDataTypes[pMeta->schema[i].type].name, prec, scale);
varDataSetLen(buf, len);
} else {
STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name);
}
COL_DATA_SET_VAL_AND_CHECK(pCol2, pBlock->info.rows, buf, false);
int32_t bytes = getSchemaBytes(pMeta->schema + i);
COL_DATA_SET_VAL_AND_CHECK(pCol3, pBlock->info.rows, (const char*)&bytes, false);
@ -182,7 +191,7 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
STR_TO_VARSTR(buf, "VIEW COL");
}
COL_DATA_SET_VAL_AND_CHECK(pCol4, pBlock->info.rows, buf, false);
if (useCompress(pMeta->tableType) && pMeta->schemaExt) {
if (withExtSchema(pMeta->tableType) && pMeta->schemaExt) {
if (i < pMeta->tableInfo.numOfColumns) {
STR_TO_VARSTR(buf, columnEncodeStr(COMPRESS_L1_TYPE_U32(pMeta->schemaExt[i].compress)));
COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false);
@ -235,7 +244,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp**
code = setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta, biMode);
}
if (TSDB_CODE_SUCCESS == code) {
if (pDesc->pMeta && useCompress(pDesc->pMeta->tableType) && pDesc->pMeta->schemaExt) {
if (pDesc->pMeta && withExtSchema(pDesc->pMeta->tableType) && pDesc->pMeta->schemaExt) {
code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS_COMPRESS, pRsp);
} else {
code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp);
@ -537,9 +546,13 @@ static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)",
(int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
} else if (IS_DECIMAL_TYPE(pSchema->type)) {
uint8_t precision, scale;
decimalFromTypeMod(pCfg->pSchemaExt[i].typeMod, &precision, &scale);
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d,%d)", precision, scale);
}
if (useCompress(pCfg->tableType) && pCfg->pSchemaExt) {
if (withExtSchema(pCfg->tableType) && pCfg->pSchemaExt) {
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " ENCODE \'%s\'",
columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress)));
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " COMPRESS \'%s\'",
@ -1027,6 +1040,8 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p
} else {
infoData.info.type = pExpr->resType.type;
infoData.info.bytes = pExpr->resType.bytes;
infoData.info.precision = pExpr->resType.precision;
infoData.info.scale = pExpr->resType.scale;
}
QRY_ERR_RET(blockDataAppendColInfo(pBlock, &infoData));
}

18
source/libs/decimal/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
aux_source_directory(src DECIMAL_SRC)
add_library(decimal STATIC ${DECIMAL_SRC})
target_include_directories(
decimal
PUBLIC "${TD_SOURCE_DIR}/include/libs/decimal"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
decimal
PRIVATE os common wideInteger
)
if(${BUILD_TEST})
ADD_SUBDIRECTORY(test)
endif(${BUILD_TEST})
ADD_SUBDIRECTORY(src/detail)

144
source/libs/decimal/inc/wideInteger.h Normal file
View File

@ -0,0 +1,144 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_WIDE_INTEGER_H_
#define _TD_WIDE_INTEGER_H_
#include <stdbool.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
struct uint128 {
uint64_t low;
uint64_t high;
};
struct int128 {
uint64_t low;
int64_t high;
};
struct uint256 {
struct uint128 low;
struct uint128 high;
};
struct int256 {
struct uint128 low;
struct int128 high;
};
#define UInt128 struct uint128
#define UInt256 struct uint256
#define Int128 struct int128
#define Int256 struct int256
#define SAFE_SIGNED_OP(a, b, SIGNED_TYPE, UNSIGNED_TYPE, OP) (SIGNED_TYPE)((UNSIGNED_TYPE)(a)OP(UNSIGNED_TYPE)(b))
#define SAFE_INT64_ADD(a, b) SAFE_SIGNED_OP(a, b, int64_t, uint64_t, +)
#define SAFE_INT64_SUBTRACT(a, b) SAFE_SIGNED_OP(a, b, int64_t, uint64_t, -)
void makeUInt128(UInt128* pInt, uint64_t hi, uint64_t lo);
uint64_t uInt128Hi(const UInt128* pInt);
uint64_t uInt128Lo(const UInt128* pInt);
void uInt128Add(UInt128* pLeft, const UInt128* pRight);
void uInt128Subtract(UInt128* pLeft, const UInt128* pRight);
void uInt128Multiply(UInt128* pLeft, const UInt128* pRight);
void uInt128Divide(UInt128* pLeft, const UInt128* pRight);
void uInt128Mod(UInt128* pLeft, const UInt128* pRight);
bool uInt128Lt(const UInt128* pLeft, const UInt128* pRight);
bool uInt128Gt(const UInt128* pLeft, const UInt128* pRight);
bool uInt128Eq(const UInt128* pLeft, const UInt128* pRight);
extern const UInt128 uInt128_1e18;
extern const UInt128 uInt128Zero;
extern const uint64_t k1e18;
extern const UInt128 uInt128One;
extern const UInt128 uInt128Two;
Int128 makeInt128(int64_t high, uint64_t low);
int64_t int128Hi(const Int128* pUint128);
uint64_t int128Lo(const Int128* pUint128);
Int128 int128Abs(const Int128* pInt128);
Int128 int128Negate(const Int128* pInt128);
Int128 int128Add(const Int128* pLeft, const Int128* pRight);
Int128 int128Subtract(const Int128* pLeft, const Int128* pRight);
Int128 int128Multiply(const Int128* pLeft, const Int128* pRight);
Int128 int128Divide(const Int128* pLeft, const Int128* pRight);
Int128 int128Mod(const Int128* pLeft, const Int128* pRight);
bool int128Lt(const Int128* pLeft, const Int128* pRight);
bool int128Gt(const Int128* pLeft, const Int128* pRight);
bool int128Eq(const Int128* pLeft, const Int128* pRight);
Int128 int128RightShift(const Int128* pLeft, int32_t shift);
extern const Int128 int128Zero;
extern const Int128 int128One;
UInt256 makeUint256(UInt128 high, UInt128 low);
UInt128 uInt256Hi(const UInt256* pUint256);
UInt128 uInt256Lo(const UInt256* pUint256);
UInt256 uInt256Add(const UInt256* pLeft, const UInt256* pRight);
UInt256 uInt256Subtract(const UInt256* pLeft, const UInt256* pRight);
UInt256 uInt256Multiply(const UInt256* pLeft, const UInt256* pRight);
UInt256 uInt256Divide(const UInt256* pLeft, const UInt256* pRight);
UInt256 uInt256Mod(const UInt256* pLeft, const UInt256* pRight);
bool uInt256Lt(const UInt256* pLeft, const UInt256* pRight);
bool uInt256Gt(const UInt256* pLeft, const UInt256* pRight);
bool uInt256Eq(const UInt256* pLeft, const UInt256* pRight);
UInt256 uInt256RightShift(const UInt256* pLeft, int32_t shift);
extern const UInt256 uInt256Zero;
extern const UInt256 uInt256One;
Int256 makeInt256(Int128 high, UInt128 low);
Int128 int256Hi(const Int256* pUint256);
UInt128 int256Lo(const Int256* pUint256);
Int256 int256Abs(const Int256* pInt256);
Int256 int256Negate(const Int256* pInt256);
Int256 int256Add(const Int256* pLeft, const Int256* pRight);
Int256 int256Subtract(const Int256* pLeft, const Int256* pRight);
Int256 int256Multiply(const Int256* pLeft, const Int256* pRight);
Int256 int256Divide(const Int256* pLeft, const Int256* pRight);
Int256 int256Mod(const Int256* pLeft, const Int256* pRight);
bool int256Lt(const Int256* pLeft, const Int256* pRight);
bool int256Gt(const Int256* pLeft, const Int256* pRight);
bool int256Eq(const Int256* pLeft, const Int256* pRight);
Int256 int256RightShift(const Int256* pLeft, int32_t shift);
extern const Int256 int256Zero;
extern const Int256 int256One;
extern const Int256 int256Two;
#ifdef __cplusplus
}
#endif
static inline int32_t countLeadingZeros(uint64_t v) {
#if defined(__clang__) || defined(__GNUC__)
if (v == 0) return 64;
return __builtin_clzll(v);
#else
int32_t bitpos = 0;
while (v != 0) {
v >>= 1;
++bitpos;
}
return 64 - bitpos;
#endif
}
#endif /* _TD_WIDE_INTEGER_H_ */

2024
source/libs/decimal/src/decimal.c Normal file
View File

File diff suppressed because it is too large Load Diff

15
source/libs/decimal/src/detail/CMakeLists.txt Normal file
View File

@ -0,0 +1,15 @@
MESSAGE(STATUS "Building decimal/src/detail")
aux_source_directory(. WIDE_INTEGER_SRC)
SET(CMAKE_CXX_STANDARD 14)
add_library(wideInteger STATIC ${WIDE_INTEGER_SRC})
target_include_directories(
wideInteger
PUBLIC "${TD_SOURCE_DIR}/source/libs/decimal/inc/"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/intx/"
)
target_link_libraries(
wideInteger
PUBLIC
)

885
source/libs/decimal/src/detail/intx/int128.hpp Normal file
View File

@ -0,0 +1,885 @@
// intx: extended precision integer library.
// Copyright 2019-2020 Pawel Bylica.
// Licensed under the Apache License, Version 2.0.
#pragma once
#include <algorithm>
#include <climits>
#include <cstdint>
#include <limits>
#include <stdexcept>
#include <string>
#include <tuple>
#include <type_traits>
#ifdef _MSC_VER
#include <intrin.h>
#endif
#ifdef _MSC_VER
#define INTX_UNREACHABLE __assume(0)
#else
#define INTX_UNREACHABLE __builtin_unreachable()
#endif
#ifdef _MSC_VER
#define INTX_UNLIKELY(EXPR) (bool{EXPR})
#else
#define INTX_UNLIKELY(EXPR) __builtin_expect(bool{EXPR}, false)
#endif
#ifdef NDEBUG
#define INTX_REQUIRE(X) (X) ? (void)0 : INTX_UNREACHABLE
#else
#include <cassert>
#define INTX_REQUIRE assert
#endif
namespace intx
{
template <unsigned N>
struct uint;
/// The 128-bit unsigned integer.
///
/// This type is defined as a specialization of uint<> to easier integration with full intx package,
/// however, uint128 may be used independently.
template <>
struct uint<128>
{
static constexpr unsigned num_bits = 128;
uint64_t lo = 0;
uint64_t hi = 0;
constexpr uint() noexcept = default;
constexpr uint(uint64_t high, uint64_t low) noexcept : lo{low}, hi{high} {}
template <typename T,
typename = typename std::enable_if_t<std::is_convertible<T, uint64_t>::value>>
constexpr uint(T x) noexcept : lo(static_cast<uint64_t>(x)) // NOLINT
{}
#ifdef __SIZEOF_INT128__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
constexpr uint(unsigned __int128 x) noexcept // NOLINT
: lo{uint64_t(x)}, hi{uint64_t(x >> 64)}
{}
constexpr explicit operator unsigned __int128() const noexcept
{
return (static_cast<unsigned __int128>(hi) << 64) | lo;
}
#pragma GCC diagnostic pop
#endif
constexpr explicit operator bool() const noexcept { return hi | lo; }
/// Explicit converting operator for all builtin integral types.
template <typename Int, typename = typename std::enable_if<std::is_integral<Int>::value>::type>
constexpr explicit operator Int() const noexcept
{
return static_cast<Int>(lo);
}
};
using uint128 = uint<128>;
/// Contains result of add/sub/etc with a carry flag.
template <typename T>
struct result_with_carry
{
T value;
bool carry;
/// Conversion to tuple of references, to allow usage with std::tie().
constexpr operator std::tuple<T&, bool&>() noexcept { return {value, carry}; }
};
/// Linear arithmetic operators.
/// @{
constexpr inline result_with_carry<uint64_t> add_with_carry(
uint64_t x, uint64_t y, bool carry = false) noexcept
{
const auto s = x + y;
const auto carry1 = s < x;
const auto t = s + carry;
const auto carry2 = t < s;
return {t, carry1 || carry2};
}
template <unsigned N>
constexpr result_with_carry<uint<N>> add_with_carry(
const uint<N>& a, const uint<N>& b, bool carry = false) noexcept
{
const auto lo = add_with_carry(a.lo, b.lo, carry);
const auto hi = add_with_carry(a.hi, b.hi, lo.carry);
return {{hi.value, lo.value}, hi.carry};
}
constexpr inline uint128 operator+(uint128 x, uint128 y) noexcept
{
return add_with_carry(x, y).value;
}
constexpr inline uint128 operator+(uint128 x) noexcept
{
return x;
}
constexpr inline result_with_carry<uint64_t> sub_with_carry(
uint64_t x, uint64_t y, bool carry = false) noexcept
{
const auto d = x - y;
const auto carry1 = d > x;
const auto e = d - carry;
const auto carry2 = e > d;
return {e, carry1 || carry2};
}
/// Performs subtraction of two unsigned numbers and returns the difference
/// and the carry bit (aka borrow, overflow).
template <unsigned N>
constexpr inline result_with_carry<uint<N>> sub_with_carry(
const uint<N>& a, const uint<N>& b, bool carry = false) noexcept
{
const auto lo = sub_with_carry(a.lo, b.lo, carry);
const auto hi = sub_with_carry(a.hi, b.hi, lo.carry);
return {{hi.value, lo.value}, hi.carry};
}
constexpr inline uint128 operator-(uint128 x, uint128 y) noexcept
{
return sub_with_carry(x, y).value;
}
constexpr inline uint128 operator-(uint128 x) noexcept
{
// Implementing as subtraction is better than ~x + 1.
// Clang9: Perfect.
// GCC8: Does something weird.
return 0 - x;
}
inline uint128& operator++(uint128& x) noexcept
{
return x = x + 1;
}
inline uint128& operator--(uint128& x) noexcept
{
return x = x - 1;
}
inline uint128 operator++(uint128& x, int) noexcept
{
auto ret = x;
++x;
return ret;
}
inline uint128 operator--(uint128& x, int) noexcept
{
auto ret = x;
--x;
return ret;
}
/// Optimized addition.
///
/// This keeps the multiprecision addition until CodeGen so the pattern is not
/// broken during other optimizations.
constexpr uint128 fast_add(uint128 x, uint128 y) noexcept
{
#ifdef __SIZEOF_INT128__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
using uint128_native = unsigned __int128;
return uint128_native{x} + uint128_native{y};
#pragma GCC diagnostic pop
#else
// Fallback to regular addition.
return x + y;
#endif
}
/// @}
/// Comparison operators.
///
/// In all implementations bitwise operators are used instead of logical ones
/// to avoid branching.
///
/// @{
constexpr bool operator==(uint128 x, uint128 y) noexcept
{
// Clang7: generates perfect xor based code,
// much better than __int128 where it uses vector instructions.
// GCC8: generates a bit worse cmp based code
// although it generates the xor based one for __int128.
return (x.lo == y.lo) & (x.hi == y.hi);
}
constexpr bool operator!=(uint128 x, uint128 y) noexcept
{
// Analogous to ==, but == not used directly, because that confuses GCC 8-9.
return (x.lo != y.lo) | (x.hi != y.hi);
}
constexpr bool operator<(uint128 x, uint128 y) noexcept
{
// OPT: This should be implemented by checking the borrow of x - y,
// but compilers (GCC8, Clang7)
// have problem with properly optimizing subtraction.
return (x.hi < y.hi) | ((x.hi == y.hi) & (x.lo < y.lo));
}
constexpr bool operator<=(uint128 x, uint128 y) noexcept
{
return !(y < x);
}
constexpr bool operator>(uint128 x, uint128 y) noexcept
{
return y < x;
}
constexpr bool operator>=(uint128 x, uint128 y) noexcept
{
return !(x < y);
}
/// @}
/// Bitwise operators.
/// @{
constexpr uint128 operator~(uint128 x) noexcept
{
return {~x.hi, ~x.lo};
}
constexpr uint128 operator|(uint128 x, uint128 y) noexcept
{
// Clang7: perfect.
// GCC8: stupidly uses a vector instruction in all bitwise operators.
return {x.hi | y.hi, x.lo | y.lo};
}
constexpr uint128 operator&(uint128 x, uint128 y) noexcept
{
return {x.hi & y.hi, x.lo & y.lo};
}
constexpr uint128 operator^(uint128 x, uint128 y) noexcept
{
return {x.hi ^ y.hi, x.lo ^ y.lo};
}
constexpr uint128 operator<<(uint128 x, unsigned shift) noexcept
{
return (shift < 64) ?
// Find the part moved from lo to hi.
// For shift == 0 right shift by (64 - shift) is invalid so
// split it into 2 shifts by 1 and (63 - shift).
uint128{(x.hi << shift) | ((x.lo >> 1) >> (63 - shift)), x.lo << shift} :
// Guarantee "defined" behavior for shifts larger than 128.
(shift < 128) ? uint128{x.lo << (shift - 64), 0} : 0;
}
constexpr uint128 operator<<(uint128 x, uint128 shift) noexcept
{
if (shift < 128)
return x << unsigned(shift);
return 0;
}
constexpr uint128 operator>>(uint128 x, unsigned shift) noexcept
{
return (shift < 64) ?
// Find the part moved from lo to hi.
// For shift == 0 left shift by (64 - shift) is invalid so
// split it into 2 shifts by 1 and (63 - shift).
uint128{x.hi >> shift, (x.lo >> shift) | ((x.hi << 1) << (63 - shift))} :
// Guarantee "defined" behavior for shifts larger than 128.
(shift < 128) ? uint128{0, x.hi >> (shift - 64)} : 0;
}
constexpr uint128 operator>>(uint128 x, uint128 shift) noexcept
{
if (shift < 128)
return x >> unsigned(shift);
return 0;
}
/// @}
/// Multiplication
/// @{
/// Portable full unsigned multiplication 64 x 64 -> 128.
constexpr uint128 constexpr_umul(uint64_t x, uint64_t y) noexcept
{
uint64_t xl = x & 0xffffffff;
uint64_t xh = x >> 32;
uint64_t yl = y & 0xffffffff;
uint64_t yh = y >> 32;
uint64_t t0 = xl * yl;
uint64_t t1 = xh * yl;
uint64_t t2 = xl * yh;
uint64_t t3 = xh * yh;
uint64_t u1 = t1 + (t0 >> 32);
uint64_t u2 = t2 + (u1 & 0xffffffff);
uint64_t lo = (u2 << 32) | (t0 & 0xffffffff);
uint64_t hi = t3 + (u2 >> 32) + (u1 >> 32);
return {hi, lo};
}
/// Full unsigned multiplication 64 x 64 -> 128.
inline uint128 umul(uint64_t x, uint64_t y) noexcept
{
#if defined(__SIZEOF_INT128__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
const auto p = static_cast<unsigned __int128>(x) * y;
return {uint64_t(p >> 64), uint64_t(p)};
#pragma GCC diagnostic pop
#elif defined(_MSC_VER)
unsigned __int64 hi;
const auto lo = _umul128(x, y, &hi);
return {hi, lo};
#else
return constexpr_umul(x, y);
#endif
}
inline uint128 operator*(uint128 x, uint128 y) noexcept
{
auto p = umul(x.lo, y.lo);
p.hi += (x.lo * y.hi) + (x.hi * y.lo);
return {p.hi, p.lo};
}
constexpr uint128 constexpr_mul(uint128 x, uint128 y) noexcept
{
auto p = constexpr_umul(x.lo, y.lo);
p.hi += (x.lo * y.hi) + (x.hi * y.lo);
return {p.hi, p.lo};
}
/// @}
/// Assignment operators.
/// @{
constexpr uint128& operator+=(uint128& x, uint128 y) noexcept
{
return x = x + y;
}
constexpr uint128& operator-=(uint128& x, uint128 y) noexcept
{
return x = x - y;
}
inline uint128& operator*=(uint128& x, uint128 y) noexcept
{
return x = x * y;
}
constexpr uint128& operator|=(uint128& x, uint128 y) noexcept
{
return x = x | y;
}
constexpr uint128& operator&=(uint128& x, uint128 y) noexcept
{
return x = x & y;
}
constexpr uint128& operator^=(uint128& x, uint128 y) noexcept
{
return x = x ^ y;
}
constexpr uint128& operator<<=(uint128& x, unsigned shift) noexcept
{
return x = x << shift;
}
constexpr uint128& operator>>=(uint128& x, unsigned shift) noexcept
{
return x = x >> shift;
}
/// @}
constexpr unsigned clz_generic(uint32_t x) noexcept
{
unsigned n = 32;
for (int i = 4; i >= 0; --i)
{
const auto s = unsigned{1} << i;
const auto hi = x >> s;
if (hi != 0)
{
n -= s;
x = hi;
}
}
return n - x;
}
constexpr unsigned clz_generic(uint64_t x) noexcept
{
unsigned n = 64;
for (int i = 5; i >= 0; --i)
{
const auto s = unsigned{1} << i;
const auto hi = x >> s;
if (hi != 0)
{
n -= s;
x = hi;
}
}
return n - static_cast<unsigned>(x);
}
constexpr inline unsigned clz(uint32_t x) noexcept
{
#ifdef _MSC_VER
return clz_generic(x);
#else
return x != 0 ? unsigned(__builtin_clz(x)) : 32;
#endif
}
constexpr inline unsigned clz(uint64_t x) noexcept
{
#ifdef _MSC_VER
return clz_generic(x);
#else
return x != 0 ? unsigned(__builtin_clzll(x)) : 64;
#endif
}
constexpr inline unsigned clz(uint128 x) noexcept
{
// In this order `h == 0` we get less instructions than in case of `h != 0`.
return x.hi == 0 ? clz(x.lo) + 64 : clz(x.hi);
}
inline uint64_t bswap(uint64_t x) noexcept
{
#ifdef _MSC_VER
return _byteswap_uint64(x);
#else
return __builtin_bswap64(x);
#endif
}
inline uint128 bswap(uint128 x) noexcept
{
return {bswap(x.lo), bswap(x.hi)};
}
/// Division.
/// @{
template <typename QuotT, typename RemT = QuotT>
struct div_result
{
QuotT quot;
RemT rem;
/// Conversion to tuple of references, to allow usage with std::tie().
constexpr operator std::tuple<QuotT&, RemT&>() noexcept { return {quot, rem}; }
};
namespace internal
{
constexpr uint16_t reciprocal_table_item(uint8_t d9) noexcept
{
return uint16_t(0x7fd00 / (0x100 | d9));
}
#define REPEAT4(x) \
reciprocal_table_item((x) + 0), reciprocal_table_item((x) + 1), \
reciprocal_table_item((x) + 2), reciprocal_table_item((x) + 3)
#define REPEAT32(x) \
REPEAT4((x) + 4 * 0), REPEAT4((x) + 4 * 1), REPEAT4((x) + 4 * 2), REPEAT4((x) + 4 * 3), \
REPEAT4((x) + 4 * 4), REPEAT4((x) + 4 * 5), REPEAT4((x) + 4 * 6), REPEAT4((x) + 4 * 7)
#define REPEAT256() \
REPEAT32(32 * 0), REPEAT32(32 * 1), REPEAT32(32 * 2), REPEAT32(32 * 3), REPEAT32(32 * 4), \
REPEAT32(32 * 5), REPEAT32(32 * 6), REPEAT32(32 * 7)
/// Reciprocal lookup table.
constexpr uint16_t reciprocal_table[] = {REPEAT256()};
#undef REPEAT4
#undef REPEAT32
#undef REPEAT256
} // namespace internal
/// Computes the reciprocal (2^128 - 1) / d - 2^64 for normalized d.
///
/// Based on Algorithm 2 from "Improved division by invariant integers".
inline uint64_t reciprocal_2by1(uint64_t d) noexcept
{
INTX_REQUIRE(d & 0x8000000000000000); // Must be normalized.
const uint64_t d9 = d >> 55;
const uint32_t v0 = internal::reciprocal_table[d9 - 256];
const uint64_t d40 = (d >> 24) + 1;
const uint64_t v1 = (v0 << 11) - uint32_t(v0 * v0 * d40 >> 40) - 1;
const uint64_t v2 = (v1 << 13) + (v1 * (0x1000000000000000 - v1 * d40) >> 47);
const uint64_t d0 = d & 1;
const uint64_t d63 = (d >> 1) + d0; // ceil(d/2)
const uint64_t e = ((v2 >> 1) & (0 - d0)) - v2 * d63;
const uint64_t v3 = (umul(v2, e).hi >> 1) + (v2 << 31);
const uint64_t v4 = v3 - (umul(v3, d) + d).hi - d;
return v4;
}
inline uint64_t reciprocal_3by2(uint128 d) noexcept
{
auto v = reciprocal_2by1(d.hi);
auto p = d.hi * v;
p += d.lo;
if (p < d.lo)
{
--v;
if (p >= d.hi)
{
--v;
p -= d.hi;
}
p -= d.hi;
}
const auto t = umul(v, d.lo);
p += t.hi;
if (p < t.hi)
{
--v;
if (p >= d.hi)
{
if (p > d.hi || t.lo >= d.lo)
--v;
}
}
return v;
}
inline div_result<uint64_t> udivrem_2by1(uint128 u, uint64_t d, uint64_t v) noexcept
{
auto q = umul(v, u.hi);
q = fast_add(q, u);
++q.hi;
auto r = u.lo - q.hi * d;
if (r > q.lo)
{
--q.hi;
r += d;
}
if (r >= d)
{
++q.hi;
r -= d;
}
return {q.hi, r};
}
inline div_result<uint64_t, uint128> udivrem_3by2(
uint64_t u2, uint64_t u1, uint64_t u0, uint128 d, uint64_t v) noexcept
{
auto q = umul(v, u2);
q = fast_add(q, {u2, u1});
auto r1 = u1 - q.hi * d.hi;
auto t = umul(d.lo, q.hi);
auto r = uint128{r1, u0} - t - d;
r1 = r.hi;
++q.hi;
if (r1 >= q.lo)
{
--q.hi;
r += d;
}
if (r >= d)
{
++q.hi;
r -= d;
}
return {q.hi, r};
}
inline div_result<uint128> udivrem(uint128 x, uint128 y) noexcept
{
if (y.hi == 0)
{
INTX_REQUIRE(y.lo != 0); // Division by 0.
const auto lsh = clz(y.lo);
const auto rsh = (64 - lsh) % 64;
const auto rsh_mask = uint64_t{lsh == 0} - 1;
const auto yn = y.lo << lsh;
const auto xn_lo = x.lo << lsh;
const auto xn_hi = (x.hi << lsh) | ((x.lo >> rsh) & rsh_mask);
const auto xn_ex = (x.hi >> rsh) & rsh_mask;
const auto v = reciprocal_2by1(yn);
const auto res1 = udivrem_2by1({xn_ex, xn_hi}, yn, v);
const auto res2 = udivrem_2by1({res1.rem, xn_lo}, yn, v);
return {{res1.quot, res2.quot}, res2.rem >> lsh};
}
if (y.hi > x.hi)
return {0, x};
const auto lsh = clz(y.hi);
if (lsh == 0)
{
const auto q = unsigned{y.hi < x.hi} | unsigned{y.lo <= x.lo};
return {q, x - (q ? y : 0)};
}
const auto rsh = 64 - lsh;
const auto yn_lo = y.lo << lsh;
const auto yn_hi = (y.hi << lsh) | (y.lo >> rsh);
const auto xn_lo = x.lo << lsh;
const auto xn_hi = (x.hi << lsh) | (x.lo >> rsh);
const auto xn_ex = x.hi >> rsh;
const auto v = reciprocal_3by2({yn_hi, yn_lo});
const auto res = udivrem_3by2(xn_ex, xn_hi, xn_lo, {yn_hi, yn_lo}, v);
return {res.quot, res.rem >> lsh};
}
inline div_result<uint128> sdivrem(uint128 x, uint128 y) noexcept
{
constexpr auto sign_mask = uint128{1} << 127;
const auto x_is_neg = (x & sign_mask) != 0;
const auto y_is_neg = (y & sign_mask) != 0;
const auto x_abs = x_is_neg ? -x : x;
const auto y_abs = y_is_neg ? -y : y;
const auto q_is_neg = x_is_neg ^ y_is_neg;
const auto res = udivrem(x_abs, y_abs);
return {q_is_neg ? -res.quot : res.quot, x_is_neg ? -res.rem : res.rem};
}
inline uint128 operator/(uint128 x, uint128 y) noexcept
{
return udivrem(x, y).quot;
}
inline uint128 operator%(uint128 x, uint128 y) noexcept
{
return udivrem(x, y).rem;
}
inline uint128& operator/=(uint128& x, uint128 y) noexcept
{
return x = x / y;
}
inline uint128& operator%=(uint128& x, uint128 y) noexcept
{
return x = x % y;
}
/// @}
} // namespace intx
namespace std
{
template <unsigned N>
struct numeric_limits<intx::uint<N>>
{
using type = intx::uint<N>;
static constexpr bool is_specialized = true;
static constexpr bool is_integer = true;
static constexpr bool is_signed = false;
static constexpr bool is_exact = true;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr float_denorm_style has_denorm = denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr float_round_style round_style = round_toward_zero;
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr int digits = CHAR_BIT * sizeof(type);
static constexpr int digits10 = int(0.3010299956639812 * digits);
static constexpr int max_digits10 = 0;
static constexpr int radix = 2;
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool traps = std::numeric_limits<unsigned>::traps;
static constexpr bool tinyness_before = false;
static constexpr type min() noexcept { return 0; }
static constexpr type lowest() noexcept { return min(); }
static constexpr type max() noexcept { return ~type{0}; }
static constexpr type epsilon() noexcept { return 0; }
static constexpr type round_error() noexcept { return 0; }
static constexpr type infinity() noexcept { return 0; }
static constexpr type quiet_NaN() noexcept { return 0; }
static constexpr type signaling_NaN() noexcept { return 0; }
static constexpr type denorm_min() noexcept { return 0; }
};
} // namespace std
namespace intx
{
template <typename T>
[[noreturn]] inline void throw_(const char* what)
{
#if __cpp_exceptions
throw T{what};
#else
std::fputs(what, stderr);
std::abort();
#endif
}
constexpr inline int from_dec_digit(char c)
{
if (c < '0' || c > '9')
throw_<std::invalid_argument>("invalid digit");
return c - '0';
}
constexpr inline int from_hex_digit(char c)
{
if (c >= 'a' && c <= 'f')
return c - ('a' - 10);
if (c >= 'A' && c <= 'F')
return c - ('A' - 10);
return from_dec_digit(c);
}
template <typename Int>
constexpr Int from_string(const char* str)
{
auto s = str;
auto x = Int{};
int num_digits = 0;
if (s[0] == '0' && s[1] == 'x')
{
s += 2;
while (const auto c = *s++)
{
if (++num_digits > int{sizeof(x) * 2})
throw_<std::out_of_range>(str);
x = (x << 4) | from_hex_digit(c);
}
return x;
}
while (const auto c = *s++)
{
if (num_digits++ > std::numeric_limits<Int>::digits10)
throw_<std::out_of_range>(str);
const auto d = from_dec_digit(c);
x = constexpr_mul(x, Int{10}) + d;
if (x < d)
throw_<std::out_of_range>(str);
}
return x;
}
template <typename Int>
constexpr Int from_string(const std::string& s)
{
return from_string<Int>(s.c_str());
}
constexpr uint128 operator""_u128(const char* s)
{
return from_string<uint128>(s);
}
template <unsigned N>
inline std::string to_string(uint<N> x, int base = 10)
{
if (base < 2 || base > 36)
base = 10;
if (x == 0)
return "0";
auto s = std::string{};
while (x != 0)
{
// TODO: Use constexpr udivrem_1?
const auto res = udivrem(x, uint<N>{base});
const auto d = int(res.rem);
const auto c = d < 10 ? '0' + d : 'a' + d - 10;
s.push_back(char(c));
x = res.quot;
}
std::reverse(s.begin(), s.end());
return s;
}
template <unsigned N>
inline std::string hex(uint<N> x)
{
return to_string(x, 16);
}
} // namespace intx

1221
source/libs/decimal/src/detail/intx/intx.hpp Normal file
View File

File diff suppressed because it is too large Load Diff

262
source/libs/decimal/src/detail/wideInteger.cpp Normal file
View File

@ -0,0 +1,262 @@
#include "wideInteger.h"
#include "intx/int128.hpp"
#include "intx/intx.hpp"
const UInt128 uInt128Zero = {0, 0};
const uint64_t k1e18 = 1000000000000000000LL;
const UInt128 uInt128_1e18 = {k1e18, 0};
const UInt128 uInt128One = {1, 0};
const UInt128 uInt128Two = {2, 0};
void makeUInt128(uint128* pUint128, uint64_t high, uint64_t low) {
intx::uint128* pIntxUint = (intx::uint128*)pUint128;
pIntxUint->hi = high;
pIntxUint->lo = low;
}
uint64_t uInt128Hi(const UInt128* pInt) {
intx::uint128 *pIntUint = (intx::uint128*)pInt;
return pIntUint->hi;
}
uint64_t uInt128Lo(const UInt128* pInt) {
intx::uint128 *pIntUint = (intx::uint128*)pInt;
return pIntUint->lo;
}
void uInt128Add(UInt128* pLeft, const UInt128* pRight) {
intx::uint128 *pX = (intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
*pX += *pY;
}
void uInt128Subtract(UInt128* pLeft, const UInt128* pRight) {
intx::uint128 *pX = (intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
*pX -= *pY;
}
void uInt128Multiply(UInt128* pLeft, const UInt128* pRight) {
intx::uint128 *pX = (intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
*pX *= *pY;
/* __uint128_t *px = (__uint128_t*)pLeft;
const __uint128_t *py = (__uint128_t*)pRight;
*px = *px * *py; */
}
void uInt128Divide(UInt128* pLeft, const UInt128* pRight) {
intx::uint128 *pX = (intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
*pX /= *pY;
/* __uint128_t *px = (__uint128_t*)pLeft;
const __uint128_t *py = (__uint128_t*)pRight;
*px = *px / *py; */
}
void uInt128Mod(UInt128* pLeft, const UInt128* pRight) {
intx::uint128 *pX = (intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
*pX %= *pY;
/* __uint128_t *px = (__uint128_t*)pLeft;
const __uint128_t *py = (__uint128_t*)pRight;
*px = *px % *py; */
}
bool uInt128Lt(const UInt128* pLeft, const UInt128* pRight) {
const intx::uint128 *pX = (const intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
return *pX < *pY;
}
bool uInt128Gt(const UInt128* pLeft, const UInt128* pRight) {
const intx::uint128 *pX = (const intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
return *pX > *pY;
}
bool uInt128Eq(const UInt128* pLeft, const UInt128* pRight) {
const intx::uint128 *pX = (const intx::uint128*)pLeft;
const intx::uint128 *pY = (const intx::uint128*)pRight;
return *pX == *pY;
}
Int128 makeInt128(int64_t high, uint64_t low) {
Int128 int128 = {low, high};
return int128;
}
int64_t int128Hi(const Int128* pUint128) {
return pUint128->high;
}
uint64_t int128Lo(const Int128* pUint128) {
return pUint128->low;
}
Int128 int128Abs(const Int128* pInt128) {
if (int128Lt(pInt128, &int128Zero)) {
return int128Negate(pInt128);
}
return *pInt128;
}
Int128 int128Negate(const Int128* pInt128) {
uint64_t low = ~pInt128->low + 1;
int64_t high = ~pInt128->high;
if (low == 0) high += 1;
return makeInt128(high, low);
}
Int128 int128Add(const Int128* pLeft, const Int128* pRight) {
intx::uint128 result = *(intx::uint128*)pLeft + *(intx::uint128*)pRight;
return *(Int128*)&result;
}
Int128 int128Subtract(const Int128* pLeft, const Int128* pRight) {
intx::uint128 result = *(intx::uint128*)pLeft - *(intx::uint128*)pRight;
return *(Int128*)&result;
}
Int128 int128Multiply(const Int128* pLeft, const Int128* pRight) {
intx::uint128 result = *(intx::uint128*)pLeft * *(intx::uint128*)pRight;
return *(Int128*)&result;
}
Int128 int128Divide(const Int128* pLeft, const Int128* pRight) {
intx::uint128 result = *(intx::uint128*)pLeft / *(intx::uint128*)pRight;
return *(Int128*)&result;
}
Int128 int128Mod(const Int128* pLeft, const Int128* pRight) {
intx::uint128 result = *(intx::uint128*)pLeft % *(intx::uint128*)pRight;
return *(Int128*)&result;
}
bool int128Lt(const Int128* pLeft, const Int128* pRight) {
return pLeft->high < pRight->high || (pLeft->high == pRight->high && pLeft->low < pRight->low);
}
bool int128Gt(const Int128* pLeft, const Int128* pRight) {
return int128Lt(pRight, pLeft);
}
bool int128Eq(const Int128* pLeft, const Int128* pRight) {
return pLeft->high == pRight->high && pLeft->low == pRight->low;
}
Int128 int128RightShift(const Int128* pLeft, int32_t shift) {
intx::uint128 result = *(intx::uint128*)pLeft >> shift;
return *(Int128*)&result;
}
const Int128 int128Zero = {0, 0};
const Int128 int128One = {1, 0};
UInt256 makeUint256(UInt128 high, UInt128 low) {
UInt256 uint256 = {high, low};
return uint256;
}
uint128 uInt256Hi(const UInt256* pUint256) {
return pUint256->high;
}
uint128 uInt256Lo(const UInt256* pUint256) {
return pUint256->low;
}
UInt256 uInt256Add(const UInt256* pLeft, const UInt256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft + *(intx::uint256*)pRight;
return *(UInt256*)&result;
}
UInt256 uInt256Subtract(const UInt256* pLeft, const UInt256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft - *(intx::uint256*)pRight;
return *(UInt256*)&result;
}
UInt256 uInt256Multiply(const UInt256* pLeft, const UInt256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft * *(intx::uint256*)pRight;
return *(UInt256*)&result;
}
UInt256 uInt256Divide(const UInt256* pLeft, const UInt256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft / *(intx::uint256*)pRight;
return *(UInt256*)&result;
}
UInt256 uInt256Mod(const UInt256* pLeft, const UInt256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft % *(intx::uint256*)pRight;
return *(UInt256*)&result;
}
bool uInt256Lt(const UInt256* pLeft, const UInt256* pRight) {
return *(intx::uint256*)pLeft < *(intx::uint256*)pRight;
}
bool uInt256Gt(const UInt256* pLeft, const UInt256* pRight) {
return *(intx::uint256*)pLeft > *(intx::uint256*)pRight;
}
bool uInt256Eq(const UInt256* pLeft, const UInt256* pRight) {
return *(intx::uint256*)pLeft == *(intx::uint256*)pRight;
}
UInt256 uInt256RightShift(const UInt256* pLeft, int32_t shift) {
intx::uint256 result = *(intx::uint256*)pLeft >> shift;
return *(UInt256*)&result;
}
Int256 makeInt256(Int128 high, UInt128 low) {
Int256 int256 = {low, high};
return int256;
}
Int128 int256Hi(const Int256* pUint256) {
return pUint256->high;
}
UInt128 int256Lo(const Int256* pUint256) {
return pUint256->low;
}
Int256 int256Abs(const Int256* pInt256) {
if (int256Lt(pInt256, &int256Zero)) {
return int256Negate(pInt256);
}
return *pInt256;
}
Int256 int256Negate(const Int256* pInt256) {
return int256Subtract(&int256Zero, pInt256);
}
Int256 int256Add(const Int256* pLeft, const Int256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft + *(intx::uint256*)pRight;
return *(Int256*)&result;
}
Int256 int256Subtract(const Int256* pLeft, const Int256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft - *(intx::uint256*)pRight;
return *(Int256*)&result;
}
Int256 int256Multiply(const Int256* pLeft, const Int256* pRight) {
intx::uint256 result = *(intx::uint256*)pLeft * *(intx::uint256*)pRight;
return *(Int256*)&result;
}
Int256 int256Divide(const Int256* pLeft, const Int256* pRight) {
Int256 l = *pLeft, r = *pRight;
bool leftNegative = int256Lt(pLeft, &int256Zero), rightNegative = int256Lt(pRight, &int256Zero);
if (leftNegative) {
l = int256Abs(pLeft);
}
if (rightNegative) {
r = int256Abs(pRight);
}
intx::uint256 result = *(intx::uint256*)&l / *(intx::uint256*)&r;
Int256 res = *(Int256*)&result;
if (leftNegative != rightNegative)
res = int256Negate(&res);
return res;
}
Int256 int256Mod(const Int256* pLeft, const Int256* pRight) {
Int256 left = *pLeft, right = *pRight;
bool leftNegative = int256Lt(pLeft, &int256Zero);
if (leftNegative) {
left = int256Abs(&left);
}
bool rightNegate = int256Lt(pRight, &int256Zero);
if (rightNegate) right = int256Abs(pRight);
intx::uint256 result = *(intx::uint256*)&left % *(intx::uint256*)&right;
Int256 res = *(Int256*)&result;
if (leftNegative) res = int256Negate(&res);
return res;
}
bool int256Lt(const Int256* pLeft, const Int256* pRight) {
Int128 hiLeft = int256Hi(pLeft), hiRight = int256Hi(pRight);
UInt128 lowLeft = int256Lo(pLeft), lowRight = int256Lo(pRight);
return int128Lt(&hiLeft, &hiRight) || (int128Eq(&hiLeft, &hiRight) && uInt128Lt(&lowLeft, &lowRight));
}
bool int256Gt(const Int256* pLeft, const Int256* pRight) {
return int256Lt(pRight, pLeft);
}
bool int256Eq(const Int256* pLeft, const Int256* pRight) {
Int128 hiLeft = int256Hi(pLeft), hiRight = int256Hi(pRight);
UInt128 lowLeft = int256Lo(pLeft), lowRight = int256Lo(pRight);
return int128Eq(&hiLeft, &hiRight) && uInt128Eq(&lowLeft, &lowRight);
}
Int256 int256RightShift(const Int256* pLeft, int32_t shift) {
intx::uint256 result = *(intx::uint256*)pLeft >> shift;
return *(Int256*)&result;
}
const Int256 int256One = {uInt128One, int128Zero};
const Int256 int256Zero = {uInt128Zero, int128Zero};
const Int256 int256Two = {uInt128Two, int128Zero};

23
source/libs/decimal/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,23 @@
MESSAGE(STATUS "build decimal unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(decimalTest ${SOURCE_LIST})
TARGET_INCLUDE_DIRECTORIES(
decimalTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/decimal/"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
TARGET_LINK_LIBRARIES(
decimalTest
PUBLIC gtest taos os common
)
add_test(
NAME decimalTest
COMMAND decimalTest
)

1545
source/libs/decimal/test/decimalTest.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

70
source/libs/executor/inc/executorInt.h
View File

@ -393,6 +393,7 @@ typedef enum EStreamScanMode {
STREAM_SCAN_FROM_DELETE_DATA,
STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
STREAM_SCAN_FROM_DATAREADER_RANGE,
STREAM_SCAN_FROM_CREATE_TABLERES,
} EStreamScanMode;
enum {
@ -416,6 +417,7 @@ typedef struct SStreamAggSupporter {
struct SUpdateInfo* pUpdateInfo;
int32_t windowCount;
int32_t windowSliding;
SStreamStateCur* pCur;
} SStreamAggSupporter;
typedef struct SWindowSupporter {
@ -459,8 +461,17 @@ typedef struct SStreamNotifyEventSupp {
typedef struct SSteamOpBasicInfo {
int32_t primaryPkIndex;
bool updateOperatorInfo;
int16_t operatorFlag;
SStreamNotifyEventSupp notifyEventSup;
bool recvCkBlock;
SSDataBlock* pCheckpointRes;
SSHashObj* pSeDeleted;
void* pDelIterator;
SSDataBlock* pDelRes;
SArray* pUpdated;
bool destHasPrimaryKey;
STableTsDataState* pTsDataState;
int32_t numOfRecv;
} SSteamOpBasicInfo;
typedef struct SStreamFillSupporter {
@ -485,8 +496,26 @@ typedef struct SStreamFillSupporter {
int32_t pkColBytes;
__compar_fn_t comparePkColFn;
int32_t* pOffsetInfo;
bool normalFill;
void* pEmptyRow;
SArray* pResultRange;
} SStreamFillSupporter;
typedef struct SStreamRecParam {
char pSql[2048];
int32_t sqlCapcity;
char pUrl[TSDB_EP_LEN + 17]; // "http://localhost:6041/rest/sql"
char pAuth[512 + 22]; // Authorization: Basic token
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
void* pIteData;
int32_t iter;
TSKEY gap;
} SStreamRecParam;
typedef struct SStreamScanInfo {
SSteamOpBasicInfo basic;
SExprInfo* pPseudoExpr;
@ -549,6 +578,17 @@ typedef struct SStreamScanInfo {
bool useGetResultRange;
STimeWindow lastScanRange;
SSDataBlock* pRangeScanRes; // update SSDataBlock
bool hasPart;
//nonblock data scan
TSKEY recalculateInterval;
__compar_fn_t comparePkColFn;
SScanRange curRange;
struct SOperatorInfo* pRecTableScanOp;
bool scanAllTables;
SSHashObj* pRecRangeMap;
SArray* pRecRangeRes;
SStreamRecParam recParam;
} SStreamScanInfo;
typedef struct {
@ -680,11 +720,20 @@ typedef struct SWindowRowsSup {
uint64_t groupId;
} SWindowRowsSup;
typedef struct SResultWindowInfo {
SRowBuffPos* pStatePos;
SSessionKey sessionWin;
bool isOutput;
} SResultWindowInfo;
typedef int32_t (*AggImplFn)(struct SOperatorInfo* pOperator, SSDataBlock* pBlock);
typedef struct SNonBlockAggSupporter {
int32_t numOfKeep;
TSKEY tsOfKeep;
AggImplFn pWindowAggFn;
SArray* pPullWins;
int32_t pullIndex;
SSDataBlock* pPullDataRes;
SSHashObj* pHistoryGroup;
SSHashObj* pPullDataMap;
int32_t numOfChild;
SStreamRecParam recParam;
} SNonBlockAggSupporter;
typedef struct SSessionAggOperatorInfo {
SOptrBasicInfo binfo;
@ -745,7 +794,7 @@ typedef struct SStreamSessionAggOperatorInfo {
int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup;
SSDataBlock* pWinBlock; // window result
SSDataBlock* pDelRes; // delete result
SSDataBlock* pDelRes; // delete result, refactor
SSDataBlock* pUpdateRes; // update window
bool returnUpdate;
SSHashObj* pStDeleted;
@ -766,6 +815,7 @@ typedef struct SStreamSessionAggOperatorInfo {
bool destHasPrimaryKey;
SSHashObj* pPkDeleted;
struct SOperatorInfo* pOperator;
SNonBlockAggSupporter nbSup;
} SStreamSessionAggOperatorInfo;
typedef struct SStreamStateAggOperatorInfo {
@ -795,6 +845,7 @@ typedef struct SStreamStateAggOperatorInfo {
bool destHasPrimaryKey;
struct SOperatorInfo* pOperator;
int64_t trueForLimit;
SNonBlockAggSupporter nbSup;
} SStreamStateAggOperatorInfo;
typedef struct SStreamEventAggOperatorInfo {
@ -828,6 +879,7 @@ typedef struct SStreamEventAggOperatorInfo {
SNodeList* pStartCondCols;
SNodeList* pEndCondCols;
int64_t trueForLimit;
SNonBlockAggSupporter nbSup;
} SStreamEventAggOperatorInfo;
typedef struct SStreamCountAggOperatorInfo {
@ -889,6 +941,7 @@ typedef struct SStreamFillOperatorInfo {
SGroupResInfo groupResInfo;
SStreamState* pState;
SStateStore stateStore;
SNonBlockAggSupporter nbSup;
} SStreamFillOperatorInfo;
typedef struct SStreamTimeSliceOperatorInfo {
@ -948,6 +1001,7 @@ typedef struct SStreamIntervalSliceOperatorInfo {
bool hasFill;
bool hasInterpoFunc;
int32_t* pOffsetInfo;
SNonBlockAggSupporter nbSup;
} SStreamIntervalSliceOperatorInfo;
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
@ -1083,7 +1137,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex, int8_t stateType, int32_t ratio);
int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type,
int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic);
int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic, int64_t recalculateInterval);
int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins);
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey);

18
source/libs/executor/inc/operator.h
View File

@ -116,6 +116,10 @@ int32_t createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createSemiIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createFinalIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
@ -146,7 +150,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
@ -170,6 +176,16 @@ int32_t createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numO
int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createSemiSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createFinalSessionNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createStateNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
int32_t createEventNonblockOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** ppOptInfo);
// clang-format on
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,

1
source/libs/executor/inc/querytask.h
View File

@ -71,6 +71,7 @@ typedef struct {
SVersionRange fillHistoryVer;
STimeWindow fillHistoryWindow;
SStreamState* pState;
SStreamState* pOtherState;
int32_t eventTypes; // event types to notify
SSchemaWrapper* notifyResultSchema; // agg result to notify
char* stbFullName; // used to generate dest child table name

Some files were not shown because too many files have changed in this diff Show More