xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

merge from 3.0

This commit is contained in:
Liu Jicong 2022-03-28 20:33:41 +08:00
parent a5386ee102 fa9530a126
commit ea53910d4d
104 changed files with 5474 additions and 5172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
include/common/tcommon.h
View File

@ -54,16 +54,13 @@ typedef struct SColumnDataAgg {
} SColumnDataAgg; } SColumnDataAgg;
typedef struct SDataBlockInfo { typedef struct SDataBlockInfo {
STimeWindow window; STimeWindow window;
int32_t rows; int32_t rows;
int32_t rowSize; int32_t rowSize;
int16_t numOfCols; int16_t numOfCols;
int16_t hasVarCol; int16_t hasVarCol;
union { union {int64_t uid; int64_t blockId;};
int64_t uid; int64_t groupId; // no need to serialize
int64_t blockId;
};
int64_t groupId; // no need to serialize
} SDataBlockInfo; } SDataBlockInfo;
typedef struct SSDataBlock { typedef struct SSDataBlock {
@ -96,6 +93,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
int32_t tEncodeDataBlocks(void** buf, const SArray* blocks); int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
void* tDecodeDataBlocks(const void* buf, SArray** blocks); void* tDecodeDataBlocks(const void* buf, SArray** blocks);
void colDataDestroy(SColumnInfoData* pColData) ;
static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
// WARNING: do not use info.numOfCols, // WARNING: do not use info.numOfCols,
@ -103,13 +101,7 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t i = 0; i < numOfOutput; ++i) {
SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { colDataDestroy(pColInfoData);
taosMemoryFreeClear(pColInfoData->varmeta.offset);
} else {
taosMemoryFreeClear(pColInfoData->nullbitmap);
}
taosMemoryFreeClear(pColInfoData->pData);
} }
taosArrayDestroy(pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock);

48
include/common/tdatablock.h
View File

@ -101,6 +101,54 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
: ((p1_)->pData + ((r_) * (p1_)->info.bytes))) : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
// There is a placehold for each NULL value of binary or nchar type.
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type.
} else {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
}
pColumnInfoData->hasNull = true;
}
static FORCE_INLINE int32_t colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) {
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT ||
pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL);
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
*(int8_t*)p = *(int8_t*)v;
}
static FORCE_INLINE int32_t colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) {
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT);
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
*(int16_t*)p = *(int16_t*)v;
}
static FORCE_INLINE int32_t colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) {
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_INT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UINT);
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
*(int32_t*)p = *(int32_t*)v;
}
static FORCE_INLINE int32_t colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) {
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_BIGINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UBIGINT);
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
*(int64_t*)p = *(int64_t*)v;
}
static FORCE_INLINE int32_t colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) {
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_FLOAT);
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
*(float*)p = *(float*)v;
}
static FORCE_INLINE int32_t colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) {
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_DOUBLE);
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
*(double*)p = *(double*)v;
}
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource,
uint32_t numOfRow2); uint32_t numOfRow2);

32
include/common/tdataformat.h
View File

@ -63,7 +63,7 @@ extern "C" {
typedef struct { typedef struct {
col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
int32_t type : 8; // column type int32_t type : 8; // column type
int32_t bytes : 24; // column bytes (restore to int32_t in case of misuse) int32_t bytes : 24; // column bytes (0~16M)
int32_t sma : 8; // block SMA: 0, no SMA, 1, sum/min/max, 2, ... int32_t sma : 8; // block SMA: 0, no SMA, 1, sum/min/max, 2, ...
int32_t offset : 24; // point offset in STpRow after the header part. int32_t offset : 24; // point offset in STpRow after the header part.
} STColumn; } STColumn;
@ -81,12 +81,12 @@ typedef struct {
// ----------------- TSDB SCHEMA DEFINITION // ----------------- TSDB SCHEMA DEFINITION
typedef struct { typedef struct {
int32_t version; // version int32_t numOfCols; // Number of columns appended
int32_t numOfCols; // Number of columns appended schema_ver_t version; // schema version
int32_t tlen; // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + uint16_t flen; // First part length in a STpRow after the header part
// (bytes)) int32_t vlen; // pure value part length, excluded the overhead (bytes only)
uint16_t flen; // First part length in a STpRow after the header part int32_t tlen; // maximum length of a STpRow without the header part
uint16_t vlen; // pure value part length, excluded the overhead (bytes only) // (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
STColumn columns[]; STColumn columns[];
} STSchema; } STSchema;
@ -120,13 +120,13 @@ static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
// ----------------- SCHEMA BUILDER DEFINITION // ----------------- SCHEMA BUILDER DEFINITION
typedef struct { typedef struct {
int32_t tCols; int32_t tCols;
int32_t nCols; int32_t nCols;
int32_t tlen; schema_ver_t version;
uint16_t flen; uint16_t flen;
uint16_t vlen; int32_t vlen;
int32_t version; int32_t tlen;
STColumn *columns; STColumn *columns;
} STSchemaBuilder; } STSchemaBuilder;
#define TD_VTYPE_BITS 2 // val type #define TD_VTYPE_BITS 2 // val type
@ -136,9 +136,9 @@ typedef struct {
#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS)) #define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8)) #define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes); int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);

4
include/common/tmsg.h
View File

@ -363,7 +363,7 @@ typedef struct {
int8_t createType; int8_t createType;
int8_t superUser; // denote if it is a super user or not int8_t superUser; // denote if it is a super user or not
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN]; char pass[TSDB_USET_PASSWORD_LEN];
} SCreateUserReq; } SCreateUserReq;
int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq);
@ -373,7 +373,7 @@ typedef struct {
int8_t alterType; int8_t alterType;
int8_t superUser; int8_t superUser;
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN]; char pass[TSDB_USET_PASSWORD_LEN];
char dbname[TSDB_DB_FNAME_LEN]; char dbname[TSDB_DB_FNAME_LEN];
} SAlterUserReq; } SAlterUserReq;

201
include/common/ttokendef.h
View File

@ -82,102 +82,111 @@
#define TK_SINGLE_STABLE 64 #define TK_SINGLE_STABLE 64
#define TK_STREAM_MODE 65 #define TK_STREAM_MODE 65
#define TK_RETENTIONS 66 #define TK_RETENTIONS 66
#define TK_FILE_FACTOR 67 #define TK_TABLE 67
#define TK_NK_FLOAT 68 #define TK_NK_LP 68
#define TK_TABLE 69 #define TK_NK_RP 69
#define TK_NK_LP 70 #define TK_STABLE 70
#define TK_NK_RP 71 #define TK_ADD 71
#define TK_STABLE 72 #define TK_COLUMN 72
#define TK_ADD 73 #define TK_MODIFY 73
#define TK_COLUMN 74 #define TK_RENAME 74
#define TK_MODIFY 75 #define TK_TAG 75
#define TK_RENAME 76 #define TK_SET 76
#define TK_TAG 77 #define TK_NK_EQ 77
#define TK_SET 78 #define TK_USING 78
#define TK_NK_EQ 79 #define TK_TAGS 79
#define TK_USING 80 #define TK_NK_DOT 80
#define TK_TAGS 81 #define TK_NK_COMMA 81
#define TK_NK_DOT 82 #define TK_COMMENT 82
#define TK_NK_COMMA 83 #define TK_BOOL 83
#define TK_COMMENT 84 #define TK_TINYINT 84
#define TK_BOOL 85 #define TK_SMALLINT 85
#define TK_TINYINT 86 #define TK_INT 86
#define TK_SMALLINT 87 #define TK_INTEGER 87
#define TK_INT 88 #define TK_BIGINT 88
#define TK_INTEGER 89 #define TK_FLOAT 89
#define TK_BIGINT 90 #define TK_DOUBLE 90
#define TK_FLOAT 91 #define TK_BINARY 91
#define TK_DOUBLE 92 #define TK_TIMESTAMP 92
#define TK_BINARY 93 #define TK_NCHAR 93
#define TK_TIMESTAMP 94 #define TK_UNSIGNED 94
#define TK_NCHAR 95 #define TK_JSON 95
#define TK_UNSIGNED 96 #define TK_VARCHAR 96
#define TK_JSON 97 #define TK_MEDIUMBLOB 97
#define TK_VARCHAR 98 #define TK_BLOB 98
#define TK_MEDIUMBLOB 99 #define TK_VARBINARY 99
#define TK_BLOB 100 #define TK_DECIMAL 100
#define TK_VARBINARY 101 #define TK_SMA 101
#define TK_DECIMAL 102 #define TK_ROLLUP 102
#define TK_SMA 103 #define TK_FILE_FACTOR 103
#define TK_ROLLUP 104 #define TK_NK_FLOAT 104
#define TK_SHOW 105 #define TK_DELAY 105
#define TK_DATABASES 106 #define TK_SHOW 106
#define TK_TABLES 107 #define TK_DATABASES 107
#define TK_STABLES 108 #define TK_TABLES 108
#define TK_MNODES 109 #define TK_STABLES 109
#define TK_MODULES 110 #define TK_MNODES 110
#define TK_QNODES 111 #define TK_MODULES 111
#define TK_FUNCTIONS 112 #define TK_QNODES 112
#define TK_INDEXES 113 #define TK_FUNCTIONS 113
#define TK_FROM 114 #define TK_INDEXES 114
#define TK_LIKE 115 #define TK_FROM 115
#define TK_INDEX 116 #define TK_LIKE 116
#define TK_FULLTEXT 117 #define TK_INDEX 117
#define TK_FUNCTION 118 #define TK_FULLTEXT 118
#define TK_INTERVAL 119 #define TK_FUNCTION 119
#define TK_TOPIC 120 #define TK_INTERVAL 120
#define TK_AS 121 #define TK_TOPIC 121
#define TK_NK_BOOL 122 #define TK_AS 122
#define TK_NK_VARIABLE 123 #define TK_NK_BOOL 123
#define TK_BETWEEN 124 #define TK_NK_VARIABLE 124
#define TK_IS 125 #define TK_NK_UNDERLINE 125
#define TK_NULL 126 #define TK_ROWTS 126
#define TK_NK_LT 127 #define TK_TBNAME 127
#define TK_NK_GT 128 #define TK_QSTARTTS 128
#define TK_NK_LE 129 #define TK_QENDTS 129
#define TK_NK_GE 130 #define TK_WSTARTTS 130
#define TK_NK_NE 131 #define TK_WENDTS 131
#define TK_MATCH 132 #define TK_WDURATION 132
#define TK_NMATCH 133 #define TK_BETWEEN 133
#define TK_IN 134 #define TK_IS 134
#define TK_JOIN 135 #define TK_NULL 135
#define TK_INNER 136 #define TK_NK_LT 136
#define TK_SELECT 137 #define TK_NK_GT 137
#define TK_DISTINCT 138 #define TK_NK_LE 138
#define TK_WHERE 139 #define TK_NK_GE 139
#define TK_PARTITION 140 #define TK_NK_NE 140
#define TK_BY 141 #define TK_MATCH 141
#define TK_SESSION 142 #define TK_NMATCH 142
#define TK_STATE_WINDOW 143 #define TK_IN 143
#define TK_SLIDING 144 #define TK_JOIN 144
#define TK_FILL 145 #define TK_INNER 145
#define TK_VALUE 146 #define TK_SELECT 146
#define TK_NONE 147 #define TK_DISTINCT 147
#define TK_PREV 148 #define TK_WHERE 148
#define TK_LINEAR 149 #define TK_PARTITION 149
#define TK_NEXT 150 #define TK_BY 150
#define TK_GROUP 151 #define TK_SESSION 151
#define TK_HAVING 152 #define TK_STATE_WINDOW 152
#define TK_ORDER 153 #define TK_SLIDING 153
#define TK_SLIMIT 154 #define TK_FILL 154
#define TK_SOFFSET 155 #define TK_VALUE 155
#define TK_LIMIT 156 #define TK_NONE 156
#define TK_OFFSET 157 #define TK_PREV 157
#define TK_ASC 158 #define TK_LINEAR 158
#define TK_DESC 159 #define TK_NEXT 159
#define TK_NULLS 160 #define TK_GROUP 160
#define TK_FIRST 161 #define TK_HAVING 161
#define TK_LAST 162 #define TK_ORDER 162
#define TK_SLIMIT 163
#define TK_SOFFSET 164
#define TK_LIMIT 165
#define TK_OFFSET 166
#define TK_ASC 167
#define TK_DESC 168
#define TK_NULLS 169
#define TK_FIRST 170
#define TK_LAST 171
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301

1
include/common/ttypes.h
View File

@ -30,6 +30,7 @@ typedef uint8_t TDRowValT;
typedef int16_t col_id_t; typedef int16_t col_id_t;
typedef int8_t col_type_t; typedef int8_t col_type_t;
typedef int32_t col_bytes_t; typedef int32_t col_bytes_t;
typedef uint16_t schema_ver_t;
#pragma pack(push, 1) #pragma pack(push, 1)
typedef struct { typedef struct {

43
include/libs/function/function.h
View File

@ -27,16 +27,22 @@ extern "C" {
struct SqlFunctionCtx; struct SqlFunctionCtx;
struct SResultRowEntryInfo; struct SResultRowEntryInfo;
typedef struct SFunctionNode SFunctionNode; struct SFunctionNode;
typedef struct SScalarParam SScalarParam;
typedef struct SFuncExecEnv { typedef struct SFuncExecEnv {
int32_t calcMemSize; int32_t calcMemSize;
} SFuncExecEnv; } SFuncExecEnv;
typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx); typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx); typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef struct SScalarFuncExecFuncs {
FScalarExecProcess process;
} SScalarFuncExecFuncs;
typedef struct SFuncExecFuncs { typedef struct SFuncExecFuncs {
FExecGetEnv getEnv; FExecGetEnv getEnv;
@ -191,6 +197,7 @@ typedef struct SqlFunctionCtx {
SPoint1 start; SPoint1 start;
SPoint1 end; SPoint1 end;
SFuncExecFuncs fpSet; SFuncExecFuncs fpSet;
SScalarFuncExecFuncs sfp;
} SqlFunctionCtx; } SqlFunctionCtx;
enum { enum {
@ -203,7 +210,7 @@ enum {
}; };
typedef struct tExprNode { typedef struct tExprNode {
uint8_t nodeType; int32_t nodeType;
union { union {
struct { struct {
int32_t optr; // binary operator int32_t optr; // binary operator
@ -219,7 +226,7 @@ typedef struct tExprNode {
char functionName[FUNCTIONS_NAME_MAX_LENGTH]; // todo refactor char functionName[FUNCTIONS_NAME_MAX_LENGTH]; // todo refactor
int32_t functionId; int32_t functionId;
int32_t num; int32_t num;
SFunctionNode *pFunctNode; struct SFunctionNode *pFunctNode;
// Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the
// calculation instead. // calculation instead.
// E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes. // E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes.
@ -227,6 +234,10 @@ typedef struct tExprNode {
// operator and is kept in the attribute of _node. // operator and is kept in the attribute of _node.
struct tExprNode **pChild; struct tExprNode **pChild;
} _function; } _function;
struct {
struct SNode* pRootNode;
} _optrRoot;
}; };
} tExprNode; } tExprNode;
@ -250,25 +261,11 @@ typedef struct SAggFunctionInfo {
int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
} SAggFunctionInfo; } SAggFunctionInfo;
typedef struct SScalarParam { struct SScalarParam {
void *data; SColumnInfoData *columnData;
union { SHashObj *pHashFilter;
SColumnInfoData *columnData; int32_t numOfRows;
void *data; };
} orig;
char *bitmap;
bool dataInBlock;
int32_t num;
int32_t type;
int32_t bytes;
} SScalarParam;
typedef struct SScalarFunctionInfo {
char name[FUNCTIONS_NAME_MAX_LENGTH];
int8_t type; // scalar function or aggregation function
uint32_t functionId; // index of scalar function
void (*process)(struct SScalarParam* pOutput, size_t numOfInput, const struct SScalarParam *pInput);
} SScalarFunctionInfo;
typedef struct SMultiFunctionsDesc { typedef struct SMultiFunctionsDesc {
bool stableQuery; bool stableQuery;

20
include/libs/function/functionMgt.h
View File

@ -96,20 +96,22 @@ typedef enum EFunctionType {
FUNCTION_TYPE_SERVER_SERSION, FUNCTION_TYPE_SERVER_SERSION,
FUNCTION_TYPE_SERVER_STATUS, FUNCTION_TYPE_SERVER_STATUS,
FUNCTION_TYPE_CURRENT_USER, FUNCTION_TYPE_CURRENT_USER,
FUNCTION_TYPE_USER FUNCTION_TYPE_USER,
// pseudo column function
FUNCTION_TYPE_ROWTS = 3500,
FUNCTION_TYPE_TBNAME,
FUNCTION_TYPE_QSTARTTS,
FUNCTION_TYPE_QENDTS,
FUNCTION_TYPE_WSTARTTS,
FUNCTION_TYPE_WENDTS,
FUNCTION_TYPE_WDURATION
} EFunctionType; } EFunctionType;
struct SqlFunctionCtx; struct SqlFunctionCtx;
struct SResultRowEntryInfo; struct SResultRowEntryInfo;
struct STimeWindow; struct STimeWindow;
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef struct SScalarFuncExecFuncs {
FScalarExecProcess process;
} SScalarFuncExecFuncs;
int32_t fmFuncMgtInit(); int32_t fmFuncMgtInit();
void fmFuncMgtDestroy(); void fmFuncMgtDestroy();
@ -125,6 +127,8 @@ bool fmIsStringFunc(int32_t funcId);
bool fmIsDatetimeFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId);
bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId);
bool fmIsTimeorderFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId);
bool fmIsWindowPseudoColumnFunc(int32_t funcId);
bool fmIsWindowClauseFunc(int32_t funcId);
int32_t fmFuncScanType(int32_t funcId); int32_t fmFuncScanType(int32_t funcId);

4
include/libs/nodes/cmdnodes.h
View File

@ -41,6 +41,7 @@ typedef struct SDatabaseOptions {
int32_t numOfVgroups; int32_t numOfVgroups;
int8_t singleStable; int8_t singleStable;
int8_t streamMode; int8_t streamMode;
SNodeList* pRetentions;
} SDatabaseOptions; } SDatabaseOptions;
typedef struct SCreateDatabaseStmt { typedef struct SCreateDatabaseStmt {
@ -73,6 +74,9 @@ typedef struct STableOptions {
int32_t ttl; int32_t ttl;
char comments[TSDB_STB_COMMENT_LEN]; char comments[TSDB_STB_COMMENT_LEN];
SNodeList* pSma; SNodeList* pSma;
SNodeList* pFuncs;
float filesFactor;
int32_t delay;
} STableOptions; } STableOptions;
typedef struct SColumnDefNode { typedef struct SColumnDefNode {

2
include/libs/scalar/scalar.h
View File

@ -40,7 +40,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut);
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

12
include/util/tdef.h
View File

@ -369,6 +369,18 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_DB_CACHE_LAST_ROW 3 #define TSDB_MAX_DB_CACHE_LAST_ROW 3
#define TSDB_DEFAULT_CACHE_LAST_ROW 0 #define TSDB_DEFAULT_CACHE_LAST_ROW 0
#define TSDB_MIN_DB_STREAM_MODE 0
#define TSDB_MAX_DB_STREAM_MODE 1
#define TSDB_DEFAULT_DB_STREAM_MODE 0
#define TSDB_MIN_DB_FILE_FACTOR 0
#define TSDB_MAX_DB_FILE_FACTOR 1
#define TSDB_DEFAULT_DB_FILE_FACTOR 0.1
#define TSDB_MIN_DB_DELAY 1
#define TSDB_MAX_DB_DELAY 10
#define TSDB_DEFAULT_DB_DELAY 2
#define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_JOIN_TABLE_NUM 10
#define TSDB_MAX_UNION_CLAUSE 5 #define TSDB_MAX_UNION_CLAUSE 5

2
include/util/tqueue.h
View File

@ -56,7 +56,7 @@ void taosCloseQueue(STaosQueue *queue);
void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
void *taosAllocateQitem(int32_t size); void *taosAllocateQitem(int32_t size);
void taosFreeQitem(void *pItem); void taosFreeQitem(void *pItem);
int32_t taosWriteQitem(STaosQueue *queue, void *pItem); void taosWriteQitem(STaosQueue *queue, void *pItem);
int32_t taosReadQitem(STaosQueue *queue, void **ppItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem);
bool taosQueueEmpty(STaosQueue *queue); bool taosQueueEmpty(STaosQueue *queue);
int32_t taosQueueSize(STaosQueue *queue); int32_t taosQueueSize(STaosQueue *queue);

6
include/util/tworker.h
View File

@ -70,8 +70,8 @@ void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue);
typedef struct { typedef struct {
const char *name; const char *name;
int32_t minNum; int32_t min;
int32_t maxNum; int32_t max;
FItem fp; FItem fp;
void *param; void *param;
} SSingleWorkerCfg; } SSingleWorkerCfg;
@ -84,7 +84,7 @@ typedef struct {
typedef struct { typedef struct {
const char *name; const char *name;
int32_t maxNum; int32_t max;
FItems fp; FItems fp;
void *param; void *param;
} SMultiWorkerCfg; } SMultiWorkerCfg;

2
source/client/inc/clientInt.h
View File

@ -235,7 +235,7 @@ void initMsgHandleFp();
TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port); uint16_t port);
void* doFetchRow(SRequestObj* pRequest); void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr);
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);

102
source/client/src/clientImpl.c
View File

@ -545,7 +545,33 @@ TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, c
return taos_connect(ipStr, userStr, passStr, dbStr, port); return taos_connect(ipStr, userStr, passStr, dbStr, port);
} }
void* doFetchRow(SRequestObj* pRequest) { static void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
SResultColumn* pCol = &pResultInfo->pCol[i];
int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes;
if (IS_VAR_DATA_TYPE(type)) {
if (pCol->offset[pResultInfo->current] != -1) {
char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
pResultInfo->length[i] = varDataLen(pStart);
pResultInfo->row[i] = varDataVal(pStart);
} else {
pResultInfo->row[i] = NULL;
}
} else {
if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) {
pResultInfo->row[i] = pResultInfo->pCol[i].pData + bytes * pResultInfo->current;
} else {
pResultInfo->row[i] = NULL;
}
}
}
}
void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr) {
assert(pRequest != NULL); assert(pRequest != NULL);
SReqResultInfo* pResultInfo = &pRequest->body.resInfo; SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@ -555,17 +581,20 @@ void* doFetchRow(SRequestObj* pRequest) {
if (pRequest->type == TDMT_VND_QUERY) { if (pRequest->type == TDMT_VND_QUERY) {
// All data has returned to App already, no need to try again // All data has returned to App already, no need to try again
if (pResultInfo->completed) { if (pResultInfo->completed) {
pResultInfo->numOfRows = 0;
return NULL; return NULL;
} }
SReqResultInfo* pResInfo = &pRequest->body.resInfo; SReqResultInfo* pResInfo = &pRequest->body.resInfo;
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
return NULL; return NULL;
} }
pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData); pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
return NULL; return NULL;
} }
@ -633,41 +662,11 @@ void* doFetchRow(SRequestObj* pRequest) {
} }
_return: _return:
if (setupOneRowPtr) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { doSetOneRowPtr(pResultInfo);
SResultColumn* pCol = &pResultInfo->pCol[i]; pResultInfo->current += 1;
int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes;
if (IS_VAR_DATA_TYPE(type)) {
if (pCol->offset[pResultInfo->current] != -1) {
char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
pResultInfo->length[i] = varDataLen(pStart);
pResultInfo->row[i] = varDataVal(pStart);
if (type == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(pResultInfo->convertBuf[i]));
ASSERT(len <= bytes);
pResultInfo->row[i] = varDataVal(pResultInfo->convertBuf[i]);
varDataSetLen(pResultInfo->convertBuf[i], len);
pResultInfo->length[i] = len;
}
} else {
pResultInfo->row[i] = NULL;
}
} else {
if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) {
pResultInfo->row[i] = pResultInfo->pCol[i].pData + bytes * pResultInfo->current;
} else {
pResultInfo->row[i] = NULL;
}
}
} }
pResultInfo->current += 1;
return pResultInfo->row; return pResultInfo->row;
} }
@ -681,12 +680,6 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL || pResInfo->convertBuf == NULL) { if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL || pResInfo->convertBuf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
for(int32_t i = 0; i < pResInfo->numOfCols; ++i) {
if(pResInfo->fields[i].type == TSDB_DATA_TYPE_NCHAR) {
pResInfo->convertBuf[i] = taosMemoryCalloc(1, NCHAR_WIDTH_TO_BYTES(pResInfo->fields[i].bytes));
}
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -723,6 +716,35 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
pStart += colLength[i]; pStart += colLength[i];
} }
for (int32_t i = 0; i < numOfCols; ++i) {
int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes;
if (type == TSDB_DATA_TYPE_NCHAR) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pResultInfo->convertBuf[i] = p;
SResultColumn* pCol = &pResultInfo->pCol[i];
for (int32_t j = 0; j < numOfRows; ++j) {
if (pCol->offset[j] != -1) {
pStart = pCol->offset[j] + pCol->pData;
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p));
ASSERT(len <= bytes);
varDataSetLen(p, len);
pCol->offset[j] = (p - pResultInfo->convertBuf[i]);
p += (len + VARSTR_HEADER_SIZE);
}
}
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
}
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

45
source/client/src/clientMain.c
View File

@ -138,20 +138,20 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) {
return taos_query_l(taos, sql, (int32_t) strlen(sql)); return taos_query_l(taos, sql, (int32_t) strlen(sql));
} }
TAOS_ROW taos_fetch_row(TAOS_RES *pRes) { TAOS_ROW taos_fetch_row(TAOS_RES *res) {
if (pRes == NULL) { if (res == NULL) {
return NULL; return NULL;
} }
SRequestObj *pRequest = (SRequestObj *) pRes; SRequestObj *pRequest = (SRequestObj *) res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pRequest->type == TSDB_SQL_INSERT || pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS || pRequest->code != TSDB_CODE_SUCCESS ||
taos_num_fields(pRes) == 0) { taos_num_fields(res) == 0) {
return NULL; return NULL;
} }
return doFetchRow(pRequest); return doFetchRow(pRequest, true);
} }
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
@ -246,6 +246,7 @@ int* taos_fetch_lengths(TAOS_RES *res) {
return ((SRequestObj*) res)->body.resInfo.length; return ((SRequestObj*) res)->body.resInfo.length;
} }
// todo intergrate with tDataTypes
const char *taos_data_type(int type) { const char *taos_data_type(int type) {
switch (type) { switch (type) {
case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL"; case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL";
@ -256,9 +257,11 @@ const char *taos_data_type(int type) {
case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT"; case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT";
case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT"; case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT";
case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE"; case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE";
case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY"; case TSDB_DATA_TYPE_VARCHAR: return "TSDB_DATA_TYPE_VARCHAR";
// case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_VARCHAR";
case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP"; case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP";
case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR"; case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR";
case TSDB_DATA_TYPE_JSON: return "TSDB_DATA_TYPE_JSON";
default: return "UNKNOWN"; default: return "UNKNOWN";
} }
} }
@ -316,11 +319,37 @@ void taos_stop_query(TAOS_RES *res) {
} }
bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
return false; SRequestObj* pRequestObj = res;
SReqResultInfo* pResultInfo = &pRequestObj->body.resInfo;
if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) {
return true;
}
SResultColumn* pCol = &pRequestObj->body.resInfo.pCol[col];
return colDataIsNull_f(pCol->nullbitmap, row);
} }
int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
return 0; if (res == NULL) {
return 0;
}
SRequestObj *pRequest = (SRequestObj *) res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS ||
taos_num_fields(res) == 0) {
return 0;
}
doFetchRow(pRequest, false);
// TODO refactor
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
pResultInfo->current = pResultInfo->numOfRows;
*rows = pResultInfo->row;
return pResultInfo->numOfRows;
} }
int taos_validate_sql(TAOS *taos, const char *sql) { int taos_validate_sql(TAOS *taos, const char *sql) {

10
source/common/src/tdatablock.c
View File

@ -1241,6 +1241,16 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock)); return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock));
} }
void colDataDestroy(SColumnInfoData* pColData) {
if (IS_VAR_DATA_TYPE(pColData->info.type)) {
taosMemoryFree(pColData->varmeta.offset);
} else {
taosMemoryFree(pColData->nullbitmap);
}
taosMemoryFree(pColData->pData);
}
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
int64_t tbUid = pBlock->info.uid; int64_t tbUid = pBlock->info.uid;
int16_t numOfCols = pBlock->info.numOfCols; int16_t numOfCols = pBlock->info.numOfCols;

7
source/common/src/tdataformat.c
View File

@ -123,7 +123,7 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
return buf; return buf;
} }
int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
if (pBuilder == NULL) return -1; if (pBuilder == NULL) return -1;
pBuilder->tCols = 256; pBuilder->tCols = 256;
@ -140,7 +140,7 @@ void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) {
} }
} }
void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
pBuilder->nCols = 0; pBuilder->nCols = 0;
pBuilder->tlen = 0; pBuilder->tlen = 0;
pBuilder->flen = 0; pBuilder->flen = 0;
@ -168,6 +168,9 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]); colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
} }
// TODO: set sma value by user input
pCol->sma = 1;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
colSetBytes(pCol, bytes); colSetBytes(pCol, bytes);
pBuilder->tlen += (TYPE_BYTES[type] + bytes); pBuilder->tlen += (TYPE_BYTES[type] + bytes);

2
source/dnode/mgmt/bnode/inc/bmInt.h
View File

@ -43,7 +43,7 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
// bmWorker.c // bmWorker.c
int32_t bmStartWorker(SBnodeMgmt *pMgmt); int32_t bmStartWorker(SBnodeMgmt *pMgmt);
void bmStopWorker(SBnodeMgmt *pMgmt); void bmStopWorker(SBnodeMgmt *pMgmt);
int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

8
source/dnode/mgmt/bnode/src/bmWorker.c
View File

@ -63,15 +63,17 @@ static void bmProcessQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
} }
int32_t bmProcessWriteMsg(SBnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SBnodeMgmt *pMgmt = pWrapper->pMgmt;
SMultiWorker *pWorker = &pMgmt->writeWorker; SMultiWorker *pWorker = &pMgmt->writeWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t bmStartWorker(SBnodeMgmt *pMgmt) { int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
SMultiWorkerCfg cfg = {.maxNum = 1, .name = "bnode-write", .fp = (FItems)bmProcessQueue, .param = pMgmt}; SMultiWorkerCfg cfg = {.max = 1, .name = "bnode-write", .fp = (FItems)bmProcessQueue, .param = pMgmt};
if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) { if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) {
dError("failed to start bnode write worker since %s", terrstr()); dError("failed to start bnode write worker since %s", terrstr());
return -1; return -1;

2
source/dnode/mgmt/container/inc/dnd.h
View File

@ -63,7 +63,7 @@ typedef struct SQnodeMgmt SQnodeMgmt;
typedef struct SSnodeMgmt SSnodeMgmt; typedef struct SSnodeMgmt SSnodeMgmt;
typedef struct SBnodeMgmt SBnodeMgmt; typedef struct SBnodeMgmt SBnodeMgmt;
typedef int32_t (*NodeMsgFp)(void *pMgmt, SNodeMsg *pMsg); typedef int32_t (*NodeMsgFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
typedef int32_t (*OpenNodeFp)(SMgmtWrapper *pWrapper); typedef int32_t (*OpenNodeFp)(SMgmtWrapper *pWrapper);
typedef void (*CloseNodeFp)(SMgmtWrapper *pWrapper); typedef void (*CloseNodeFp)(SMgmtWrapper *pWrapper);
typedef int32_t (*StartNodeFp)(SMgmtWrapper *pWrapper); typedef int32_t (*StartNodeFp)(SMgmtWrapper *pWrapper);

20
source/dnode/mgmt/container/src/dndExec.c
View File

@ -108,23 +108,25 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) {
} }
static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) { static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
dndCleanupServer(pDnode); // dndCleanupServer(pDnode);
for (ENodeType n = 0; n < NODE_MAX; ++n) { for (ENodeType n = 0; n < NODE_MAX; ++n) {
if (except == n) continue; if (except == n) continue;
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
dndCloseNode(pWrapper); pWrapper->required = false;
} }
} }
static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) { static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) {
dTrace("msg:%p, get from child queue", pMsg);
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
pRpc->pCont = pCont; pRpc->pCont = pCont;
dTrace("msg:%p, get from child process queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pRpc->msgType),
pRpc->handle, pRpc->ahandle);
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
int32_t code = (*msgFp)(pWrapper, pMsg); int32_t code = (*msgFp)(pWrapper, pMsg);
if (code != 0) { if (code != 0) {
dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
if (pRpc->msgType & 1U) { if (pRpc->msgType & 1U) {
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
dndSendRsp(pWrapper, &rsp); dndSendRsp(pWrapper, &rsp);
@ -136,11 +138,13 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t
} }
} }
static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, int32_t msgLen, void *pCont, int32_t contLen) {
dTrace("msg:%p, get from parent queue", pRsp); pRpc->pCont = pCont;
pRsp->pCont = pCont; dTrace("msg:%p, get from parent process queue, type:%s handle:%p app:%p", pRpc, TMSG_INFO(pRpc->msgType),
dndSendRsp(pWrapper, pRsp); pRpc->handle, pRpc->ahandle);
taosMemoryFree(pRsp);
dndSendRsp(pWrapper, pRpc);
taosMemoryFree(pRpc);
} }
static int32_t dndRunInMultiProcess(SDnode *pDnode) { static int32_t dndRunInMultiProcess(SDnode *pDnode) {

2
source/dnode/mgmt/container/src/dndMsg.c
View File

@ -62,7 +62,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
if (pWrapper->procType == PROC_SINGLE) { if (pWrapper->procType == PROC_SINGLE) {
dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user);
code = (*msgFp)(pWrapper->pMgmt, pMsg); code = (*msgFp)(pWrapper, pMsg);
} else if (pWrapper->procType == PROC_PARENT) { } else if (pWrapper->procType == PROC_PARENT) {
dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
pRpc->ahandle, pMsg->user); pRpc->ahandle, pMsg->user);

2
source/dnode/mgmt/dnode/inc/dmInt.h
View File

@ -54,7 +54,7 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
int32_t dmStartWorker(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt);
void dmStopWorker(SDnodeMgmt *pMgmt); void dmStopWorker(SDnodeMgmt *pMgmt);
int32_t dmStartThread(SDnodeMgmt *pMgmt); int32_t dmStartThread(SDnodeMgmt *pMgmt);
int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

26
source/dnode/mgmt/dnode/src/dmMsg.c
View File

@ -118,19 +118,19 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
void dmInitMsgHandles(SMgmtWrapper *pWrapper) { void dmInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by DNODE // Requests handled by DNODE
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, VND_VGID);
// Requests handled by MNODE // Requests handled by MNODE
dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, VND_VGID);
} }

10
source/dnode/mgmt/dnode/src/dmWorker.c
View File

@ -101,14 +101,14 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
int32_t dmStartWorker(SDnodeMgmt *pMgmt) { int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
SSingleWorkerCfg mgmtCfg = { SSingleWorkerCfg mgmtCfg = {
.minNum = 1, .maxNum = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; .min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) { if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) {
dError("failed to start dnode mgmt worker since %s", terrstr()); dError("failed to start dnode mgmt worker since %s", terrstr());
return -1; return -1;
} }
SSingleWorkerCfg statusCfg = { SSingleWorkerCfg statusCfg = {
.minNum = 1, .maxNum = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; .min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) { if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) {
dError("failed to start dnode status worker since %s", terrstr()); dError("failed to start dnode status worker since %s", terrstr());
return -1; return -1;
@ -140,12 +140,14 @@ void dmStopWorker(SDnodeMgmt *pMgmt) {
dDebug("dnode workers are closed"); dDebug("dnode workers are closed");
} }
int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SDnodeMgmt *pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->mgmtWorker; SSingleWorker *pWorker = &pMgmt->mgmtWorker;
if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) {
pWorker = &pMgmt->statusWorker; pWorker = &pMgmt->statusWorker;
} }
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }

13
source/dnode/mgmt/mnode/inc/mmInt.h
View File

@ -55,13 +55,14 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
// mmWorker.c // mmWorker.c
int32_t mmStartWorker(SMnodeMgmt *pMgmt); int32_t mmStartWorker(SMnodeMgmt *pMgmt);
void mmStopWorker(SMnodeMgmt *pMgmt); void mmStopWorker(SMnodeMgmt *pMgmt);
int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t mmProcessQueryMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg);
int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg);
int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
int32_t mmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
source/dnode/mgmt/mnode/src/mmInt.c
View File

@ -48,6 +48,7 @@ static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue; msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue;
msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue; msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue;
msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue; msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue;
msgCb.queueFps[SYNC_QUEUE] = mmPutMsgToWriteQueue;
msgCb.sendReqFp = dndSendReqToDnode; msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode; msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp; msgCb.sendRspFp = dndSendRsp;

162
source/dnode/mgmt/mnode/src/mmMsg.c
View File

@ -75,91 +75,91 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
void mmInitMsgHandles(SMgmtWrapper *pWrapper) { void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by DNODE // Requests handled by DNODE
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, VND_VGID);
// Requests handled by MNODE // Requests handled by MNODE
dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, VND_VGID);
// Requests handled by VNODE // Requests handled by VNODE
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MND_VGID);
} }

105
source/dnode/mgmt/mnode/src/mmWorker.c
View File

@ -19,7 +19,7 @@
static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SMnodeMgmt *pMgmt = pInfo->ahandle; SMnodeMgmt *pMgmt = pInfo->ahandle;
dTrace("msg:%p, will be processed in mnode queue", pMsg); dTrace("msg:%p, get from mnode queue", pMsg);
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
int32_t code = -1; int32_t code = -1;
@ -31,9 +31,11 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
} }
if (pRpc->msgType & 1U) { if (pRpc->msgType & 1U) {
if (pRpc->handle == NULL) return; if (pRpc->handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { if (code != 0) {
if (code != 0) code = terrno; code = terrno;
dError("msg:%p, failed to process since %s", pMsg, terrstr());
}
SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp};
dndSendRsp(pMgmt->pWrapper, &rsp); dndSendRsp(pMgmt->pWrapper, &rsp);
} }
@ -47,7 +49,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
SMnodeMgmt *pMgmt = pInfo->ahandle; SMnodeMgmt *pMgmt = pInfo->ahandle;
dTrace("msg:%p, will be processed in mnode queue", pMsg); dTrace("msg:%p, get from mnode query queue", pMsg);
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
int32_t code = -1; int32_t code = -1;
@ -55,8 +57,8 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
code = mndProcessMsg(pMsg); code = mndProcessMsg(pMsg);
if (pRpc->msgType & 1U) { if (pRpc->msgType & 1U) {
if (pRpc->handle == NULL) return; if (pRpc->handle != NULL && code != 0) {
if (code != 0) { dError("msg:%p, failed to process since %s", pMsg, terrstr());
SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle}; SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle};
dndSendRsp(pMgmt->pWrapper, &rsp); dndSendRsp(pMgmt->pWrapper, &rsp);
} }
@ -67,83 +69,86 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
static void mmPutMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) {
static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SNodeMsg *pMsg) {
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
} }
int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t mmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); SMnodeMgmt *pMgmt = pWrapper->pMgmt;
mmPutMsgToWorker(&pMgmt->writeWorker, pMsg);
return 0;
} }
int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); SMnodeMgmt *pMgmt = pWrapper->pMgmt;
mmPutMsgToWorker(&pMgmt->syncWorker, pMsg);
return 0;
} }
int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); SMnodeMgmt *pMgmt = pWrapper->pMgmt;
mmPutMsgToWorker(&pMgmt->readWorker, pMsg);
return 0;
} }
int32_t mmProcessQueryMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); SMnodeMgmt *pMgmt = pWrapper->pMgmt;
mmPutMsgToWorker(&pMgmt->queryWorker, pMsg);
return 0;
} }
static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { static int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) {
SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
if (pMsg == NULL) { if (pMsg == NULL) return -1;
return -1;
}
dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
pMsg->rpcMsg = *pRpc; pMsg->rpcMsg = *pRpc;
taosWriteQitem(pWorker->queue, pMsg);
int32_t code = taosWriteQitem(pWorker->queue, pMsg); return 0;
if (code != 0) {
dTrace("msg:%p, is freed", pMsg);
taosFreeQitem(pMsg);
rpcFreeCont(pRpc->pCont);
}
return code;
}
int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc);
}
int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc);
} }
int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SMnodeMgmt *pMgmt = pWrapper->pMgmt; SMnodeMgmt *pMgmt = pWrapper->pMgmt;
return mmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc); return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pRpc);
} }
int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pRpc);
}
int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
return mmPutRpcMsgToWorker(&pMgmt->readWorker, pRpc);
}
int32_t mmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pRpc);
}
int32_t mmStartWorker(SMnodeMgmt *pMgmt) { int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
SSingleWorkerCfg cfg = {.minNum = 0, .maxNum = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt}; SSingleWorkerCfg qCfg = {.min = 0, .max = 1, .name = "mnode-query", .fp = (FItem)mmProcessQueryQueue, .param = pMgmt};
SSingleWorkerCfg queryCfg = {.minNum = 0, .maxNum = 1, .name = "mnode-query", .fp = (FItem)mmProcessQueryQueue, .param = pMgmt}; if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) {
if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) {
dError("failed to start mnode-query worker since %s", terrstr()); dError("failed to start mnode-query worker since %s", terrstr());
return -1; return -1;
} }
if (tSingleWorkerInit(&pMgmt->readWorker, &cfg) != 0) { SSingleWorkerCfg rCfg = {.min = 0, .max = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) {
dError("failed to start mnode-read worker since %s", terrstr()); dError("failed to start mnode-read worker since %s", terrstr());
return -1; return -1;
} }
if (tSingleWorkerInit(&pMgmt->writeWorker, &cfg) != 0) { SSingleWorkerCfg wCfg = {.min = 0, .max = 1, .name = "mnode-write", .fp = (FItem)mmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) {
dError("failed to start mnode-write worker since %s", terrstr()); dError("failed to start mnode-write worker since %s", terrstr());
return -1; return -1;
} }
if (tSingleWorkerInit(&pMgmt->syncWorker, &cfg) != 0) { SSingleWorkerCfg sCfg = {.min = 0, .max = 1, .name = "mnode-sync", .fp = (FItem)mmProcessQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) {
dError("failed to start mnode sync-worker since %s", terrstr()); dError("failed to start mnode sync-worker since %s", terrstr());
return -1; return -1;
} }
@ -153,8 +158,8 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
} }
void mmStopWorker(SMnodeMgmt *pMgmt) { void mmStopWorker(SMnodeMgmt *pMgmt) {
tSingleWorkerCleanup(&pMgmt->readWorker);
tSingleWorkerCleanup(&pMgmt->queryWorker); tSingleWorkerCleanup(&pMgmt->queryWorker);
tSingleWorkerCleanup(&pMgmt->readWorker);
tSingleWorkerCleanup(&pMgmt->writeWorker); tSingleWorkerCleanup(&pMgmt->writeWorker);
tSingleWorkerCleanup(&pMgmt->syncWorker); tSingleWorkerCleanup(&pMgmt->syncWorker);
dDebug("mnode workers are closed"); dDebug("mnode workers are closed");

4
source/dnode/mgmt/qnode/inc/qmInt.h
View File

@ -48,8 +48,8 @@ int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype);
int32_t qmStartWorker(SQnodeMgmt *pMgmt); int32_t qmStartWorker(SQnodeMgmt *pMgmt);
void qmStopWorker(SQnodeMgmt *pMgmt); void qmStopWorker(SQnodeMgmt *pMgmt);
int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t qmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

18
source/dnode/mgmt/qnode/src/qmMsg.c
View File

@ -56,14 +56,14 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
void qmInitMsgHandles(SMgmtWrapper *pWrapper) { void qmInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by VNODE // Requests handled by VNODE
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)qmProcessQueryMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)qmProcessQueryMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)qmProcessFetchMsg, QND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QND_VGID);
} }

35
source/dnode/mgmt/qnode/src/qmWorker.c
View File

@ -49,14 +49,22 @@ static void qmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
static int32_t qmPutMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) { static void qmPutMsgToWorker(SSingleWorker *pWorker, SNodeMsg *pMsg) {
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
} }
int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->queryWorker, pMsg); } int32_t qmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SQnodeMgmt *pMgmt = pWrapper->pMgmt;
qmPutMsgToWorker(&pMgmt->queryWorker, pMsg);
return 0;
}
int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->fetchWorker, pMsg); } int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SQnodeMgmt *pMgmt = pWrapper->pMgmt;
qmPutMsgToWorker(&pMgmt->fetchWorker, pMsg);
return 0;
}
static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) {
SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
@ -66,15 +74,8 @@ static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SR
dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
pMsg->rpcMsg = *pRpc; pMsg->rpcMsg = *pRpc;
taosWriteQitem(pWorker->queue, pMsg);
int32_t code = taosWriteQitem(pWorker->queue, pMsg); return 0;
if (code != 0) {
dTrace("msg:%p, is freed", pMsg);
taosFreeQitem(pMsg);
rpcFreeCont(pRpc->pCont);
}
return code;
} }
int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
@ -110,8 +111,8 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
int32_t maxQueryThreads = minQueryThreads; int32_t maxQueryThreads = minQueryThreads;
SSingleWorkerCfg queryCfg = {.minNum = minQueryThreads, SSingleWorkerCfg queryCfg = {.min = minQueryThreads,
.maxNum = maxQueryThreads, .max = maxQueryThreads,
.name = "qnode-query", .name = "qnode-query",
.fp = (FItem)qmProcessQueryQueue, .fp = (FItem)qmProcessQueryQueue,
.param = pMgmt}; .param = pMgmt};
@ -121,8 +122,8 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
return -1; return -1;
} }
SSingleWorkerCfg fetchCfg = {.minNum = minFetchThreads, SSingleWorkerCfg fetchCfg = {.min = minFetchThreads,
.maxNum = maxFetchThreads, .max = maxFetchThreads,
.name = "qnode-fetch", .name = "qnode-fetch",
.fp = (FItem)qmProcessFetchQueue, .fp = (FItem)qmProcessFetchQueue,
.param = pMgmt}; .param = pMgmt};

8
source/dnode/mgmt/snode/inc/smInt.h
View File

@ -46,10 +46,10 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
// smWorker.c // smWorker.c
int32_t smStartWorker(SSnodeMgmt *pMgmt); int32_t smStartWorker(SSnodeMgmt *pMgmt);
void smStopWorker(SSnodeMgmt *pMgmt); void smStopWorker(SSnodeMgmt *pMgmt);
int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t smProcessExecMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

4
source/dnode/mgmt/snode/src/smMsg.c
View File

@ -56,6 +56,6 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
void smInitMsgHandles(SMgmtWrapper *pWrapper) { void smInitMsgHandles(SMgmtWrapper *pWrapper) {
// Requests handled by SNODE // Requests handled by SNODE
dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, VND_VGID);
} }

32
source/dnode/mgmt/snode/src/smWorker.c
View File

@ -57,7 +57,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
return -1; return -1;
} }
SMultiWorkerCfg cfg = {.maxNum = 1, .name = "snode-unique", .fp = smProcessUniqueQueue, .param = pMgmt}; SMultiWorkerCfg cfg = {.max = 1, .name = "snode-unique", .fp = smProcessUniqueQueue, .param = pMgmt};
if (tMultiWorkerInit(pUniqueWorker, &cfg) != 0) { if (tMultiWorkerInit(pUniqueWorker, &cfg) != 0) {
dError("failed to start snode-unique worker since %s", terrstr()); dError("failed to start snode-unique worker since %s", terrstr());
@ -69,8 +69,8 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
} }
} }
SSingleWorkerCfg cfg = {.minNum = SND_SHARED_THREAD_NUM, SSingleWorkerCfg cfg = {.min = SND_SHARED_THREAD_NUM,
.maxNum = SND_SHARED_THREAD_NUM, .max = SND_SHARED_THREAD_NUM,
.name = "snode-shared", .name = "snode-shared",
.fp = (FItem)smProcessSharedQueue, .fp = (FItem)smProcessSharedQueue,
.param = pMgmt}; .param = pMgmt};
@ -107,7 +107,8 @@ static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {
return 0; return 0;
} }
int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
@ -115,10 +116,12 @@ int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg);
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index); SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index);
if (pWorker == NULL) { if (pWorker == NULL) {
@ -127,21 +130,24 @@ int32_t smProcessUniqueMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) {
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t smProcessSharedMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->sharedWorker; SSingleWorker *pWorker = &pMgmt->sharedWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t smProcessExecMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { int32_t smProcessExecMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg); int32_t workerType = smGetSWTypeFromMsg(&pMsg->rpcMsg);
if (workerType == SND_WORKER_TYPE__SHARED) { if (workerType == SND_WORKER_TYPE__SHARED) {
return smProcessSharedMsg(pMgmt, pMsg); return smProcessSharedMsg(pWrapper, pMsg);
} else { } else {
return smProcessUniqueMsg(pMgmt, pMsg); return smProcessUniqueMsg(pWrapper, pMsg);
} }
} }

12
source/dnode/mgmt/vnode/inc/vmInt.h
View File

@ -108,12 +108,12 @@ int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype); int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype);
int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrappert, SNodeMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

11
source/dnode/mgmt/vnode/src/vmMsg.c
View File

@ -281,12 +281,11 @@ void vmInitMsgHandles(SMgmtWrapper *pWrapper) {
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, VND_VGID);
dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, VND_VGID);
} }

65
source/dnode/mgmt/vnode/src/vmWorker.c
View File

@ -179,9 +179,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
} }
static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) {
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
int32_t code = -1;
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
pHead->contLen = ntohl(pHead->contLen); pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId); pHead->vgId = ntohl(pHead->vgId);
@ -192,28 +190,30 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
return -1; return -1;
} }
int32_t code = 0;
switch (qtype) { switch (qtype) {
case QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, will be written into vnode-query queue", pMsg); dTrace("msg:%p, will be written into vnode-query queue", pMsg);
code = taosWriteQitem(pVnode->pQueryQ, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, will be written into vnode-fetch queue", pMsg); dTrace("msg:%p, will be written into vnode-fetch queue", pMsg);
code = taosWriteQitem(pVnode->pFetchQ, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case WRITE_QUEUE: case WRITE_QUEUE:
dTrace("msg:%p, will be written into vnode-write queue", pMsg); dTrace("msg:%p, will be written into vnode-write queue", pMsg);
code = taosWriteQitem(pVnode->pWriteQ, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg);
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:
dTrace("msg:%p, will be written into vnode-sync queue", pMsg); dTrace("msg:%p, will be written into vnode-sync queue", pMsg);
code = taosWriteQitem(pVnode->pSyncQ, pMsg); taosWriteQitem(pVnode->pSyncQ, pMsg);
break; break;
case MERGE_QUEUE: case MERGE_QUEUE:
dTrace("msg:%p, will be written into vnode-merge queue", pMsg); dTrace("msg:%p, will be written into vnode-merge queue", pMsg);
code = taosWriteQitem(pVnode->pMergeQ, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg);
break; break;
default: default:
code = -1;
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
break; break;
} }
@ -222,52 +222,73 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
return code; return code;
} }
int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } int32_t vmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
return vmPutNodeMsgToQueue(pMgmt, pMsg, SYNC_QUEUE);
}
int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, WRITE_QUEUE); } int32_t vmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
return vmPutNodeMsgToQueue(pMgmt, pMsg, WRITE_QUEUE);
}
int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, QUERY_QUEUE); } int32_t vmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
return vmPutNodeMsgToQueue(pMgmt, pMsg, QUERY_QUEUE);
}
int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } int32_t vmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE);
}
int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, MERGE_QUEUE); } int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
return vmPutNodeMsgToQueue(pMgmt, pMsg, MERGE_QUEUE);
}
int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
SSingleWorker *pWorker = &pMgmt->mgmtWorker; SSingleWorker *pWorker = &pMgmt->mgmtWorker;
dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name);
return taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) {
SVnodesMgmt *pMgmt = pWrapper->pMgmt; SVnodesMgmt *pMgmt = pWrapper->pMgmt;
int32_t code = -1;
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) return -1; if (pVnode == NULL) return -1;
SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
if (pMsg != NULL) { int32_t code = 0;
if (pMsg == NULL) {
code = -1;
} else {
dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
pMsg->rpcMsg = *pRpc; pMsg->rpcMsg = *pRpc;
switch (qtype) { switch (qtype) {
case QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, will be put into vnode-query queue", pMsg); dTrace("msg:%p, will be put into vnode-query queue", pMsg);
code = taosWriteQitem(pVnode->pQueryQ, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); dTrace("msg:%p, will be put into vnode-fetch queue", pMsg);
code = taosWriteQitem(pVnode->pFetchQ, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case APPLY_QUEUE: case APPLY_QUEUE:
dTrace("msg:%p, will be put into vnode-apply queue", pMsg); dTrace("msg:%p, will be put into vnode-apply queue", pMsg);
code = taosWriteQitem(pVnode->pApplyQ, pMsg); taosWriteQitem(pVnode->pApplyQ, pMsg);
break; break;
case MERGE_QUEUE: case MERGE_QUEUE:
dTrace("msg:%p, will be put into vnode-merge queue", pMsg); dTrace("msg:%p, will be put into vnode-merge queue", pMsg);
code = taosWriteQitem(pVnode->pMergeQ, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg);
break; break;
default: default:
code = -1;
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
break; break;
} }
@ -394,7 +415,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
if (tWWorkerInit(pWPool) != 0) return -1; if (tWWorkerInit(pWPool) != 0) return -1;
SSingleWorkerCfg cfg = { SSingleWorkerCfg cfg = {
.minNum = 1, .maxNum = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; .min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) {
dError("failed to start vnode-mgmt worker since %s", terrstr()); dError("failed to start vnode-mgmt worker since %s", terrstr());
return -1; return -1;

26
source/dnode/mnode/impl/src/mndDb.c
View File

@ -276,6 +276,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->quorum > pCfg->replications) return -1; if (pCfg->quorum > pCfg->replications) return -1;
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1; if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1;
if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
if (pCfg->streamMode < TSDB_MIN_DB_STREAM_MODE || pCfg->streamMode > TSDB_MAX_DB_STREAM_MODE) return -1;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -285,8 +286,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP;
if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0;
if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep1;
if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MIN_ROW_FBLOCK; if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MIN_ROW_FBLOCK;
if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK; if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK;
if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME;
@ -298,6 +299,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->quorum < 0) pCfg->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION; if (pCfg->quorum < 0) pCfg->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE_OPTION; if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE_OPTION;
if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
if (pCfg->streamMode < 0) pCfg->streamMode = TSDB_DEFAULT_DB_STREAM_MODE;
if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
} }
static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
@ -431,11 +434,11 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
.daysToKeep2 = pCreate->daysToKeep2, .daysToKeep2 = pCreate->daysToKeep2,
.minRows = pCreate->minRows, .minRows = pCreate->minRows,
.maxRows = pCreate->maxRows, .maxRows = pCreate->maxRows,
.fsyncPeriod = pCreate->fsyncPeriod,
.commitTime = pCreate->commitTime, .commitTime = pCreate->commitTime,
.fsyncPeriod = pCreate->fsyncPeriod,
.walLevel = pCreate->walLevel,
.precision = pCreate->precision, .precision = pCreate->precision,
.compression = pCreate->compression, .compression = pCreate->compression,
.walLevel = pCreate->walLevel,
.replications = pCreate->replications, .replications = pCreate->replications,
.quorum = pCreate->quorum, .quorum = pCreate->quorum,
.update = pCreate->update, .update = pCreate->update,
@ -445,7 +448,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
dbObj.cfg.pRetensions = pCreate->pRetensions; dbObj.cfg.pRetensions = pCreate->pRetensions;
pCreate = NULL; pCreate->pRetensions = NULL;
mndSetDefaultDbCfg(&dbObj.cfg); mndSetDefaultDbCfg(&dbObj.cfg);
@ -1521,6 +1524,14 @@ static void setInformationSchemaDbCfg(SDbObj *pDbObj) {
pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI;
} }
static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
SVgObj *pVgroup = pObj;
int32_t *numOfTables = p1;
*numOfTables += pVgroup->numOfTables;
return true;
}
static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rowsCapacity) { static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rowsCapacity) {
SMnode *pMnode = pReq->pNode; SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -1533,7 +1544,10 @@ static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
break; break;
} }
dumpDbInfoToPayload(data, pDb, pShow, numOfRows, rowsCapacity, 0); int32_t numOfTables = 0;
sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
dumpDbInfoToPayload(data, pDb, pShow, numOfRows, rowsCapacity, numOfTables);
numOfRows++; numOfRows++;
sdbRelease(pSdb, pDb); sdbRelease(pSdb, pDb);
} }

6
source/dnode/mnode/impl/src/mndUser.c
View File

@ -87,12 +87,6 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) {
return -1; return -1;
} }
#if 0
if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) {
return -1;
}
#endif
return 0; return 0;
} }

11
source/dnode/vnode/src/inc/tsdbReadImpl.h
View File

@ -67,12 +67,13 @@ typedef struct {
uint8_t last : 1; uint8_t last : 1;
uint8_t blkVer : 7; uint8_t blkVer : 7;
uint8_t numOfSubBlocks; uint8_t numOfSubBlocks;
int16_t numOfCols; // not including timestamp column col_id_t numOfCols; // not including timestamp column
uint32_t len; // data block length uint32_t len; // data block length
uint32_t keyLen : 24; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols uint32_t keyLen : 20; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
uint32_t algorithm : 4;
uint32_t reserve : 8; uint32_t reserve : 8;
int32_t algorithm : 8; col_id_t numOfBSma;
int32_t numOfRows : 24; uint16_t numOfRows;
int64_t offset; int64_t offset;
uint64_t aggrStat : 1; uint64_t aggrStat : 1;
uint64_t aggrOffset : 63; uint64_t aggrOffset : 63;
@ -80,7 +81,7 @@ typedef struct {
TSKEY keyLast; TSKEY keyLast;
} SBlockV0; } SBlockV0;
#define SBlock SBlockV0 // latest SBlock definition #define SBlock SBlockV0 // latest SBlock definition
#endif #endif

50
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -43,20 +43,20 @@ typedef struct {
#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) #define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) #define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh))
#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) #define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh)))
#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) #define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet))
#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) #define TSDB_COMMIT_TABLE(ch) ((ch)->pTable)
#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) #define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD)
#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) #define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA)
#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) #define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST)
#define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD) #define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD)
#define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL) #define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL)
#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) #define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh))
#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) #define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh))
#define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh)) #define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh))
#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock)
#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
static void tsdbStartCommit(STsdb *pRepo); static void tsdbStartCommit(STsdb *pRepo);
static void tsdbEndCommit(STsdb *pTsdb, int eno); static void tsdbEndCommit(STsdb *pTsdb, int eno);
@ -1204,9 +1204,10 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) {
STsdbCfg * pCfg = REPO_CFG(pRepo); STsdbCfg *pCfg = REPO_CFG(pRepo);
SBlockData * pBlockData = NULL; SBlockData *pBlockData = NULL;
SAggrBlkData *pAggrBlkData = NULL; SAggrBlkData *pAggrBlkData = NULL;
STSchema *pSchema = pTable->pSchema;
int64_t offset = 0, offsetAggr = 0; int64_t offset = 0, offsetAggr = 0;
int rowsToWrite = pDataCols->numOfRows; int rowsToWrite = pDataCols->numOfRows;
@ -1225,10 +1226,12 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
pAggrBlkData = (SAggrBlkData *)(*ppExBuf); pAggrBlkData = (SAggrBlkData *)(*ppExBuf);
// Get # of cols not all NULL(not including key column) // Get # of cols not all NULL(not including key column)
int nColsNotAllNull = 0; col_id_t nColsNotAllNull = 0;
col_id_t nColsOfBlockSma = 0;
for (int ncol = 1; ncol < pDataCols->numOfCols; ++ncol) { // ncol from 1, we skip the timestamp column for (int ncol = 1; ncol < pDataCols->numOfCols; ++ncol) { // ncol from 1, we skip the timestamp column
SDataCol * pDataCol = pDataCols->cols + ncol; STColumn *pColumn = pSchema->columns + ncol;
SBlockCol * pBlockCol = pBlockData->cols + nColsNotAllNull; SDataCol *pDataCol = pDataCols->cols + ncol;
SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull;
SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull; SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull;
if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it
@ -1260,7 +1263,12 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
} else { } else {
TD_SET_COL_ROWS_MISC(pBlockCol); TD_SET_COL_ROWS_MISC(pBlockCol);
} }
nColsNotAllNull++;
++nColsNotAllNull;
if (pColumn->sma) {
++nColsOfBlockSma;
}
} }
ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols);
@ -1357,9 +1365,8 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
return -1; return -1;
} }
uint32_t aggrStatus = nColsNotAllNull > 0 ? 1 : 0; uint32_t aggrStatus = nColsOfBlockSma > 0 ? 1 : 0;
if (aggrStatus > 0) { if (aggrStatus > 0) {
taosCalcChecksumAppend(0, (uint8_t *)pAggrBlkData, tsizeAggr); taosCalcChecksumAppend(0, (uint8_t *)pAggrBlkData, tsizeAggr);
tsdbUpdateDFileMagic(pDFileAggr, POINTER_SHIFT(pAggrBlkData, tsizeAggr - sizeof(TSCKSUM))); tsdbUpdateDFileMagic(pDFileAggr, POINTER_SHIFT(pAggrBlkData, tsizeAggr - sizeof(TSCKSUM)));
@ -1378,6 +1385,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
pBlock->keyLen = keyLen; pBlock->keyLen = keyLen;
pBlock->numOfSubBlocks = isSuper ? 1 : 0; pBlock->numOfSubBlocks = isSuper ? 1 : 0;
pBlock->numOfCols = nColsNotAllNull; pBlock->numOfCols = nColsNotAllNull;
pBlock->numOfBSma = nColsOfBlockSma;
pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyFirst = dataColsKeyFirst(pDataCols);
pBlock->keyLast = dataColsKeyLast(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols);
pBlock->aggrStat = aggrStatus; pBlock->aggrStat = aggrStatus;

2
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
View File

@ -321,7 +321,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
return -1; return -1;
} }
size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfBSma, (uint32_t)pBlock->blkVer);
if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1; if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1;
int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr); int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr);

6
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -189,12 +189,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
} }
} break; } break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA case TDMT_VND_CREATE_SMA: { // timeRangeSMA
#if 0 #if 1
SSmaCfg vCreateSmaReq = {0}; SSmaCfg vCreateSmaReq = {0};
if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) { if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
vWarn("vgId%d: TDMT_VND_CREATE_SMA received but deserialize failed since %s", pVnode->config.vgId, terrstr(terrno));
return -1; return -1;
} }
vWarn("vgId%d: TDMT_VND_CREATE_SMA received for %s:%" PRIi64, pVnode->config.vgId, vCreateSmaReq.tSma.indexName,
vCreateSmaReq.tSma.indexUid);
// record current timezone of server side // record current timezone of server side
tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezoneStr, TD_TIMEZONE_LEN);

9
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -330,7 +330,6 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
// step 2: insert data // step 2: insert data
STSmaDataWrapper *pSmaData = NULL;
STsdb *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); STsdb *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb));
STsdbCfg *pCfg = &pTsdb->config; STsdbCfg *pCfg = &pTsdb->config;
@ -416,6 +415,8 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
col_id_t numOfCols = 4096; col_id_t numOfCols = 4096;
ASSERT_GT(numOfCols, 0); ASSERT_GT(numOfCols, 0);
#if 0
STSmaDataWrapper *pSmaData = NULL;
pSmaData = (STSmaDataWrapper *)buf; pSmaData = (STSmaDataWrapper *)buf;
printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData); printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData);
pSmaData->skey = skey1; pSmaData->skey = skey1;
@ -459,9 +460,13 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
pSmaData->dataLen = (len - sizeof(STSmaDataWrapper)); pSmaData->dataLen = (len - sizeof(STSmaDataWrapper));
ASSERT_GE(bufSize, pSmaData->dataLen); ASSERT_GE(bufSize, pSmaData->dataLen);
// execute // execute
ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
#endif
SSDataBlock *pSmaData = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
// step 3: query // step 3: query
uint32_t checkDataCnt = 0; uint32_t checkDataCnt = 0;

2
source/libs/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_subdirectory(transport) add_subdirectory(transport)
add_subdirectory(sync) add_subdirectory(sync)
# add_subdirectory(tdb) add_subdirectory(tdb)
add_subdirectory(index) add_subdirectory(index)
add_subdirectory(wal) add_subdirectory(wal)
add_subdirectory(parser) add_subdirectory(parser)

2
source/libs/executor/inc/executorimpl.h
View File

@ -469,7 +469,7 @@ typedef struct SOptrBasicInfo {
int32_t* rowCellInfoOffset; // offset value for each row result cell info int32_t* rowCellInfoOffset; // offset value for each row result cell info
SqlFunctionCtx* pCtx; SqlFunctionCtx* pCtx;
SSDataBlock* pRes; SSDataBlock* pRes;
int32_t capacity; int32_t capacity; // TODO remove it
} SOptrBasicInfo; } SOptrBasicInfo;
//TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset //TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset

210
source/libs/executor/src/executorimpl.c
View File

@ -1246,17 +1246,40 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunction
} }
} }
static void projectApplyFunctions(SSDataBlock* pResult, SqlFunctionCtx *pCtx, int32_t numOfOutput) { static void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx *pCtx, int32_t numOfOutput) {
for (int32_t k = 0; k < numOfOutput; ++k) { for (int32_t k = 0; k < numOfOutput; ++k) {
if (pCtx[k].fpSet.init == NULL) { // it is a project query if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, k); SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, k);
colDataAssign(pColInfoData, pCtx[k].input.pData[0], pCtx[k].input.numOfRows); colDataAssign(pColInfoData, pCtx[k].input.pData[0], pCtx[k].input.numOfRows);
} else { // TODO: arithmetic and other process.
pResult->info.rows = pCtx[0].input.numOfRows;
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_OPERATOR) {
SArray* pBlockList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pBlockList, &pSrcBlock);
SScalarParam dest = {0};
dest.columnData = taosArrayGet(pResult->pDataBlock, k);
scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
pResult->info.rows = dest.numOfRows;
taosArrayDestroy(pBlockList);
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) {
ASSERT(!fmIsAggFunc(pCtx->functionId));
SScalarParam p = {.numOfRows = pSrcBlock->info.rows};
int32_t slotId = pExpr[k].base.pParam[0].pCol->slotId;
p.columnData = taosArrayGet(pSrcBlock->pDataBlock, slotId);
SScalarParam dest = {0};
dest.columnData = taosArrayGet(pResult->pDataBlock, k);
pCtx[k].sfp.process(&p, 1, &dest);
pResult->info.rows = dest.numOfRows;
} else {
ASSERT(0); ASSERT(0);
} }
} }
pResult->info.rows = pCtx[0].input.numOfRows;
} }
void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs,
@ -2013,107 +2036,6 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
int32_t** rowCellInfoOffset) {
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));
if (pFuncCtx == NULL) {
return NULL;
}
*rowCellInfoOffset = taosMemoryCalloc(numOfOutput, sizeof(int32_t));
if (*rowCellInfoOffset == 0) {
taosMemoryFreeClear(pFuncCtx);
return NULL;
}
for (int32_t i = 0; i < numOfOutput; ++i) {
SExprBasicInfo *pFunct = &pExpr[i].base;
SqlFunctionCtx* pCtx = &pFuncCtx[i];
#if 0
SColIndex *pIndex = &pFunct->colInfo;
if (TSDB_COL_REQ_NULL(pIndex->flag)) {
pCtx->requireNull = true;
pIndex->flag &= ~(TSDB_COL_NULL);
} else {
pCtx->requireNull = false;
}
#endif
// pCtx->inputBytes = pFunct->colBytes;
// pCtx->inputType = pFunct->colType;
pCtx->ptsOutputBuf = NULL;
pCtx->resDataInfo.bytes = pFunct->resSchema.bytes;
pCtx->resDataInfo.type = pFunct->resSchema.type;
pCtx->order = pQueryAttr->order.order;
// pCtx->functionId = pFunct->functionId;
pCtx->stableQuery = pQueryAttr->stableQuery;
// pCtx->resDataInfo.interBufSize = pFunct->interBytes;
pCtx->start.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
pCtx->numOfParams = pFunct->numOfParams;
for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
int16_t type = pFunct->pParam[j].param.nType;
int16_t bytes = pFunct->pParam[j].param.nType;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
// taosVariantCreateFromBinary(&pCtx->param[j], pFunct->param[j].pz, bytes, type);
} else {
// taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pFunct->param[j].i, bytes, type);
}
}
// set the order information for top/bottom query
int32_t functionId = pCtx->functionId;
if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
int32_t f = getExprFunctionId(&pExpr[0]);
assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);
pCtx->param[2].i = pQueryAttr->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[3].i = functionId;
pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[1].i = pQueryAttr->order.col.colId;
} else if (functionId == FUNCTION_INTERP) {
pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
if (pQueryAttr->fillVal != NULL) {
if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
} else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value
if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
}
}
}
} else if (functionId == FUNCTION_TS_COMP) {
pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client
pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT;
} else if (functionId == FUNCTION_TWA) {
pCtx->param[1].i = pQueryAttr->window.skey;
pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[2].i = pQueryAttr->window.ekey;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
} else if (functionId == FUNCTION_ARITHM) {
// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
}
}
// for(int32_t i = 1; i < numOfOutput; ++i) {
// (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes);
// }
setCtxTagColumnInfo(pFuncCtx, numOfOutput);
return pFuncCtx;
}
static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset) { static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset) {
SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));
if (pFuncCtx == NULL) { if (pFuncCtx == NULL) {
@ -2132,15 +2054,22 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num
SExprBasicInfo *pFunct = &pExpr->base; SExprBasicInfo *pFunct = &pExpr->base;
SqlFunctionCtx* pCtx = &pFuncCtx[i]; SqlFunctionCtx* pCtx = &pFuncCtx[i];
if (pExpr->pExpr->_function.pFunctNode != NULL) { pCtx->functionId = -1;
if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) {
SFuncExecEnv env = {0}; SFuncExecEnv env = {0};
pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId;
fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); if (fmIsAggFunc(pCtx->functionId)) {
pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);
} else {
fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp);
}
pCtx->resDataInfo.interBufSize = env.calcMemSize; pCtx->resDataInfo.interBufSize = env.calcMemSize;
} else { } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN) {
pCtx->functionId = -1;
} else if (pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR) {
} }
pCtx->input.numOfInputCols = pFunct->numOfParams; pCtx->input.numOfInputCols = pFunct->numOfParams;
@ -5578,11 +5507,11 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
SOperatorNode *node = (SOperatorNode *)pNode; SOperatorNode *node = (SOperatorNode *)pNode;
if (OP_TYPE_EQUAL == node->opType) { if (OP_TYPE_EQUAL == node->opType) {
*(int32_t *)pContext = 1; *(int32_t *)pContext = 1;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
*(int32_t *)pContext = 0; *(int32_t *)pContext = 0;
return DEAL_RES_IGNORE_CHILD; return DEAL_RES_IGNORE_CHILD;
} }
@ -5590,14 +5519,14 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
if (1 != *(int32_t *)pContext) { if (1 != *(int32_t *)pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
SColumnNode *node = (SColumnNode *)pNode; SColumnNode *node = (SColumnNode *)pNode;
if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) { if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) {
*(int32_t *)pContext = 2; *(int32_t *)pContext = 2;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
*(int32_t *)pContext = 0; *(int32_t *)pContext = 0;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
case QUERY_NODE_VALUE: { case QUERY_NODE_VALUE: {
@ -5607,7 +5536,7 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
SValueNode *node = (SValueNode *)pNode; SValueNode *node = (SValueNode *)pNode;
char *dbName = nodesGetValueFromNode(node); char *dbName = nodesGetValueFromNode(node);
strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); strncpy(pContext, varDataVal(dbName), varDataLen(dbName));
*((char *)pContext + varDataLen(dbName)) = 0; *((char *)pContext + varDataLen(dbName)) = 0;
return DEAL_RES_ERROR; // stop walk return DEAL_RES_ERROR; // stop walk
} }
@ -6654,7 +6583,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup)
blockDataCleanup(pRes); blockDataCleanup(pRes);
if (pProjectInfo->existDataBlock) { // TODO refactor if (pProjectInfo->existDataBlock) { // TODO refactor
// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
SSDataBlock* pBlock = pProjectInfo->existDataBlock; SSDataBlock* pBlock = pProjectInfo->existDataBlock;
pProjectInfo->existDataBlock = NULL; pProjectInfo->existDataBlock = NULL;
*newgroup = true; *newgroup = true;
@ -6668,9 +6596,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup)
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC);
blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
projectApplyFunctions(pInfo->pRes, pInfo->pCtx, pOperator->numOfOutput); projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput);
pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput, NULL);
if (pRes->info.rows >= pProjectInfo->binfo.capacity*0.8) { if (pRes->info.rows >= pProjectInfo->binfo.capacity*0.8) {
copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput);
resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput);
@ -6713,15 +6639,15 @@ static SSDataBlock* doProjectOperation(SOperatorInfo *pOperator, bool* newgroup)
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC); setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC);
updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
projectApplyFunctions(pInfo->pRes, pInfo->pCtx, pOperator->numOfOutput); projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput);
if (pRes->info.rows >= pProjectInfo->threshold) { if (pRes->info.rows >= pOperator->resultInfo.threshold) {
break; break;
} }
} }
copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); // copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput);
return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL;
} }
@ -7811,7 +7737,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo));
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
@ -7836,7 +7762,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper
SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
pInfo->colIndex = -1; pInfo->colIndex = -1;
pInfo->reptScan = false; pInfo->reptScan = false;
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
@ -7901,7 +7827,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo));
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
@ -7925,7 +7851,7 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo));
pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
// pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity);
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8);
@ -8533,16 +8459,18 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
// it is a project query, or group by column // it is a project query, or group by column
if (nodeType(pTargetNode->pExpr) == QUERY_NODE_COLUMN) { if (nodeType(pTargetNode->pExpr) == QUERY_NODE_COLUMN) {
pExp->pExpr->nodeType = QUERY_NODE_COLUMN;
SColumnNode* pColNode = (SColumnNode*) pTargetNode->pExpr; SColumnNode* pColNode = (SColumnNode*) pTargetNode->pExpr;
SDataType* pType = &pColNode->node.resType; SDataType* pType = &pColNode->node.resType;
pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pColNode->colName); pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pColNode->colName);
pCol->slotId = pColNode->slotId; pCol->slotId = pColNode->slotId; // TODO refactor
pCol->bytes = pType->bytes; pCol->bytes = pType->bytes;
pCol->type = pType->type; pCol->type = pType->type;
pCol->scale = pType->scale; pCol->scale = pType->scale;
pCol->precision = pType->precision; pCol->precision = pType->precision;
} else { } else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_FUNCTION) {
pExp->pExpr->nodeType = QUERY_NODE_FUNCTION;
SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr;
SDataType* pType = &pFuncNode->node.resType; SDataType* pType = &pFuncNode->node.resType;
@ -8556,7 +8484,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList);
for (int32_t j = 0; j < numOfParam; ++j) { for (int32_t j = 0; j < numOfParam; ++j) {
SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j);
SColumnNode* pcn = (SColumnNode*)p1; SColumnNode* pcn = (SColumnNode*)p1; // TODO refactor
pCol->slotId = pcn->slotId; pCol->slotId = pcn->slotId;
pCol->bytes = pcn->node.resType.bytes; pCol->bytes = pcn->node.resType.bytes;
@ -8565,6 +8493,22 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
pCol->precision = pcn->node.resType.precision; pCol->precision = pcn->node.resType.precision;
pCol->dataBlockId = pcn->dataBlockId; pCol->dataBlockId = pcn->dataBlockId;
} }
} else if (nodeType(pTargetNode->pExpr) == QUERY_NODE_OPERATOR) {
pExp->pExpr->nodeType = QUERY_NODE_OPERATOR;
SOperatorNode* pNode = (SOperatorNode*) pTargetNode->pExpr;
SDataType* pType = &pNode->node.resType;
pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pNode->node.aliasName);
pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr;
pCol->slotId = pTargetNode->slotId; // TODO refactor
pCol->bytes = pType->bytes;
pCol->type = pType->type;
pCol->scale = pType->scale;
pCol->precision = pType->precision;
} else {
ASSERT(0);
} }
} }
@ -8627,7 +8571,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa
SArray* colList = extractScanColumnId(pScanNode->pScanCols); SArray* colList = extractScanColumnId(pScanNode->pScanCols);
SOperatorInfo* pOperator = createSysTableScanOperatorInfo(pHandle->meta, pResBlock, &pScanNode->tableName, SOperatorInfo* pOperator = createSysTableScanOperatorInfo(pHandle->meta, pResBlock, &pScanNode->tableName,
pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet,
colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId);
return pOperator; return pOperator;
} else { } else {

2
source/libs/function/inc/builtins.h
View File

@ -33,6 +33,8 @@ extern "C" {
#define FUNC_MGT_DATETIME_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(4) #define FUNC_MGT_DATETIME_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
#define FUNC_MGT_TIMELINE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(5) #define FUNC_MGT_TIMELINE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
#define FUNC_MGT_TIMEORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(6) #define FUNC_MGT_TIMEORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
#define FUNC_MGT_PSEUDO_COLUMN_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
#define FUNC_MGT_WINDOW_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)

10
source/libs/function/inc/builtinsimpl.h
View File

@ -25,23 +25,23 @@ extern "C" {
bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
void functionFinalize(SqlFunctionCtx *pCtx); void functionFinalize(SqlFunctionCtx *pCtx);
bool getCountFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
void countFunction(SqlFunctionCtx *pCtx); void countFunction(SqlFunctionCtx *pCtx);
bool getSumFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
void sumFunction(SqlFunctionCtx *pCtx); void sumFunction(SqlFunctionCtx *pCtx);
bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
bool getMinmaxFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
void minFunction(SqlFunctionCtx* pCtx); void minFunction(SqlFunctionCtx* pCtx);
void maxFunction(SqlFunctionCtx *pCtx); void maxFunction(SqlFunctionCtx *pCtx);
bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
void stddevFunction(SqlFunctionCtx* pCtx); void stddevFunction(SqlFunctionCtx* pCtx);
void stddevFinalize(SqlFunctionCtx* pCtx); void stddevFinalize(SqlFunctionCtx* pCtx);
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
void firstFunction(SqlFunctionCtx *pCtx); void firstFunction(SqlFunctionCtx *pCtx);
void lastFunction(SqlFunctionCtx *pCtx); void lastFunction(SqlFunctionCtx *pCtx);

85
source/libs/function/src/builtins.c
View File

@ -291,6 +291,76 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = NULL, .initFunc = NULL,
.sprocessFunc = NULL, .sprocessFunc = NULL,
.finalizeFunc = NULL .finalizeFunc = NULL
},
{
.name = "_rowts",
.type = FUNCTION_TYPE_ROWTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "tbname",
.type = FUNCTION_TYPE_TBNAME,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_qstartts",
.type = FUNCTION_TYPE_QSTARTTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_qendts",
.type = FUNCTION_TYPE_QENDTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_wstartts",
.type = FUNCTION_TYPE_QSTARTTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_wendts",
.type = FUNCTION_TYPE_QENDTS,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_wduration",
.type = FUNCTION_TYPE_WDURATION,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
} }
}; };
@ -329,8 +399,23 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
break; break;
} }
case FUNCTION_TYPE_CONCAT: case FUNCTION_TYPE_CONCAT:
case FUNCTION_TYPE_ROWTS:
case FUNCTION_TYPE_TBNAME:
case FUNCTION_TYPE_QSTARTTS:
case FUNCTION_TYPE_QENDTS:
case FUNCTION_TYPE_WSTARTTS:
case FUNCTION_TYPE_WENDTS:
case FUNCTION_TYPE_WDURATION:
// todo // todo
break; break;
case FUNCTION_TYPE_ABS: {
SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0);
int32_t paraType = pParam->node.resType.type;
pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType };
break;
}
default: default:
ASSERT(0); // to found the fault ASAP. ASSERT(0); // to found the fault ASAP.
} }

24
source/libs/function/src/functionMgt.c
View File

@ -44,6 +44,13 @@ static void doInitFunctionHashTable() {
} }
} }
static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
}
int32_t fmFuncMgtInit() { int32_t fmFuncMgtInit() {
taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable); taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable);
return initFunctionCode; return initFunctionCode;
@ -89,10 +96,19 @@ int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
} }
bool fmIsAggFunc(int32_t funcId) { bool fmIsAggFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return isSpecificClassifyFunc(funcId, FUNC_MGT_AGG_FUNC);
return false; }
}
return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, FUNC_MGT_AGG_FUNC); bool fmIsScalarFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC);
}
bool fmIsWindowPseudoColumnFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC);
}
bool fmIsWindowClauseFunc(int32_t funcId) {
return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId);
} }
void fmFuncMgtDestroy() { void fmFuncMgtDestroy() {

4
source/libs/function/src/taggfunction.c
View File

@ -3078,8 +3078,8 @@ static void arithmetic_function(SqlFunctionCtx *pCtx) {
GET_RES_INFO(pCtx)->numOfRes += pCtx->size; GET_RES_INFO(pCtx)->numOfRes += pCtx->size;
//SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz;
SScalarParam output = {0}; // SScalarParam output = {0};
output.data = pCtx->pOutput; // output.data = pCtx->pOutput;
//evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData);
} }

3
source/libs/parser/inc/parAst.h
View File

@ -53,7 +53,6 @@ typedef enum EDatabaseOptionType {
DB_OPTION_SINGLE_STABLE, DB_OPTION_SINGLE_STABLE,
DB_OPTION_STREAM_MODE, DB_OPTION_STREAM_MODE,
DB_OPTION_RETENTIONS, DB_OPTION_RETENTIONS,
DB_OPTION_FILE_FACTOR,
DB_OPTION_MAX DB_OPTION_MAX
} EDatabaseOptionType; } EDatabaseOptionType;
@ -63,6 +62,8 @@ typedef enum ETableOptionType {
TABLE_OPTION_TTL, TABLE_OPTION_TTL,
TABLE_OPTION_COMMENT, TABLE_OPTION_COMMENT,
TABLE_OPTION_SMA, TABLE_OPTION_SMA,
TABLE_OPTION_FILE_FACTOR,
TABLE_OPTION_DELAY,
TABLE_OPTION_MAX TABLE_OPTION_MAX
} ETableOptionType; } ETableOptionType;

39
source/libs/parser/inc/sql.y
View File

@ -15,6 +15,7 @@
#include <assert.h> #include <assert.h>
#include <stdbool.h> #include <stdbool.h>
#include "functionMgt.h"
#include "nodes.h" #include "nodes.h"
#include "parToken.h" #include "parToken.h"
#include "ttokendef.h" #include "ttokendef.h"
@ -146,7 +147,6 @@ db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C).
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); }
db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); } db_options(A) ::= db_options(B) STREAM_MODE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STREAM_MODE, &C); }
db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); } db_options(A) ::= db_options(B) RETENTIONS NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, &C); }
db_options(A) ::= db_options(B) FILE_FACTOR NK_FLOAT(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FILE_FACTOR, &C); }
alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); } alter_db_options(A) ::= alter_db_option(B). { A = createDefaultAlterDatabaseOptions(pCxt); A = setDatabaseOption(pCxt, A, B.type, &B.val); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseOption(pCxt, B, C.type, &C.val); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setDatabaseOption(pCxt, B, C.type, &C.val); }
@ -263,6 +263,8 @@ table_options(A) ::= table_options(B) KEEP NK_INTEGER(C).
table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); }
table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableSmaOption(pCxt, B, C); }
table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); } table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableRollupOption(pCxt, B, C); }
table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C). { A = setTableOption(pCxt, B, TABLE_OPTION_FILE_FACTOR, &C); }
table_options(A) ::= table_options(B) DELAY NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_DELAY, &C); }
alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); } alter_table_options(A) ::= alter_table_option(B). { A = createDefaultAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); }
alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); } alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); }
@ -416,7 +418,7 @@ topic_name(A) ::= NK_ID(B).
/************************************************ expression **********************************************************/ /************************************************ expression **********************************************************/
expression(A) ::= literal(B). { A = B; } expression(A) ::= literal(B). { A = B; }
//expression(A) ::= NK_QUESTION(B). { A = B; } //expression(A) ::= NK_QUESTION(B). { A = B; }
//expression(A) ::= pseudo_column(B). { A = B; } expression(A) ::= pseudo_column(B). { A = B; }
expression(A) ::= column_reference(B). { A = B; } expression(A) ::= column_reference(B). { A = B; }
expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); } expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); }
@ -466,7 +468,38 @@ expression_list(A) ::= expression_list(B) NK_COMMA expression(C).
column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); } column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); } column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
//pseudo_column(A) ::= NK_NOW. { A = createFunctionNode(pCxt, NULL, NULL); } //pseudo_column(A) ::= NK_NOW. { A = createFunctionNode(pCxt, NULL, NULL); }
pseudo_column(A) ::= NK_UNDERLINE(B) ROWTS(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
}
pseudo_column(A) ::= TBNAME(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= NK_UNDERLINE(B) QSTARTTS(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
}
pseudo_column(A) ::= NK_UNDERLINE(B) QENDTS(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
}
pseudo_column(A) ::= NK_UNDERLINE(B) WSTARTTS(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
}
pseudo_column(A) ::= NK_UNDERLINE(B) WENDTS(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
}
pseudo_column(A) ::= NK_UNDERLINE(B) WDURATION(C). {
SToken t = B;
t.n = (C.z + C.n) - B.z;
A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
}
/************************************************ predicate ***********************************************************/ /************************************************ predicate ***********************************************************/
predicate(A) ::= expression(B) compare_op(C) expression(D). { predicate(A) ::= expression(B) compare_op(C) expression(D). {

73
source/libs/parser/src/parAstCreater.c
View File

@ -248,12 +248,37 @@ static SDatabaseOptions* setDbStreamMode(SAstCreateContext* pCxt, SDatabaseOptio
} }
static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) { static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo pOptions->pRetentions = nodesMakeList();
return pOptions; if (NULL == pOptions->pRetentions) {
} pCxt->valid = false;
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory");
return pOptions;
}
char val[20] = {0};
int32_t len = trimString(pVal->z, pVal->n, val, sizeof(val));
char* pStart = val;
char* pEnd = val + len;
int32_t sepOrder = 1;
while (1) {
char* pPos = strchr(pStart, (0 == sepOrder % 2) ? ',' : ':');
SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)};
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) {
pCxt->valid = false;
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory");
return pOptions;
}
if (NULL == pPos) {
break;
}
pStart = pPos + 1;
}
if (LIST_LENGTH(pOptions->pRetentions) % 2 != 0) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", val);
pCxt->valid = false;
}
static SDatabaseOptions* setDbFileFactor(SAstCreateContext* pCxt, SDatabaseOptions* pOptions, const SToken* pVal) {
// todo
return pOptions; return pOptions;
} }
@ -276,7 +301,6 @@ static void initSetDatabaseOptionFp() {
setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable; setDbOptionFuncs[DB_OPTION_SINGLE_STABLE] = setDbSingleStable;
setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode; setDbOptionFuncs[DB_OPTION_STREAM_MODE] = setDbStreamMode;
setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions; setDbOptionFuncs[DB_OPTION_RETENTIONS] = setDbRetentions;
setDbOptionFuncs[DB_OPTION_FILE_FACTOR] = setDbFileFactor;
} }
static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) { static STableOptions* setTableKeep(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
@ -314,10 +338,36 @@ static STableOptions* setTableComment(SAstCreateContext* pCxt, STableOptions* pO
return pOptions; return pOptions;
} }
static STableOptions* setTableFileFactor(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
double val = strtod(pVal->z, NULL);
if (val < TSDB_MIN_DB_FILE_FACTOR || val > TSDB_MAX_DB_FILE_FACTOR) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option file_factor: %f valid range: [%d, %d]", val, TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR);
pCxt->valid = false;
return pOptions;
}
pOptions->filesFactor = val;
return pOptions;
}
static STableOptions* setTableDelay(SAstCreateContext* pCxt, STableOptions* pOptions, const SToken* pVal) {
int64_t val = strtol(pVal->z, NULL, 10);
if (val < TSDB_MIN_DB_DELAY || val > TSDB_MAX_DB_DELAY) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen,
"invalid table option delay: %"PRId64" valid range: [%d, %d]", val, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
pCxt->valid = false;
return pOptions;
}
pOptions->delay = val;
return pOptions;
}
static void initSetTableOptionFp() { static void initSetTableOptionFp() {
setTableOptionFuncs[TABLE_OPTION_KEEP] = setTableKeep; setTableOptionFuncs[TABLE_OPTION_KEEP] = setTableKeep;
setTableOptionFuncs[TABLE_OPTION_TTL] = setTableTtl; setTableOptionFuncs[TABLE_OPTION_TTL] = setTableTtl;
setTableOptionFuncs[TABLE_OPTION_COMMENT] = setTableComment; setTableOptionFuncs[TABLE_OPTION_COMMENT] = setTableComment;
setTableOptionFuncs[TABLE_OPTION_FILE_FACTOR] = setTableFileFactor;
setTableOptionFuncs[TABLE_OPTION_DELAY] = setTableDelay;
} }
void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) { void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
@ -906,6 +956,8 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) {
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->keep = TSDB_DEFAULT_KEEP; pOptions->keep = TSDB_DEFAULT_KEEP;
pOptions->ttl = TSDB_DEFAULT_DB_TTL_OPTION; pOptions->ttl = TSDB_DEFAULT_DB_TTL_OPTION;
pOptions->filesFactor = TSDB_DEFAULT_DB_FILE_FACTOR;
pOptions->delay = TSDB_DEFAULT_DB_DELAY;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
@ -914,6 +966,8 @@ SNode* createDefaultAlterTableOptions(SAstCreateContext* pCxt) {
CHECK_OUT_OF_MEM(pOptions); CHECK_OUT_OF_MEM(pOptions);
pOptions->keep = -1; pOptions->keep = -1;
pOptions->ttl = -1; pOptions->ttl = -1;
pOptions->filesFactor = -1;
pOptions->delay = -1;
return (SNode*)pOptions; return (SNode*)pOptions;
} }
@ -927,7 +981,12 @@ SNode* setTableSmaOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pS
} }
SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) { SNode* setTableRollupOption(SAstCreateContext* pCxt, SNode* pOptions, SNodeList* pFuncs) {
// todo if (1 != LIST_LENGTH(pFuncs)) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table option rollup: only one function is allowed");
pCxt->valid = false;
return pOptions;
}
((STableOptions*)pOptions)->pFuncs = pFuncs;
return pOptions; return pOptions;
} }

17
source/libs/parser/src/parTokenizer.c
View File

@ -48,6 +48,7 @@ static SKeyword keywordTable[] = {
{"DATABASE", TK_DATABASE}, {"DATABASE", TK_DATABASE},
{"DATABASES", TK_DATABASES}, {"DATABASES", TK_DATABASES},
{"DAYS", TK_DAYS}, {"DAYS", TK_DAYS},
{"DELAY", TK_DELAY},
{"DESC", TK_DESC}, {"DESC", TK_DESC},
{"DISTINCT", TK_DISTINCT}, {"DISTINCT", TK_DISTINCT},
{"DNODE", TK_DNODE}, {"DNODE", TK_DNODE},
@ -55,7 +56,7 @@ static SKeyword keywordTable[] = {
{"DOUBLE", TK_DOUBLE}, {"DOUBLE", TK_DOUBLE},
{"DROP", TK_DROP}, {"DROP", TK_DROP},
{"EXISTS", TK_EXISTS}, {"EXISTS", TK_EXISTS},
// {"FILE", TK_FILE}, {"FILE_FACTOR", TK_FILE_FACTOR},
{"FILL", TK_FILL}, {"FILL", TK_FILL},
{"FLOAT", TK_FLOAT}, {"FLOAT", TK_FLOAT},
{"FROM", TK_FROM}, {"FROM", TK_FROM},
@ -103,10 +104,15 @@ static SKeyword keywordTable[] = {
{"PRECISION", TK_PRECISION}, {"PRECISION", TK_PRECISION},
{"PRIVILEGE", TK_PRIVILEGE}, {"PRIVILEGE", TK_PRIVILEGE},
{"PREV", TK_PREV}, {"PREV", TK_PREV},
{"QENDTS", TK_QENDTS},
{"QNODE", TK_QNODE}, {"QNODE", TK_QNODE},
{"QNODES", TK_QNODES}, {"QNODES", TK_QNODES},
{"QSTARTTS", TK_QSTARTTS},
{"QUORUM", TK_QUORUM}, {"QUORUM", TK_QUORUM},
{"REPLICA", TK_REPLICA}, {"REPLICA", TK_REPLICA},
{"RETENTIONS", TK_RETENTIONS},
{"ROLLUP", TK_ROLLUP},
{"ROWTS", TK_ROWTS},
{"SELECT", TK_SELECT}, {"SELECT", TK_SELECT},
{"SESSION", TK_SESSION}, {"SESSION", TK_SESSION},
{"SHOW", TK_SHOW}, {"SHOW", TK_SHOW},
@ -124,6 +130,7 @@ static SKeyword keywordTable[] = {
{"TABLE", TK_TABLE}, {"TABLE", TK_TABLE},
{"TABLES", TK_TABLES}, {"TABLES", TK_TABLES},
{"TAGS", TK_TAGS}, {"TAGS", TK_TAGS},
{"TBNAME", TK_TBNAME},
{"TIMESTAMP", TK_TIMESTAMP}, {"TIMESTAMP", TK_TIMESTAMP},
{"TINYINT", TK_TINYINT}, {"TINYINT", TK_TINYINT},
{"TOPIC", TK_TOPIC}, {"TOPIC", TK_TOPIC},
@ -138,7 +145,10 @@ static SKeyword keywordTable[] = {
{"VARCHAR", TK_VARCHAR}, {"VARCHAR", TK_VARCHAR},
{"VGROUPS", TK_VGROUPS}, {"VGROUPS", TK_VGROUPS},
{"WAL", TK_WAL}, {"WAL", TK_WAL},
{"WDURATION", TK_WDURATION},
{"WENDTS", TK_WENDTS},
{"WHERE", TK_WHERE}, {"WHERE", TK_WHERE},
{"WSTARTTS", TK_WSTARTTS},
// {"ID", TK_ID}, // {"ID", TK_ID},
// {"STRING", TK_STRING}, // {"STRING", TK_STRING},
// {"EQ", TK_EQ}, // {"EQ", TK_EQ},
@ -230,7 +240,6 @@ static SKeyword keywordTable[] = {
// {"TRIGGER", TK_TRIGGER}, // {"TRIGGER", TK_TRIGGER},
// {"VIEW", TK_VIEW}, // {"VIEW", TK_VIEW},
// {"SEMI", TK_SEMI}, // {"SEMI", TK_SEMI},
// {"TBNAME", TK_TBNAME},
// {"VNODES", TK_VNODES}, // {"VNODES", TK_VNODES},
// {"PARTITIONS", TK_PARTITIONS}, // {"PARTITIONS", TK_PARTITIONS},
// {"TOPICS", TK_TOPICS}, // {"TOPICS", TK_TOPICS},
@ -424,6 +433,10 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
*tokenId = TK_NK_QUESTION; *tokenId = TK_NK_QUESTION;
return 1; return 1;
} }
case '_': {
*tokenId = TK_NK_UNDERLINE;
return 1;
}
case '`': case '`':
case '\'': case '\'':
case '"': { case '"': {

146
source/libs/parser/src/parTranslater.c
View File

@ -920,7 +920,35 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
return code; return code;
} }
static void buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt, SCreateDbReq* pReq) { static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbReq* pReq) {
if (NULL != pRetentions) {
pReq->pRetensions = taosArrayInit(LIST_LENGTH(pRetentions) / 2, sizeof(SRetention));
if (NULL == pReq->pRetensions) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SValueNode* pFreq = NULL;
SValueNode* pKeep = NULL;
SNode* pNode = NULL;
int32_t index = 0;
FOREACH(pNode, pRetentions) {
if (0 == index % 2) {
pFreq = (SValueNode*)pNode;
} else {
pKeep = (SValueNode*)pNode;
SRetention retention = {
.freq = pFreq->datum.i,
.freqUnit = pFreq->unit,
.keep = pKeep->datum.i,
.keepUnit = pKeep->unit
};
taosArrayPush(pReq->pRetensions, &retention);
}
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt, SCreateDbReq* pReq) {
SName name = {0}; SName name = {0};
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, pReq->db); tNameGetFullDbName(&name, pReq->db);
@ -944,27 +972,45 @@ static void buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt
pReq->cacheLastRow = pStmt->pOptions->cachelast; pReq->cacheLastRow = pStmt->pOptions->cachelast;
pReq->ignoreExist = pStmt->ignoreExists; pReq->ignoreExist = pStmt->ignoreExists;
pReq->streamMode = pStmt->pOptions->streamMode; pReq->streamMode = pStmt->pOptions->streamMode;
return; return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq);
}
static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) {
if (NULL != pStmt->pOptions->pRetentions) {
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pOptions->pRetentions) {
if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pNode)) {
return pCxt->errCode;
}
}
}
return TSDB_CODE_SUCCESS;
} }
static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) { static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* pStmt) {
SCreateDbReq createReq = {0}; SCreateDbReq createReq = {0};
buildCreateDbReq(pCxt, pStmt, &createReq);
pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); int32_t code = checkCreateDatabase(pCxt, pStmt);
if (NULL == pCxt->pCmdMsg) { if (TSDB_CODE_SUCCESS == code) {
return TSDB_CODE_OUT_OF_MEMORY; code = buildCreateDbReq(pCxt, pStmt, &createReq);
} }
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DB;
pCxt->pCmdMsg->msgLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSCreateDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq);
return TSDB_CODE_SUCCESS; if (TSDB_CODE_SUCCESS == code) {
pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DB;
pCxt->pCmdMsg->msgLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tSerializeSCreateDbReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createReq);
}
return code;
} }
static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* pStmt) { static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* pStmt) {
@ -1035,7 +1081,7 @@ static int32_t calcTypeBytes(SDataType dt) {
} }
} }
static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) { static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) {
*pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
SNode* pNode; SNode* pNode;
FOREACH(pNode, pList) { FOREACH(pNode, pList) {
@ -1047,13 +1093,77 @@ static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) {
if (NULL == pList) {
return TSDB_CODE_SUCCESS;
}
*pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
SNode* pNode;
FOREACH(pNode, pList) {
SColumnNode* pCol = (SColumnNode*)pNode;
SField field = { .type = pCol->node.resType.type, .bytes = calcTypeBytes(pCol->node.resType) };
strcpy(field.name, pCol->colName);
taosArrayPush(*pArray, &field);
}
return TSDB_CODE_SUCCESS;
}
static const SColumnDefNode* findColDef(const SNodeList* pCols, const SColumnNode* pCol) {
SNode* pColDef = NULL;
FOREACH(pColDef, pCols) {
if (0 == strcmp(pCol->colName, ((SColumnDefNode*)pColDef)->colName)) {
return (SColumnDefNode*)pColDef;
}
}
return NULL;
}
static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
if (NULL != pStmt->pOptions->pSma) {
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pOptions->pSma) {
SColumnNode* pSmaCol = (SColumnNode*)pNode;
const SColumnDefNode* pColDef = findColDef(pStmt->pCols, pSmaCol);
if (NULL == pColDef) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pSmaCol->colName);
}
pSmaCol->node.resType = pColDef->dataType;
}
}
if (NULL != pStmt->pOptions->pFuncs) {
SFunctionNode* pFunc = nodesListGetNode(pStmt->pOptions->pFuncs, 0);
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t getAggregationMethod(SNodeList* pFuncs) {
if (NULL == pFuncs) {
return -1;
}
return ((SFunctionNode*)nodesListGetNode(pFuncs, 0))->funcId;
}
static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
int32_t code = checkCreateTable(pCxt, pStmt);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
SMCreateStbReq createReq = {0}; SMCreateStbReq createReq = {0};
createReq.igExists = pStmt->ignoreExists; createReq.igExists = pStmt->ignoreExists;
columnNodeToField(pStmt->pCols, &createReq.pColumns); createReq.aggregationMethod = getAggregationMethod(pStmt->pOptions->pFuncs);
columnNodeToField(pStmt->pTags, &createReq.pTags); createReq.xFilesFactor = pStmt->pOptions->filesFactor;
createReq.delay = pStmt->pOptions->delay;
columnDefNodeToField(pStmt->pCols, &createReq.pColumns);
columnDefNodeToField(pStmt->pTags, &createReq.pTags);
columnNodeToField(pStmt->pOptions->pSma, &createReq.pSmas);
createReq.numOfColumns = LIST_LENGTH(pStmt->pCols); createReq.numOfColumns = LIST_LENGTH(pStmt->pCols);
createReq.numOfTags = LIST_LENGTH(pStmt->pTags); createReq.numOfTags = LIST_LENGTH(pStmt->pTags);
createReq.numOfSmas = LIST_LENGTH(pStmt->pOptions->pSma);
SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId }; SName tableName = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
strcpy(tableName.dbname, pStmt->dbName); strcpy(tableName.dbname, pStmt->dbName);

3993
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

12
source/libs/parser/test/parserAstTest.cpp
View File

@ -206,6 +206,13 @@ TEST_F(ParserTest, selectExpression) {
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(ParserTest, selectPseudoColumn) {
setDatabase("root", "test");
bind("SELECT _wstartts, _wendts, count(*) FROM t1 interval(10s)");
ASSERT_TRUE(run());
}
TEST_F(ParserTest, selectClause) { TEST_F(ParserTest, selectClause) {
setDatabase("root", "test"); setDatabase("root", "test");
@ -416,6 +423,7 @@ TEST_F(ParserTest, createDatabase) {
"VGROUPS 100 " "VGROUPS 100 "
"SINGLE_STABLE 0 " "SINGLE_STABLE 0 "
"STREAM_MODE 1 " "STREAM_MODE 1 "
"RETENTIONS '15s:7d,1m:21d,15m:5y'"
); );
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
@ -469,7 +477,7 @@ TEST_F(ParserTest, createTable) {
"c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), c14 JSON, c15 VARCHAR(50)) " "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), c14 JSON, c15 VARCHAR(50)) "
"TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 BINARY(20), a8 SMALLINT, " "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 BINARY(20), a8 SMALLINT, "
"a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), a14 JSON, a15 VARCHAR(50)) " "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), a14 JSON, a15 VARCHAR(50)) "
"KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)" "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"
); );
ASSERT_TRUE(run()); ASSERT_TRUE(run());
@ -491,7 +499,7 @@ TEST_F(ParserTest, createTable) {
"c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), c14 JSON, c15 VARCHAR(50)) " "c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, c13 NCHAR(30), c14 JSON, c15 VARCHAR(50)) "
"TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 BINARY(20), a8 SMALLINT, " "TAGS (tsa TIMESTAMP, a1 INT, a2 INT UNSIGNED, a3 BIGINT, a4 BIGINT UNSIGNED, a5 FLOAT, a6 DOUBLE, a7 BINARY(20), a8 SMALLINT, "
"a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), a14 JSON, a15 VARCHAR(50)) " "a9 SMALLINT UNSIGNED COMMENT 'test column comment', a10 TINYINT, a11 TINYINT UNSIGNED, a12 BOOL, a13 NCHAR(30), a14 JSON, a15 VARCHAR(50)) "
"KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)" "KEEP 100 TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (min) FILE_FACTOR 0.1 DELAY 2"
); );
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }

2
source/libs/planner/src/planLogicCreater.c
View File

@ -411,7 +411,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
} }
static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) {
int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs); int32_t code = nodesCollectFuncs(pSelect, fmIsWindowClauseFunc, &pWindow->pFuncs);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);

13
source/libs/planner/src/planPhysiCreater.c
View File

@ -36,10 +36,6 @@ typedef struct SPhysiPlanContext {
} SPhysiPlanContext; } SPhysiPlanContext;
static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
if (QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode)) {
return getSlotKey(((SOrderByExprNode*)pNode)->pExpr, pStmtName, pKey);
}
if (QUERY_NODE_COLUMN == nodeType(pNode)) { if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode; SColumnNode* pCol = (SColumnNode*)pNode;
if (NULL != pStmtName) { if (NULL != pStmtName) {
@ -184,15 +180,16 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0; int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0;
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, pList) { FOREACH(pNode, pList) {
SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode;
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0}; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0};
int32_t len = getSlotKey(pNode, pStmtName, name); int32_t len = getSlotKey(pExpr, pStmtName, name);
SSlotIndex* pIndex = taosHashGet(pHash, name, len); SSlotIndex* pIndex = taosHashGet(pHash, name, len);
if (NULL == pIndex) { if (NULL == pIndex) {
code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, nextSlotId, output)); code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output));
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash); code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
} }
pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; pDataBlockDesc->resultRowSize += ((SExprNode*)pExpr)->resType.bytes;
slotId = nextSlotId; slotId = nextSlotId;
++nextSlotId; ++nextSlotId;
} else { } else {
@ -600,7 +597,7 @@ static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
return collectAndRewrite(pContext, pNode); return collectAndRewrite(pContext, pNode);
} }
case QUERY_NODE_FUNCTION: { case QUERY_NODE_FUNCTION: {
if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) { if (fmIsScalarFunc(((SFunctionNode*)(*pNode))->funcId)) {
return collectAndRewrite(pContext, pNode); return collectAndRewrite(pContext, pNode);
} }
} }

3
source/libs/planner/test/plannerTest.cpp
View File

@ -192,6 +192,9 @@ TEST_F(PlannerTest, interval) {
bind("SELECT count(*) FROM t1 interval(10s)"); bind("SELECT count(*) FROM t1 interval(10s)");
ASSERT_TRUE(run()); ASSERT_TRUE(run());
bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)");
ASSERT_TRUE(run());
} }
TEST_F(PlannerTest, sessionWindow) { TEST_F(PlannerTest, sessionWindow) {

8
source/libs/scalar/inc/sclInt.h
View File

@ -43,10 +43,12 @@ typedef struct SScalarCtx {
#define SCL_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define SCL_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
#define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type)
#define GET_PARAM_BYTES(_c) ((_c)->pColumnInfoData->info.bytes)
int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx);
bool sclIsNull(SScalarParam* param, int32_t idx);
void sclSetNull(SScalarParam* param, int32_t idx);
void sclFreeParam(SScalarParam *param); void sclFreeParam(SScalarParam *param);
#ifdef __cplusplus #ifdef __cplusplus

12
source/libs/scalar/inc/sclfunc.h
View File

@ -22,19 +22,7 @@ extern "C" {
#include "function.h" #include "function.h"
#include "scalar.h" #include "scalar.h"
typedef struct SScalarFunctionSupport {
struct SExprInfo *pExprInfo;
int32_t numOfCols;
SColumnInfo *colList;
void *exprList; // client side used
int32_t offset;
char** data;
} SScalarFunctionSupport;
extern struct SScalarFunctionInfo scalarFunc[8];
int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput,
void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t));
#ifdef __cplusplus #ifdef __cplusplus

60
source/libs/scalar/inc/sclvector.h
View File

@ -20,10 +20,66 @@
extern "C" { extern "C" {
#endif #endif
#include "sclfunc.h" typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index);
typedef double (*_mathFunc)(double, double, bool *); static FORCE_INLINE double getVectorDoubleValue_TINYINT(void *src, int32_t index) {
return (double)*((int8_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_UTINYINT(void *src, int32_t index) {
return (double)*((uint8_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_SMALLINT(void *src, int32_t index) {
return (double)*((int16_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_USMALLINT(void *src, int32_t index) {
return (double)*((uint16_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_INT(void *src, int32_t index) {
return (double)*((int32_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_UINT(void *src, int32_t index) {
return (double)*((uint32_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_BIGINT(void *src, int32_t index) {
return (double)*((int64_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_UBIGINT(void *src, int32_t index) {
return (double)*((uint64_t *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_FLOAT(void *src, int32_t index) {
return (double)*((float *)src + index);
}
static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index) {
return (double)*((double *)src + index);
}
static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL;
if (srcType == TSDB_DATA_TYPE_TINYINT) {
p = getVectorDoubleValue_TINYINT;
} else if (srcType == TSDB_DATA_TYPE_UTINYINT) {
p = getVectorDoubleValue_UTINYINT;
} else if (srcType == TSDB_DATA_TYPE_SMALLINT) {
p = getVectorDoubleValue_SMALLINT;
} else if (srcType == TSDB_DATA_TYPE_USMALLINT) {
p = getVectorDoubleValue_USMALLINT;
} else if (srcType == TSDB_DATA_TYPE_INT) {
p = getVectorDoubleValue_INT;
} else if (srcType == TSDB_DATA_TYPE_UINT) {
p = getVectorDoubleValue_UINT;
} else if (srcType == TSDB_DATA_TYPE_BIGINT) {
p = getVectorDoubleValue_BIGINT;
} else if (srcType == TSDB_DATA_TYPE_UBIGINT) {
p = getVectorDoubleValue_UBIGINT;
} else if (srcType == TSDB_DATA_TYPE_FLOAT) {
p = getVectorDoubleValue_FLOAT;
} else if (srcType == TSDB_DATA_TYPE_DOUBLE) {
p = getVectorDoubleValue_DOUBLE;
} else {
assert(0);
}
return p;
}
typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType); typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType);
typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output, int32_t order); typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output, int32_t order);

58
source/libs/scalar/src/filter.c
View File

@ -318,7 +318,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt
r->prev = NULL; r->prev = NULL;
r->next = NULL; r->next = NULL;
} else { } else {
r = taosMemoryCalloc(1, sizeof(SFilterRangeNode)); r = taosMemoryCalloc(1, sizeof(SFilterRangeNode));
} }
FILTER_COPY_RA(&r->ra, ra); FILTER_COPY_RA(&r->ra, ra);
@ -1021,26 +1021,21 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) {
SNodeListNode *listNode = (SNodeListNode *)node->pRight; SNodeListNode *listNode = (SNodeListNode *)node->pRight;
SListCell *cell = listNode->pNodeList->pHead; SListCell *cell = listNode->pNodeList->pHead;
SScalarParam in = {.num = 1}, out = {.num = 1, .type = type};
SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
out.columnData->info.type = type;
for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { for (int32_t i = 0; i < listNode->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode; SValueNode *valueNode = (SValueNode *)cell->pNode;
in.type = valueNode->node.resType.type; code = doConvertDataType(valueNode, &out);
in.bytes = valueNode->node.resType.bytes;
in.data = nodesGetValueFromNode(valueNode);
out.data = taosMemoryMalloc(sizeof(int64_t));
code = vectorConvertImpl(&in, &out);
if (code) { if (code) {
fltError("convert from %d to %d failed", in.type, out.type); // fltError("convert from %d to %d failed", in.type, out.type);
taosMemoryFreeClear(out.data);
FLT_ERR_RET(code); FLT_ERR_RET(code);
} }
len = tDataTypes[type].bytes; len = tDataTypes[type].bytes;
filterAddField(info, NULL, &out.data, FLD_TYPE_VALUE, &right, len, true); filterAddField(info, NULL, (void**) &out.columnData->pData, FLD_TYPE_VALUE, &right, len, true);
filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx);
SFilterGroup fgroup = {0}; SFilterGroup fgroup = {0};
@ -1054,7 +1049,6 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
filterAddFieldFromNode(info, node->pRight, &right); filterAddFieldFromNode(info, node->pRight, &right);
FLT_ERR_RET(filterAddUnit(info, node->opType, &left, &right, &uidx)); FLT_ERR_RET(filterAddUnit(info, node->opType, &left, &right, &uidx));
SFilterGroup fgroup = {0}; SFilterGroup fgroup = {0};
filterAddUnitToGroup(&fgroup, uidx); filterAddUnitToGroup(&fgroup, uidx);
@ -1080,7 +1074,6 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right.
t = FILTER_GET_FIELD(dst, right); t = FILTER_GET_FIELD(dst, right);
FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH);
} else { } else {
filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false);
@ -1101,14 +1094,12 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) { int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) {
SFilterUnit *u = &info->units[uidx]; SFilterUnit *u = &info->units[uidx];
u->compare.optr2 = optr; u->compare.optr2 = optr;
u->right2 = *right; u->right2 = *right;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
SFilterFieldId left, right, right2; SFilterFieldId left, right, right2;
uint32_t uidx = 0; uint32_t uidx = 0;
@ -1800,9 +1791,12 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
if (dType->type == type) { if (dType->type == type) {
assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
} else { } else {
SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
SScalarParam out = {.data = fi->data, .num = 1, .type = type}; out.columnData->info.type = type;
if (vectorConvertImpl(&in, &out)) {
// todo refactor the convert
int32_t code = doConvertDataType(var, &out);
if (code != TSDB_CODE_SUCCESS) {
qError("convert value to type[%d] failed", type); qError("convert value to type[%d] failed", type);
return TSDB_CODE_TSC_INVALID_OPERATION; return TSDB_CODE_TSC_INVALID_OPERATION;
} }
@ -3636,7 +3630,7 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options)
if (*pInfo == NULL) { if (*pInfo == NULL) {
*pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo)); *pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo));
if (NULL == *pInfo) { if (NULL == *pInfo) {
fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo)); fltError("taosMemoryCalloc %d failed", (int32_t)sizeof(SFilterInfo));
FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
} }
@ -3676,18 +3670,18 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData
FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output));
taosArrayDestroy(pList); taosArrayDestroy(pList);
// TODO Fix it
*p = output.orig.data; // *p = output.orig.data;
output.orig.data = NULL; // output.orig.data = NULL;
//
sclFreeParam(&output); // sclFreeParam(&output);
//
int8_t *r = output.data; // int8_t *r = output.data;
for (int32_t i = 0; i < output.num; ++i) { // for (int32_t i = 0; i < output.num; ++i) {
if (0 == *(r+i)) { // if (0 == *(r+i)) {
return false; // return false;
} // }
} // }
return true; return true;
} }

342
source/libs/scalar/src/scalar.c
View File

@ -6,6 +6,7 @@
#include "sclvector.h" #include "sclvector.h"
#include "tcommon.h" #include "tcommon.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "scalar.h"
int32_t scalarGetOperatorParamNum(EOperatorType type) { int32_t scalarGetOperatorParamNum(EOperatorType type) {
if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type
@ -16,6 +17,41 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) {
return 2; return 2;
} }
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
if (pColumnData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pColumnData->info.type = pType->type;
pColumnData->info.bytes = pType->bytes;
pColumnData->info.scale = pType->scale;
pColumnData->info.precision = pType->precision;
int32_t code = blockDataEnsureColumnCapacity(pColumnData, numOfRows);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pColumnData);
return NULL;
} else {
return pColumnData;
}
}
int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) {
SScalarParam in = {.numOfRows = 1};
in.columnData = createColumnInfoData(&pValueNode->node.resType, 1);
colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false);
blockDataEnsureColumnCapacity(out->columnData, 1);
int32_t code = vectorConvertImpl(&in, out);
sclFreeParam(&in);
return code;
}
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false);
if (NULL == pObj) { if (NULL == pObj) {
@ -28,10 +64,8 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
int32_t code = 0; int32_t code = 0;
SNodeListNode *nodeList = (SNodeListNode *)pNode; SNodeListNode *nodeList = (SNodeListNode *)pNode;
SListCell *cell = nodeList->pNodeList->pHead; SListCell *cell = nodeList->pNodeList->pHead;
SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
int8_t dummy = 0;
int32_t bufLen = 60;
out.data = taosMemoryMalloc(bufLen);
int32_t len = 0; int32_t len = 0;
void *buf = NULL; void *buf = NULL;
@ -39,22 +73,21 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
SValueNode *valueNode = (SValueNode *)cell->pNode; SValueNode *valueNode = (SValueNode *)cell->pNode;
if (valueNode->node.resType.type != type) { if (valueNode->node.resType.type != type) {
in.type = valueNode->node.resType.type; out.columnData->info.type = type;
in.bytes = valueNode->node.resType.bytes; out.columnData->info.bytes = tDataTypes[type].bytes;
in.data = nodesGetValueFromNode(valueNode);
code = doConvertDataType(valueNode, &out);
code = vectorConvertImpl(&in, &out); if (code != TSDB_CODE_SUCCESS) {
if (code) { // sclError("convert data from %d to %d failed", in.type, out.type);
sclError("convert from %d to %d failed", in.type, out.type);
SCL_ERR_JRET(code); SCL_ERR_JRET(code);
} }
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
len = varDataLen(out.data); len = varDataLen(out.columnData->pData);
buf = varDataVal(out.data); buf = varDataVal(out.columnData->pData);
} else { } else {
len = tDataTypes[type].bytes; len = tDataTypes[type].bytes;
buf = out.data; buf = out.columnData->pData;
} }
} else { } else {
buf = nodesGetValueFromNode(valueNode); buf = nodesGetValueFromNode(valueNode);
@ -63,11 +96,10 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
buf = varDataVal(buf); buf = varDataVal(buf);
} else { } else {
len = valueNode->node.resType.bytes; len = valueNode->node.resType.bytes;
buf = out.data; }
}
} }
if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) {
sclError("taosHashPut failed"); sclError("taosHashPut failed");
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
@ -75,40 +107,14 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) {
cell = cell->pNext; cell = cell->pNext;
} }
taosMemoryFreeClear(out.data);
*data = pObj; *data = pObj;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
taosMemoryFreeClear(out.data);
taosHashCleanup(pObj); taosHashCleanup(pObj);
SCL_RET(code); SCL_RET(code);
} }
FORCE_INLINE bool sclIsNull(SScalarParam* param, int32_t idx) {
if (param->dataInBlock) {
return colDataIsNull(param->orig.columnData, 0, idx, NULL);
}
return param->bitmap ? colDataIsNull_f(param->bitmap, idx) : false;
}
FORCE_INLINE void sclSetNull(SScalarParam* param, int32_t idx) {
if (NULL == param->bitmap) {
param->bitmap = taosMemoryCalloc(BitmapLen(param->num), sizeof(char));
if (NULL == param->bitmap) {
sclError("calloc %d failed", param->num);
return;
}
}
colDataSetNull_f(param->bitmap, idx);
}
void sclFreeRes(SHashObj *res) { void sclFreeRes(SHashObj *res) {
SScalarParam *p = NULL; SScalarParam *p = NULL;
void *pIter = taosHashIterate(res, NULL); void *pIter = taosHashIterate(res, NULL);
@ -118,31 +124,22 @@ void sclFreeRes(SHashObj *res) {
if (p) { if (p) {
sclFreeParam(p); sclFreeParam(p);
} }
pIter = taosHashIterate(res, pIter); pIter = taosHashIterate(res, pIter);
} }
taosHashCleanup(res); taosHashCleanup(res);
} }
void sclFreeParamNoData(SScalarParam *param) {
taosMemoryFreeClear(param->bitmap);
}
void sclFreeParam(SScalarParam *param) { void sclFreeParam(SScalarParam *param) {
sclFreeParamNoData(param); if (param->columnData != NULL) {
colDataDestroy(param->columnData);
if (!param->dataInBlock) { taosMemoryFreeClear(param->columnData);
if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { }
taosHashCleanup((SHashObj *)param->orig.data);
} else { if (param->pHashFilter != NULL) {
taosMemoryFreeClear(param->orig.data); taosHashCleanup(param->pHashFilter);
}
} }
} }
int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) {
if (TSDB_DATA_TYPE_NULL == pNode->node.resType.type) { if (TSDB_DATA_TYPE_NULL == pNode->node.resType.type) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -155,7 +152,6 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) {
} }
memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -163,35 +159,26 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
switch (nodeType(node)) { switch (nodeType(node)) {
case QUERY_NODE_VALUE: { case QUERY_NODE_VALUE: {
SValueNode *valueNode = (SValueNode *)node; SValueNode *valueNode = (SValueNode *)node;
//SCL_ERR_RET(sclCopyValueNodeValue(valueNode, &param->data));
param->data = nodesGetValueFromNode(valueNode); param->numOfRows = 1;
param->orig.data = param->data; param->columnData = createColumnInfoData(&valueNode->node.resType, 1);
param->num = 1; if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) {
param->type = valueNode->node.resType.type; colDataAppend(param->columnData, 0, NULL, true);
param->bytes = valueNode->node.resType.bytes; } else {
if (TSDB_DATA_TYPE_NULL == param->type) { colDataAppend(param->columnData, 0, nodesGetValueFromNode(valueNode), false);
sclSetNull(param, 0);
} }
param->dataInBlock = false;
break; break;
} }
case QUERY_NODE_NODE_LIST: { case QUERY_NODE_NODE_LIST: {
SNodeListNode *nodeList = (SNodeListNode *)node; SNodeListNode *nodeList = (SNodeListNode *)node;
if (nodeList->pNodeList->length <= 0) { if (LIST_LENGTH(nodeList->pNodeList) <= 0) {
sclError("invalid length in nodeList, length:%d", nodeList->pNodeList->length); sclError("invalid length in nodeList, length:%d", LIST_LENGTH(nodeList->pNodeList));
SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SCL_ERR_RET(scalarGenerateSetFromList(&param->data, node, nodeList->dataType.type)); SCL_ERR_RET(scalarGenerateSetFromList((void**) &param->pHashFilter, node, nodeList->dataType.type));
param->orig.data = param->data;
param->num = 1;
param->type = SCL_DATA_TYPE_DUMMY_HASH;
param->dataInBlock = false;
if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) {
taosHashCleanup(param->orig.data); taosHashCleanup(param->pHashFilter);
param->orig.data = NULL;
sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
@ -210,73 +197,38 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
} }
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, ref->dataBlockId); SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, ref->dataBlockId);
if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(block->pDataBlock, ref->slotId); SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(block->pDataBlock, ref->slotId);
param->data = NULL; param->numOfRows = block->info.rows;
param->orig.columnData = columnData; param->columnData = columnData;
param->dataInBlock = true;
param->num = block->info.rows;
param->type = columnData->info.type;
param->bytes = columnData->info.bytes;
break; break;
} }
case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION:
case QUERY_NODE_OPERATOR: { case QUERY_NODE_OPERATOR:
case QUERY_NODE_LOGIC_CONDITION: {
SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES);
if (NULL == res) { if (NULL == res) {
sclError("no result for node, type:%d, node:%p", nodeType(node), node); sclError("no result for node, type:%d, node:%p", nodeType(node), node);
SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
*param = *res; *param = *res;
break; break;
} }
default: default:
break; break;
} }
if (param->num > *rowNum) { if (param->numOfRows > *rowNum) {
if ((1 != param->num) && (1 < *rowNum)) { if ((1 != param->numOfRows) && (1 < *rowNum)) {
sclError("different row nums, rowNum:%d, newRowNum:%d", *rowNum, param->num); sclError("different row nums, rowNum:%d, newRowNum:%d", *rowNum, param->numOfRows);
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
*rowNum = param->num; *rowNum = param->numOfRows;
}
return TSDB_CODE_SUCCESS;
}
int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) {
SScalarParam *param = NULL;
for (int32_t i = 0; i < listNum; ++i) {
param = params + i;
if (1 == param->num) {
continue;
}
if (param->dataInBlock) {
param->data = colDataGetData(param->orig.columnData, idx);
} else if (idx) {
if (IS_VAR_DATA_TYPE(param->type)) {
param->data = (char *)(param->data) + varDataTLen(param->data);
} else {
param->data = (char *)(param->data) + tDataTypes[param->type].bytes;
}
} else {
param->data = param->orig.data;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -298,16 +250,13 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarC
} }
SCL_ERR_JRET(sclInitParam(cell->pNode, &paramList[i], ctx, rowNum)); SCL_ERR_JRET(sclInitParam(cell->pNode, &paramList[i], ctx, rowNum));
cell = cell->pNext; cell = cell->pNext;
} }
*pParams = paramList; *pParams = paramList;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
taosMemoryFreeClear(paramList); taosMemoryFreeClear(paramList);
SCL_RET(code); SCL_RET(code);
} }
@ -332,16 +281,13 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal
} }
*pParams = paramList; *pParams = paramList;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
taosMemoryFreeClear(paramList); taosMemoryFreeClear(paramList);
SCL_RET(code); SCL_RET(code);
} }
int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) {
if (NULL == node->pParameterList || node->pParameterList->length <= 0) { if (NULL == node->pParameterList || node->pParameterList->length <= 0) {
sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0);
@ -359,37 +305,30 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
int32_t rowNum = 0; int32_t rowNum = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum)); SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
output->type = node->node.resType.type; output->columnData = createColumnInfoData(&node->node.resType, rowNum);
output->data = taosMemoryCalloc(rowNum, sizeof(tDataTypes[output->type].bytes)); if (output->columnData == NULL) {
if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes)));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
output->orig.data = output->data;
for (int32_t i = 0; i < rowNum; ++i) {
sclMoveParamListData(output, 1, i);
sclMoveParamListData(params, node->pParameterList->length, i);
// for (int32_t i = 0; i < rowNum; ++i) {
code = (*ffpSet.process)(params, node->pParameterList->length, output); code = (*ffpSet.process)(params, node->pParameterList->length, output);
if (code) { if (code) {
sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code); SCL_ERR_JRET(code);
} // }
} }
_return: _return:
for (int32_t i = 0; i < node->pParameterList->length; ++i) { for (int32_t i = 0; i < node->pParameterList->length; ++i) {
sclFreeParamNoData(params + i); // sclFreeParamNoData(params + i);
} }
taosMemoryFreeClear(params); taosMemoryFreeClear(params);
SCL_RET(code); SCL_RET(code);
} }
int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *output) { int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *output) {
if (NULL == node->pParameterList || node->pParameterList->length <= 0) { if (NULL == node->pParameterList || node->pParameterList->length <= 0) {
sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0);
@ -409,28 +348,24 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
SScalarParam *params = NULL; SScalarParam *params = NULL;
int32_t rowNum = 0; int32_t rowNum = 0;
int32_t code = 0; int32_t code = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum)); SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
output->type = node->node.resType.type; int32_t type = node->node.resType.type;
output->bytes = sizeof(bool); output->numOfRows = rowNum;
output->num = rowNum;
output->data = taosMemoryCalloc(rowNum, sizeof(bool)); SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
if (NULL == output->data) { output->columnData = createColumnInfoData(&t, rowNum);
if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool)));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
output->orig.data = output->data;
bool value = false; bool value = false;
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
sclMoveParamListData(output, 1, i);
sclMoveParamListData(params, node->pParameterList->length, i);
for (int32_t m = 0; m < node->pParameterList->length; ++m) { for (int32_t m = 0; m < node->pParameterList->length; ++m) {
GET_TYPED_DATA(value, bool, params[m].type, params[m].data); char* p = colDataGetData(params[m].columnData, i);
GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p);
if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) {
break; break;
} else if (LOGIC_COND_TYPE_OR == node->condType && value) { } else if (LOGIC_COND_TYPE_OR == node->condType && value) {
@ -440,13 +375,12 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
} }
} }
*(bool *)output->data = value; colDataAppend(output->columnData, i, (char*) &value, false);
} }
_return: _return:
for (int32_t i = 0; i < node->pParameterList->length; ++i) { for (int32_t i = 0; i < node->pParameterList->length; ++i) {
sclFreeParamNoData(params + i); // sclFreeParamNoData(params + i);
} }
taosMemoryFreeClear(params); taosMemoryFreeClear(params);
@ -459,16 +393,11 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
int32_t code = 0; int32_t code = 0;
SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum)); SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
output->type = node->node.resType.type; if (output->columnData == NULL) {
output->num = rowNum; sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes);
output->bytes = tDataTypes[output->type].bytes;
output->data = taosMemoryCalloc(rowNum, tDataTypes[output->type].bytes);
if (NULL == output->data) {
sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes);
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
output->orig.data = output->data;
_bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType);
@ -479,18 +408,14 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC); OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC);
_return: _return:
for (int32_t i = 0; i < paramNum; ++i) { for (int32_t i = 0; i < paramNum; ++i) {
sclFreeParamNoData(params + i); // sclFreeParam(&params[i]);
} }
taosMemoryFreeClear(params); taosMemoryFreeClear(params);
SCL_RET(code); SCL_RET(code);
} }
EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
SFunctionNode *node = (SFunctionNode *)*pNode; SFunctionNode *node = (SFunctionNode *)*pNode;
SScalarParam output = {0}; SScalarParam output = {0};
@ -510,11 +435,12 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
res->node.resType = node->node.resType; res->node.resType = node->node.resType;
if (IS_VAR_DATA_TYPE(output.type)) { int32_t type = output.columnData->info.type;
res->datum.p = output.data; if (IS_VAR_DATA_TYPE(type)) {
output.data = NULL; res->datum.p = output.columnData->pData;
output.columnData->pData = NULL;
} else { } else {
memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes);
} }
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
@ -527,8 +453,8 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) {
SLogicConditionNode *node = (SLogicConditionNode *)*pNode; SLogicConditionNode *node = (SLogicConditionNode *)*pNode;
SScalarParam output = {0};
SScalarParam output = {0};
ctx->code = sclExecLogic(node, ctx, &output); ctx->code = sclExecLogic(node, ctx, &output);
if (ctx->code) { if (ctx->code) {
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
@ -544,25 +470,25 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) {
res->node.resType = node->node.resType; res->node.resType = node->node.resType;
if (IS_VAR_DATA_TYPE(output.type)) { int32_t type = output.columnData->info.type;
res->datum.p = output.data; if (IS_VAR_DATA_TYPE(type)) {
output.data = NULL; res->datum.p = output.columnData->pData;
output.columnData->pData = NULL;
} else { } else {
memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes);
} }
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
*pNode = (SNode*)res; *pNode = (SNode*)res;
sclFreeParam(&output); sclFreeParam(&output);
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)*pNode; SOperatorNode *node = (SOperatorNode *)*pNode;
SScalarParam output = {0};
SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
ctx->code = sclExecOperator(node, ctx, &output); ctx->code = sclExecOperator(node, ctx, &output);
if (ctx->code) { if (ctx->code) {
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
@ -578,22 +504,21 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
res->node.resType = node->node.resType; res->node.resType = node->node.resType;
if (IS_VAR_DATA_TYPE(output.type)) { int32_t type = output.columnData->info.type;
res->datum.p = output.data; if (IS_VAR_DATA_TYPE(type)) { // todo refactor
output.data = NULL; res->datum.p = output.columnData->pData;
output.columnData->pData = NULL;
} else { } else {
memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes);
} }
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
*pNode = (SNode*)res; *pNode = (SNode*)res;
sclFreeParam(&output); sclFreeParam(&output);
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
@ -614,13 +539,10 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) {
} }
sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT; ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) {
SFunctionNode *node = (SFunctionNode *)pNode; SFunctionNode *node = (SFunctionNode *)pNode;
SScalarParam output = {0}; SScalarParam output = {0};
@ -638,7 +560,6 @@ EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
EDealRes sclWalkLogic(SNode* pNode, SScalarCtx *ctx) { EDealRes sclWalkLogic(SNode* pNode, SScalarCtx *ctx) {
SLogicConditionNode *node = (SLogicConditionNode *)pNode; SLogicConditionNode *node = (SLogicConditionNode *)pNode;
SScalarParam output = {0}; SScalarParam output = {0};
@ -656,7 +577,6 @@ EDealRes sclWalkLogic(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)pNode; SOperatorNode *node = (SOperatorNode *)pNode;
SScalarParam output = {0}; SScalarParam output = {0};
@ -699,27 +619,26 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
for (int32_t i = 0; i < res->num; ++i) { for (int32_t i = 0; i < res->numOfRows; ++i) {
sclMoveParamListData(res, 1, i); if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) {
colDataAppend(col, i, NULL, true);
colDataAppend(col, i, res->data, sclIsNull(res, i)); } else {
char *p = colDataGetData(res->columnData, i);
colDataAppend(col, i, p, false);
}
} }
sclFreeParam(res); sclFreeParam(res);
taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
EDealRes sclCalcWalker(SNode* pNode, void* pContext) { EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
SScalarCtx *ctx = (SScalarCtx *)pContext; SScalarCtx *ctx = (SScalarCtx *)pContext;
if (QUERY_NODE_FUNCTION == nodeType(pNode)) { if (QUERY_NODE_FUNCTION == nodeType(pNode)) {
return sclWalkFunction(pNode, ctx); return sclWalkFunction(pNode, ctx);
} }
@ -737,14 +656,10 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
} }
sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT; ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) {
if (NULL == pNode) { if (NULL == pNode) {
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
@ -759,15 +674,11 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) {
} }
nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx);
SCL_ERR_JRET(ctx.code); SCL_ERR_JRET(ctx.code);
*pRes = pNode; *pRes = pNode;
_return: _return:
sclFreeRes(ctx.pRes); sclFreeRes(ctx.pRes);
return code; return code;
} }
@ -786,7 +697,6 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
} }
nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx);
SCL_ERR_JRET(ctx.code); SCL_ERR_JRET(ctx.code);
if (pDst) { if (pDst) {
@ -796,18 +706,14 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
sclMoveParamListData(res, 1, 0); colDataAssign(pDst->columnData, res->columnData, res->numOfRows);
pDst->numOfRows = res->numOfRows;
*pDst = *res;
taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
} }
_return: _return:
//nodesDestroyNode(pNode); //nodesDestroyNode(pNode);
sclFreeRes(ctx.pRes); sclFreeRes(ctx.pRes);
return code; return code;
} }

701
source/libs/scalar/src/sclfunc.c
View File

@ -1,106 +1,118 @@
#include "sclfunc.h" #include "sclfunc.h"
#include <common/tdatablock.h>
#include "sclInt.h"
#include "sclvector.h" #include "sclvector.h"
static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) {
dst->type = src->type; // dst->type = src->type;
dst->bytes = src->bytes; // dst->bytes = src->bytes;
//dst->num = src->num; // dst->num = src->num;
} }
/** Math functions **/ /** Math functions **/
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
assignBasicParaInfo(pOutput, pInput); SColumnInfoData *pInputData = pInput->columnData;
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { SColumnInfoData *pOutputData = pOutput->columnData;
int32_t type = GET_PARAM_TYPE(pInput);
if (!IS_NUMERIC_TYPE(type)) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
char *input = NULL, *output = NULL; switch (type) {
for (int32_t i = 0; i < pOutput->num; ++i) { case TSDB_DATA_TYPE_FLOAT: {
if (pInput->num == 1) { float *in = (float *)pInputData->pData;
input = pInput->data; float *out = (float *)pOutputData->pData;
} else { for (int32_t i = 0; i < pInput->numOfRows; ++i) {
input = pInput->data + i * pInput->bytes; if (colDataIsNull_f(pInputData->nullbitmap, i)) {
} colDataSetNull_f(pOutputData->nullbitmap, i);
output = pOutput->data + i * pOutput->bytes; continue;
}
if (isNull(input, pInput->type)) { out[i] = (in[i] > 0)? in[i] : -in[i];
setNull(output, pOutput->type, pOutput->bytes); }
continue; break;
} }
switch (pInput->type) { case TSDB_DATA_TYPE_DOUBLE: {
case TSDB_DATA_TYPE_FLOAT: { double *in = (double *)pInputData->pData;
float v; double *out = (double *)pOutputData->pData;
GET_TYPED_DATA(v, float, pInput->type, input); for (int32_t i = 0; i < pInput->numOfRows; ++i) {
float result; if (colDataIsNull_f(pInputData->nullbitmap, i)) {
result = (v > 0) ? v : -v; colDataSetNull_f(pOutputData->nullbitmap, i);
SET_TYPED_DATA(output, pOutput->type, result); continue;
break; }
out[i] = (in[i] > 0)? in[i] : -in[i];
} }
break;
}
case TSDB_DATA_TYPE_DOUBLE: { case TSDB_DATA_TYPE_TINYINT: {
double v; int8_t *in = (int8_t *)pInputData->pData;
GET_TYPED_DATA(v, double, pInput->type, input); int8_t *out = (int8_t *)pOutputData->pData;
double result; for (int32_t i = 0; i < pInput->numOfRows; ++i) {
result = (v > 0) ? v : -v; if (colDataIsNull_f(pInputData->nullbitmap, i)) {
SET_TYPED_DATA(output, pOutput->type, result); colDataSetNull_f(pOutputData->nullbitmap, i);
break; continue;
}
out[i] = (in[i] > 0)? in[i] : -in[i];
} }
break;
}
case TSDB_DATA_TYPE_TINYINT: { case TSDB_DATA_TYPE_SMALLINT: {
int8_t v; int16_t *in = (int16_t *)pInputData->pData;
GET_TYPED_DATA(v, int8_t, pInput->type, input); int16_t *out = (int16_t *)pOutputData->pData;
int8_t result; for (int32_t i = 0; i < pInput->numOfRows; ++i) {
result = (v > 0) ? v : -v; if (colDataIsNull_f(pInputData->nullbitmap, i)) {
SET_TYPED_DATA(output, pOutput->type, result); colDataSetNull_f(pOutputData->nullbitmap, i);
break; continue;
}
out[i] = (in[i] > 0)? in[i] : -in[i];
} }
break;
}
case TSDB_DATA_TYPE_SMALLINT: { case TSDB_DATA_TYPE_INT: {
int16_t v; int32_t *in = (int32_t *)pInputData->pData;
GET_TYPED_DATA(v, int16_t, pInput->type, input); int32_t *out = (int32_t *)pOutputData->pData;
int16_t result; for (int32_t i = 0; i < pInput->numOfRows; ++i) {
result = (v > 0) ? v : -v; if (colDataIsNull_f(pInputData->nullbitmap, i)) {
SET_TYPED_DATA(output, pOutput->type, result); colDataSetNull_f(pOutputData->nullbitmap, i);
break; continue;
}
out[i] = (in[i] > 0)? in[i] : -in[i];
} }
break;
}
case TSDB_DATA_TYPE_INT: { case TSDB_DATA_TYPE_BIGINT: {
int32_t v; int64_t *in = (int64_t *)pInputData->pData;
GET_TYPED_DATA(v, int32_t, pInput->type, input); int64_t *out = (int64_t *)pOutputData->pData;
int32_t result; for (int32_t i = 0; i < pInput->numOfRows; ++i) {
result = (v > 0) ? v : -v; if (colDataIsNull_f(pInputData->nullbitmap, i)) {
SET_TYPED_DATA(output, pOutput->type, result); colDataSetNull_f(pOutputData->nullbitmap, i);
break; continue;
}
out[i] = (in[i] > 0)? in[i] : -in[i];
} }
break;
}
case TSDB_DATA_TYPE_BIGINT: { default: {
int64_t v; colDataAssign(pOutputData, pInputData, pInput->numOfRows);
GET_TYPED_DATA(v, int64_t, pInput->type, input);
int64_t result;
result = (v > 0) ? v : -v;
SET_TYPED_DATA(output, pOutput->type, result);
break;
}
default: {
memcpy(output, input, pInput->bytes);
break;
}
} }
} }
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
#if 0
if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) { if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
pOutput->type = TSDB_DATA_TYPE_DOUBLE;
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
char **input = NULL, *output = NULL; char **input = NULL, *output = NULL;
bool hasNullInput = false; bool hasNullInput = false;
input = taosMemoryCalloc(inputNum, sizeof(char *)); input = taosMemoryCalloc(inputNum, sizeof(char *));
@ -132,11 +144,13 @@ int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
} }
taosMemoryFree(input); taosMemoryFree(input);
#endif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
#if 0
if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) { if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -175,381 +189,140 @@ int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
} }
taosMemoryFree(input); taosMemoryFree(input);
#endif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { typedef float (*_float_fn)(float);
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { typedef double (*_double_fn)(double);
int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) {
int32_t type = GET_PARAM_TYPE(pInput);
if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
pOutput->type = TSDB_DATA_TYPE_DOUBLE; SColumnInfoData *pInputData = pInput->columnData;
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; SColumnInfoData *pOutputData = pOutput->columnData;
char *input = NULL, *output = NULL; _getDoubleValue_fn_t getValueFn = getVectorDoubleValueFn(type);
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) { double *out = (double *)pOutputData->pData;
setNull(output, pOutput->type, pOutput->bytes);
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
continue; continue;
} }
out[i] = valFn(getValueFn(pInputData->pData, i));
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = sqrt(v);
SET_TYPED_DATA(output, pOutput->type, result);
} }
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t sinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _float_fn f1, _double_fn d1) {
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { int32_t type = GET_PARAM_TYPE(pInput);
if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
pOutput->type = TSDB_DATA_TYPE_DOUBLE; SColumnInfoData *pInputData = pInput->columnData;
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; SColumnInfoData *pOutputData = pOutput->columnData;
char *input = NULL, *output = NULL; switch (type) {
for (int32_t i = 0; i < pOutput->num; ++i) { case TSDB_DATA_TYPE_FLOAT: {
if (pInput->num == 1) { float *in = (float *)pInputData->pData;
input = pInput->data; float *out = (float *)pOutputData->pData;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) { for (int32_t i = 0; i < pInput->numOfRows; ++i) {
setNull(output, pOutput->type, pOutput->bytes); if (colDataIsNull_f(pInputData->nullbitmap, i)) {
continue; colDataSetNull_f(pOutputData->nullbitmap, i);
continue;
}
out[i] = f1(in[i]);
}
break;
} }
double v; case TSDB_DATA_TYPE_DOUBLE: {
GET_TYPED_DATA(v, double, pInput->type, input); double *in = (double *)pInputData->pData;
double result = sin(v); double *out = (double *)pOutputData->pData;
SET_TYPED_DATA(output, pOutput->type, result);
} for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
return TSDB_CODE_SUCCESS; colDataSetNull_f(pOutputData->nullbitmap, i);
} continue;
}
int32_t cosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { out[i] = d1(in[i]);
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { }
return TSDB_CODE_FAILED; break;
} }
pOutput->type = TSDB_DATA_TYPE_DOUBLE; default: {
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; colDataAssign(pOutputData, pInputData, pInput->numOfRows);
}
char *input = NULL, *output = NULL;
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) {
setNull(output, pOutput->type, pOutput->bytes);
continue;
}
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = cos(v);
SET_TYPED_DATA(output, pOutput->type, result);
}
return TSDB_CODE_SUCCESS;
}
int32_t tanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) {
return TSDB_CODE_FAILED;
}
pOutput->type = TSDB_DATA_TYPE_DOUBLE;
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
char *input = NULL, *output = NULL;
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) {
setNull(output, pOutput->type, pOutput->bytes);
continue;
}
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = tan(v);
SET_TYPED_DATA(output, pOutput->type, result);
}
return TSDB_CODE_SUCCESS;
}
int32_t asinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) {
return TSDB_CODE_FAILED;
}
pOutput->type = TSDB_DATA_TYPE_DOUBLE;
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
char *input = NULL, *output = NULL;
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) {
setNull(output, pOutput->type, pOutput->bytes);
continue;
}
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = asin(v);
SET_TYPED_DATA(output, pOutput->type, result);
}
return TSDB_CODE_SUCCESS;
}
int32_t acosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) {
return TSDB_CODE_FAILED;
}
pOutput->type = TSDB_DATA_TYPE_DOUBLE;
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
char *input = NULL, *output = NULL;
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) {
setNull(output, pOutput->type, pOutput->bytes);
continue;
}
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = acos(v);
SET_TYPED_DATA(output, pOutput->type, result);
} }
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan);
return TSDB_CODE_FAILED; }
}
pOutput->type = TSDB_DATA_TYPE_DOUBLE; int32_t sinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; return doScalarFunctionUnique(pInput, inputNum, pOutput, sin);
}
char *input = NULL, *output = NULL; int32_t cosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
for (int32_t i = 0; i < pOutput->num; ++i) { return doScalarFunctionUnique(pInput, inputNum, pOutput, cos);
if (pInput->num == 1) { }
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) { int32_t tanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
setNull(output, pOutput->type, pOutput->bytes); return doScalarFunctionUnique(pInput, inputNum, pOutput, tan);
continue; }
}
double v; int32_t asinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
GET_TYPED_DATA(v, double, pInput->type, input); return doScalarFunctionUnique(pInput, inputNum, pOutput, asin);
double result = atan(v); }
SET_TYPED_DATA(output, pOutput->type, result);
}
return TSDB_CODE_SUCCESS; int32_t acosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doScalarFunctionUnique(pInput, inputNum, pOutput, acos);
}
int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doScalarFunctionUnique(pInput, inputNum, pOutput, sqrt);
} }
int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { return doScalarFunction(pInput, inputNum, pOutput, ceilf, ceil);
return TSDB_CODE_FAILED;
}
char *input = NULL, *output = NULL;
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) {
setNull(output, pOutput->type, pOutput->bytes);
continue;
}
switch (pInput->type) {
case TSDB_DATA_TYPE_FLOAT: {
float v;
GET_TYPED_DATA(v, float, pInput->type, input);
float result = ceilf(v);
SET_TYPED_DATA(output, pOutput->type, result);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = ceil(v);
SET_TYPED_DATA(output, pOutput->type, result);
break;
}
default: {
memcpy(output, input, pInput->bytes);
break;
}
}
}
return TSDB_CODE_SUCCESS;
} }
int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
assignBasicParaInfo(pOutput, pInput); return doScalarFunction(pInput, inputNum, pOutput, floorf, floor);
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) {
return TSDB_CODE_FAILED;
}
char *input = NULL, *output = NULL;
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) {
setNull(output, pOutput->type, pOutput->bytes);
continue;
}
switch (pInput->type) {
case TSDB_DATA_TYPE_FLOAT: {
float v;
GET_TYPED_DATA(v, float, pInput->type, input);
float result = floorf(v);
SET_TYPED_DATA(output, pOutput->type, result);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = floor(v);
SET_TYPED_DATA(output, pOutput->type, result);
break;
}
default: {
memcpy(output, input, pInput->bytes);
break;
}
}
}
return TSDB_CODE_SUCCESS;
} }
int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
assignBasicParaInfo(pOutput, pInput); return doScalarFunction(pInput, inputNum, pOutput, roundf, round);
if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) {
return TSDB_CODE_FAILED;
}
char *input = NULL, *output = NULL;
for (int32_t i = 0; i < pOutput->num; ++i) {
if (pInput->num == 1) {
input = pInput->data;
} else {
input = pInput->data + i * pInput->bytes;
}
output = pOutput->data + i * pOutput->bytes;
if (isNull(input, pInput->type)) {
setNull(output, pOutput->type, pOutput->bytes);
continue;
}
switch (pInput->type) {
case TSDB_DATA_TYPE_FLOAT: {
float v;
GET_TYPED_DATA(v, float, pInput->type, input);
float result = roundf(v);
SET_TYPED_DATA(output, pOutput->type, result);
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double v;
GET_TYPED_DATA(v, double, pInput->type, input);
double result = round(v);
SET_TYPED_DATA(output, pOutput->type, result);
break;
}
default: {
memcpy(output, input, pInput->bytes);
break;
}
}
}
return TSDB_CODE_SUCCESS;
} }
static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
assert(numOfInput == 1); assert(numOfInput == 1);
#if 0
int64_t* out = (int64_t*) pOutput->data; int64_t* out = (int64_t*) pOutput->data;
char* s = pLeft->data; char* s = pLeft->data;
for(int32_t i = 0; i < pLeft->num; ++i) { for(int32_t i = 0; i < pLeft->num; ++i) {
out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes)); out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes));
} }
#endif
} }
static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
assert(numOfInput > 0); assert(numOfInput > 0);
#if 0
int32_t rowLen = 0; int32_t rowLen = 0;
int32_t num = 1; int32_t num = 1;
for(int32_t i = 0; i < numOfInput; ++i) { for(int32_t i = 0; i < numOfInput; ++i) {
@ -577,6 +350,7 @@ static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam
rstart += rowLen; rstart += rowLen;
} }
#endif
} }
static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
@ -652,154 +426,3 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf
} }
} }
static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) {
param->bytes = bytes;
param->type = type;
param->num = numOfRows;
param->data = pInput;
}
#if 0
int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param,
char* (*getSourceDataBlock)(void*, const char*, int32_t)) {
if (pExprs == NULL) {
return 0;
}
tExprNode* pLeft = pExprs->_node.pLeft;
tExprNode* pRight = pExprs->_node.pRight;
/* the left output has result from the left child syntax tree */
SScalarFuncParam leftOutput = {0};
SScalarFuncParam rightOutput = {0};
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) {
leftOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows);
evaluateExprNodeTree(pLeft, numOfRows, &leftOutput, param, getSourceDataBlock);
}
// the right output has result from the right child syntax tree
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) {
rightOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows);
evaluateExprNodeTree(pRight, numOfRows, &rightOutput, param, getSourceDataBlock);
}
if (pExprs->nodeType == TEXPR_BINARYEXPR_NODE) {
_bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(pExprs->_node.optr);
SScalarFuncParam left = {0}, right = {0};
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) {
setScalarFuncParam(&left, leftOutput.type, leftOutput.bytes, leftOutput.data, leftOutput.num);
} else if (pLeft->nodeType == TEXPR_COL_NODE) {
SSchema* pschema = pLeft->pSchema;
char* pLeftInputData = getSourceDataBlock(param, pschema->name, pschema->colId);
setScalarFuncParam(&right, pschema->type, pschema->bytes, pLeftInputData, numOfRows);
} else if (pLeft->nodeType == TEXPR_VALUE_NODE) {
SVariant* pVar = pRight->pVal;
setScalarFuncParam(&left, pVar->nType, pVar->nLen, &pVar->i, 1);
}
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) {
setScalarFuncParam(&right, rightOutput.type, rightOutput.bytes, rightOutput.data, rightOutput.num);
} else if (pRight->nodeType == TEXPR_COL_NODE) { // exprLeft + columnRight
SSchema* pschema = pRight->pSchema;
char* pInputData = getSourceDataBlock(param, pschema->name, pschema->colId);
setScalarFuncParam(&right, pschema->type, pschema->bytes, pInputData, numOfRows);
} else if (pRight->nodeType == TEXPR_VALUE_NODE) { // exprLeft + 12
SVariant* pVar = pRight->pVal;
setScalarFuncParam(&right, pVar->nType, pVar->nLen, &pVar->i, 1);
}
void* outputBuf = pOutput->data;
if (isStringOp(pExprs->_node.optr)) {
outputBuf = taosMemoryRealloc(pOutput->data, (left.bytes + right.bytes) * left.num);
}
OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC);
// Set the result info
setScalarFuncParam(pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double), outputBuf, numOfRows);
} else if (pExprs->nodeType == TEXPR_UNARYEXPR_NODE) {
_unary_scalar_fn_t OperatorFn = getUnaryScalarOperatorFn(pExprs->_node.optr);
SScalarFuncParam left = {0};
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) {
setScalarFuncParam(&left, leftOutput.type, leftOutput.bytes, leftOutput.data, leftOutput.num);
} else if (pLeft->nodeType == TEXPR_COL_NODE) {
SSchema* pschema = pLeft->pSchema;
char* pLeftInputData = getSourceDataBlock(param, pschema->name, pschema->colId);
setScalarFuncParam(&left, pschema->type, pschema->bytes, pLeftInputData, numOfRows);
} else if (pLeft->nodeType == TEXPR_VALUE_NODE) {
SVariant* pVar = pLeft->pVal;
setScalarFuncParam(&left, pVar->nType, pVar->nLen, &pVar->i, 1);
}
// reserve enough memory buffer
if (isBinaryStringOp(pExprs->_node.optr)) {
void* outputBuf = taosMemoryRealloc(pOutput->data, left.bytes * left.num);
assert(outputBuf != NULL);
pOutput->data = outputBuf;
}
OperatorFn(&left, pOutput);
}
taosMemoryFreeClear(leftOutput.data);
taosMemoryFreeClear(rightOutput.data);
return 0;
}
#endif
//SScalarFunctionInfo scalarFunc[8] = {
// {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil},
// {"floor", FUNCTION_TYPE_SCALAR, FUNCTION_FLOOR, tfloor},
// {"abs", FUNCTION_TYPE_SCALAR, FUNCTION_ABS, _tabs},
// {"round", FUNCTION_TYPE_SCALAR, FUNCTION_ROUND, tround},
// {"length", FUNCTION_TYPE_SCALAR, FUNCTION_LENGTH, tlength},
// {"concat", FUNCTION_TYPE_SCALAR, FUNCTION_CONCAT, tconcat},
// {"ltrim", FUNCTION_TYPE_SCALAR, FUNCTION_LTRIM, tltrim},
// {"rtrim", FUNCTION_TYPE_SCALAR, FUNCTION_RTRIM, trtrim},
//};
void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) {
sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols;
sas->pExprInfo = pExprInfo;
if (sas->colList != NULL) {
return;
}
sas->colList = taosMemoryCalloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo));
for(int32_t i = 0; i < sas->numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i);
sas->colList[i] = pColData->info;
}
sas->data = taosMemoryCalloc(sas->numOfCols, POINTER_BYTES);
// set the input column data
for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) {
SColumnInfoData *pColumnInfoData = taosArrayGet(pSDataBlock->pDataBlock, f);
sas->data[f] = pColumnInfoData->pData;
}
}
SScalarFunctionSupport* createScalarFuncSupport(int32_t num) {
SScalarFunctionSupport* pSupp = taosMemoryCalloc(num, sizeof(SScalarFunctionSupport));
return pSupp;
}
void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num) {
if (pSupport == NULL) {
return;
}
for(int32_t i = 0; i < num; ++i) {
SScalarFunctionSupport* pSupp = &pSupport[i];
taosMemoryFreeClear(pSupp->data);
taosMemoryFreeClear(pSupp->colList);
}
taosMemoryFreeClear(pSupport);
}

1660
source/libs/scalar/src/sclvector.c
View File

File diff suppressed because it is too large Load Diff

541
source/libs/scalar/test/scalar/scalarTests.cpp
View File

File diff suppressed because it is too large Load Diff

2
source/libs/tdb/CMakeLists.txt
View File

@ -8,6 +8,8 @@ target_sources(tdb
"src/db/tdbBtree.c" "src/db/tdbBtree.c"
"src/db/tdbDb.c" "src/db/tdbDb.c"
"src/db/tdbEnv.c" "src/db/tdbEnv.c"
"src/db/tdbTxn.c"
"src/db/tdbOs.c"
"src/page/tdbPage.c" "src/page/tdbPage.c"
"src/page/tdbPageL.c" "src/page/tdbPageL.c"
) )

37
source/libs/tdb/inc/tdb.h
View File

@ -22,43 +22,6 @@
extern "C" { extern "C" {
#endif #endif
// typedef struct STDb TDB;
// typedef struct STDbEnv TENV;
// typedef struct STDbCurosr TDBC;
// typedef int32_t pgsz_t;
// typedef int32_t cachesz_t;
// typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
// // TEVN
// int tdbEnvCreate(TENV **ppEnv, const char *rootDir);
// int tdbEnvOpen(TENV *ppEnv);
// int tdbEnvClose(TENV *pEnv);
// int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize);
// pgsz_t tdbEnvGetPageSize(TENV *pEnv);
// cachesz_t tdbEnvGetCacheSize(TENV *pEnv);
// int tdbEnvBeginTxn(TENV *pEnv);
// int tdbEnvCommit(TENV *pEnv);
// // TDB
// int tdbCreate(TDB **ppDb);
// int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv);
// int tdbClose(TDB *pDb);
// int tdbDrop(TDB *pDb);
// int tdbSetKeyLen(TDB *pDb, int klen);
// int tdbSetValLen(TDB *pDb, int vlen);
// int tdbSetDup(TDB *pDb, int dup);
// int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn);
// int tdbGetKeyLen(TDB *pDb);
// int tdbGetValLen(TDB *pDb);
// int tdbGetDup(TDB *pDb);
// int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

126
source/libs/tdb/src/db/tdbBtree.c
View File

@ -67,7 +67,7 @@ typedef struct {
u8 *pTmpSpace; u8 *pTmpSpace;
} SCellDecoder; } SCellDecoder;
static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst); static int tdbBtCursorMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst);
static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2);
static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeOpenImpl(SBTree *pBt);
static int tdbBtreeZeroPage(SPage *pPage, void *arg); static int tdbBtreeZeroPage(SPage *pPage, void *arg);
@ -75,10 +75,10 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg);
static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell,
int *szCell); int *szCell);
static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder);
static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeBalance(SBTC *pBtc);
static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell);
static int tdbBtcMoveToNext(SBTC *pBtc); static int tdbBtcMoveToNext(SBTC *pBtc);
static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno); static int tdbBtcMoveDownward(SBTC *pBtc, SPgno pgno);
static int tdbBtcMoveUpward(SBTC *pBtc); static int tdbBtcMoveUpward(SBTC *pBtc);
int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) {
@ -87,7 +87,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S
*ppBt = NULL; *ppBt = NULL;
pBt = (SBTree *)calloc(1, sizeof(*pBt)); pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt));
if (pBt == NULL) { if (pBt == NULL) {
return -1; return -1;
} }
@ -121,7 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S
// TODO: pBt->root // TODO: pBt->root
ret = tdbBtreeOpenImpl(pBt); ret = tdbBtreeOpenImpl(pBt);
if (ret < 0) { if (ret < 0) {
free(pBt); tdbOsFree(pBt);
return -1; return -1;
} }
@ -134,7 +134,7 @@ int tdbBtreeClose(SBTree *pBt) {
return 0; return 0;
} }
int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int tdbBtCursorInsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) {
int ret; int ret;
int idx; int idx;
SPager *pPager; SPager *pPager;
@ -143,20 +143,20 @@ int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal,
int cret; int cret;
SBTree *pBt; SBTree *pBt;
ret = tdbBtCursorMoveTo(pCur, pKey, kLen, &cret); ret = tdbBtCursorMoveTo(pBtc, pKey, kLen, &cret);
if (ret < 0) { if (ret < 0) {
// TODO: handle error // TODO: handle error
return -1; return -1;
} }
if (pCur->idx == -1) { if (pBtc->idx == -1) {
ASSERT(TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0); ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0);
idx = 0; idx = 0;
} else { } else {
if (cret > 0) { if (cret > 0) {
idx = pCur->idx + 1; idx = pBtc->idx + 1;
} else if (cret < 0) { } else if (cret < 0) {
idx = pCur->idx; idx = pBtc->idx;
} else { } else {
/* TODO */ /* TODO */
ASSERT(0); ASSERT(0);
@ -164,9 +164,9 @@ int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal,
} }
// TODO: refact code here // TODO: refact code here
pBt = pCur->pBt; pBt = pBtc->pBt;
if (!pBt->pTmp) { if (!pBt->pTmp) {
pBt->pTmp = (u8 *)malloc(pBt->pageSize); pBt->pTmp = (u8 *)tdbOsMalloc(pBt->pageSize);
if (pBt->pTmp == NULL) { if (pBt->pTmp == NULL) {
return -1; return -1;
} }
@ -175,20 +175,20 @@ int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal,
pCell = pBt->pTmp; pCell = pBt->pTmp;
// Encode the cell // Encode the cell
ret = tdbBtreeEncodeCell(pCur->pPage, pKey, kLen, pVal, vLen, pCell, &szCell); ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pVal, vLen, pCell, &szCell);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
// Insert the cell to the index // Insert the cell to the index
ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell, 0); ret = tdbPageInsertCell(pBtc->pPage, idx, pCell, szCell, 0);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
// If page is overflow, balance the tree // If page is overflow, balance the tree
if (pCur->pPage->nOverflow > 0) { if (pBtc->pPage->nOverflow > 0) {
ret = tdbBtreeBalance(pCur); ret = tdbBtreeBalance(pBtc);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -226,30 +226,30 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen
return 0; return 0;
} }
static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { static int tdbBtCursorMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
int ret; int ret;
SBTree *pBt; SBTree *pBt;
SPager *pPager; SPager *pPager;
pBt = pCur->pBt; pBt = pBtc->pBt;
pPager = pBt->pPager; pPager = pBt->pPager;
if (pCur->iPage < 0) { if (pBtc->iPage < 0) {
ASSERT(pCur->iPage == -1); ASSERT(pBtc->iPage == -1);
ASSERT(pCur->idx == -1); ASSERT(pBtc->idx == -1);
// Move from the root // Move from the root
ret = tdbPagerFetchPage(pPager, pBt->root, &(pCur->pPage), tdbBtreeInitPage, pBt); ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
pCur->iPage = 0; pBtc->iPage = 0;
if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) {
// Current page is empty // Current page is empty
// ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pBtc->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF));
return 0; return 0;
} }
@ -259,7 +259,7 @@ static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst)
SPage *pPage; SPage *pPage;
SCellDecoder cd = {0}; SCellDecoder cd = {0};
pPage = pCur->pPage; pPage = pBtc->pPage;
nCells = TDB_PAGE_TOTAL_CELLS(pPage); nCells = TDB_PAGE_TOTAL_CELLS(pPage);
lidx = 0; lidx = 0;
ridx = nCells - 1; ridx = nCells - 1;
@ -297,22 +297,22 @@ static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst)
u8 flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); u8 flags = TDB_BTREE_PAGE_GET_FLAGS(pPage);
u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags);
if (leaf) { if (leaf) {
pCur->idx = midx; pBtc->idx = midx;
*pCRst = c; *pCRst = c;
break; break;
} else { } else {
if (c <= 0) { if (c <= 0) {
pCur->idx = midx; pBtc->idx = midx;
tdbBtcMoveDownward(pCur, cd.pgno); tdbBtcMoveDownward(pBtc, cd.pgno);
} else { } else {
pCur->idx = midx + 1; pBtc->idx = midx + 1;
if (midx == nCells - 1) { if (midx == nCells - 1) {
/* Move to right-most child */ /* Move to right-most child */
tdbBtcMoveDownward(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); tdbBtcMoveDownward(pBtc, ((SIntHdr *)pBtc->pPage->pData)->pgno);
} else { } else {
pCell = tdbPageGetCell(pPage, pCur->idx); pCell = tdbPageGetCell(pPage, pBtc->idx);
tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtreeDecodeCell(pPage, pCell, &cd);
tdbBtcMoveDownward(pCur, cd.pgno); tdbBtcMoveDownward(pBtc, cd.pgno);
} }
} }
} }
@ -550,7 +550,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) {
pCell = tdbPageGetCell(pParent, sIdx + i); pCell = tdbPageGetCell(pParent, sIdx + i);
szDivCell[i] = tdbBtreeCellSize(pParent, pCell); szDivCell[i] = tdbBtreeCellSize(pParent, pCell);
pDivCell[i] = malloc(szDivCell[i]); pDivCell[i] = tdbOsMalloc(szDivCell[i]);
memcpy(pDivCell[i], pCell, szDivCell[i]); memcpy(pDivCell[i], pCell, szDivCell[i]);
} }
@ -740,13 +740,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtreeDecodeCell(pPage, pCell, &cd);
// TODO: pCell here may be inserted as an overflow cell, handle it // TODO: pCell here may be inserted as an overflow cell, handle it
SCell *pNewCell = malloc(cd.kLen + 9); SCell *pNewCell = tdbOsMalloc(cd.kLen + 9);
int szNewCell; int szNewCell;
SPgno pgno; SPgno pgno;
pgno = TDB_PAGE_PGNO(pNews[iNew]); pgno = TDB_PAGE_PGNO(pNews[iNew]);
tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell);
tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0);
free(pNewCell); tdbOsFree(pNewCell);
} }
// move to next new page // move to next new page
@ -798,14 +798,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
if (pDivCell[i]) { if (pDivCell[i]) {
free(pDivCell[i]); tdbOsFree(pDivCell[i]);
} }
} }
return 0; return 0;
} }
static int tdbBtreeBalance(SBTC *pCur) { static int tdbBtreeBalance(SBTC *pBtc) {
int iPage; int iPage;
SPage *pParent; SPage *pParent;
SPage *pPage; SPage *pPage;
@ -816,8 +816,8 @@ static int tdbBtreeBalance(SBTC *pCur) {
// Main loop to balance the BTree // Main loop to balance the BTree
for (;;) { for (;;) {
iPage = pCur->iPage; iPage = pBtc->iPage;
pPage = pCur->pPage; pPage = pBtc->pPage;
flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); flags = TDB_BTREE_PAGE_GET_FLAGS(pPage);
leaf = TDB_BTREE_PAGE_IS_LEAF(flags); leaf = TDB_BTREE_PAGE_IS_LEAF(flags);
root = TDB_BTREE_PAGE_IS_ROOT(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags);
@ -833,27 +833,27 @@ static int tdbBtreeBalance(SBTC *pCur) {
// ignore the case of empty // ignore the case of empty
if (pPage->nOverflow == 0) break; if (pPage->nOverflow == 0) break;
ret = tdbBtreeBalanceDeeper(pCur->pBt, pPage, &(pCur->pgStack[1])); ret = tdbBtreeBalanceDeeper(pBtc->pBt, pPage, &(pBtc->pgStack[1]));
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
pCur->idx = 0; pBtc->idx = 0;
pCur->idxStack[0] = 0; pBtc->idxStack[0] = 0;
pCur->pgStack[0] = pCur->pPage; pBtc->pgStack[0] = pBtc->pPage;
pCur->iPage = 1; pBtc->iPage = 1;
pCur->pPage = pCur->pgStack[1]; pBtc->pPage = pBtc->pgStack[1];
} else { } else {
// Generalized balance step // Generalized balance step
pParent = pCur->pgStack[iPage - 1]; pParent = pBtc->pgStack[iPage - 1];
ret = tdbBtreeBalanceNonRoot(pCur->pBt, pParent, pCur->idxStack[pCur->iPage - 1]); ret = tdbBtreeBalanceNonRoot(pBtc->pBt, pParent, pBtc->idxStack[pBtc->iPage - 1]);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
pCur->iPage--; pBtc->iPage--;
pCur->pPage = pCur->pgStack[pCur->iPage]; pBtc->pPage = pBtc->pgStack[pBtc->iPage];
} }
} }
@ -1050,11 +1050,11 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) {
#endif #endif
int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { int tdbBtcOpen(SBTC *pBtc, SBTree *pBt) {
pCur->pBt = pBt; pBtc->pBt = pBt;
pCur->iPage = -1; pBtc->iPage = -1;
pCur->pPage = NULL; pBtc->pPage = NULL;
pCur->idx = -1; pBtc->idx = -1;
return 0; return 0;
} }
@ -1262,16 +1262,16 @@ int tdbBtcClose(SBTC *pBtc) {
return 0; return 0;
} }
static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { static int tdbBtcMoveDownward(SBTC *pBtc, SPgno pgno) {
int ret; int ret;
pCur->pgStack[pCur->iPage] = pCur->pPage; pBtc->pgStack[pBtc->iPage] = pBtc->pPage;
pCur->idxStack[pCur->iPage] = pCur->idx; pBtc->idxStack[pBtc->iPage] = pBtc->idx;
pCur->iPage++; pBtc->iPage++;
pCur->pPage = NULL; pBtc->pPage = NULL;
pCur->idx = -1; pBtc->idx = -1;
ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
} }

30
source/libs/tdb/src/db/tdbDb.c
View File

@ -16,7 +16,7 @@
#include "tdbInt.h" #include "tdbInt.h"
struct STDB { struct STDB {
STEnv *pEnv; TENV *pEnv;
SBTree *pBt; SBTree *pBt;
}; };
@ -24,8 +24,8 @@ struct STDBC {
SBTC btc; SBTC btc;
}; };
int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, TENV *pEnv, TDB **ppDb) {
STDB *pDb; TDB *pDb;
SPager *pPager; SPager *pPager;
int ret; int ret;
char fFullName[TDB_FILENAME_LEN]; char fFullName[TDB_FILENAME_LEN];
@ -34,7 +34,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF
*ppDb = NULL; *ppDb = NULL;
pDb = (STDB *)calloc(1, sizeof(*pDb)); pDb = (TDB *)tdbOsCalloc(1, sizeof(*pDb));
if (pDb == NULL) { if (pDb == NULL) {
return -1; return -1;
} }
@ -63,17 +63,17 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF
return 0; return 0;
} }
int tdbDbClose(STDB *pDb) { int tdbDbClose(TDB *pDb) {
// TODO // TODO
return 0; return 0;
} }
int tdbDbDrop(STDB *pDb) { int tdbDbDrop(TDB *pDb) {
// TODO // TODO
return 0; return 0;
} }
int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) {
SBTC btc; SBTC btc;
SBTC *pCur; SBTC *pCur;
int ret; int ret;
@ -92,16 +92,16 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v
return 0; return 0;
} }
int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen);
} }
int tdbDbcOpen(STDB *pDb, STDBC **ppDbc) { int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) {
int ret; int ret;
STDBC *pDbc = NULL; TDBC *pDbc = NULL;
*ppDbc = NULL; *ppDbc = NULL;
pDbc = malloc(sizeof(*pDbc)); pDbc = (TDBC *)tdbOsMalloc(sizeof(*pDbc));
if (pDbc == NULL) { if (pDbc == NULL) {
return -1; return -1;
} }
@ -120,13 +120,13 @@ int tdbDbcOpen(STDB *pDb, STDBC **ppDbc) {
return 0; return 0;
} }
int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen);
} }
int tdbDbcClose(STDBC *pDbc) { int tdbDbcClose(TDBC *pDbc) {
if (pDbc) { if (pDbc) {
free(pDbc); tdbOsFree(pDbc);
} }
return 0; return 0;

20
source/libs/tdb/src/db/tdbEnv.c
View File

@ -15,24 +15,24 @@
#include "tdbInt.h" #include "tdbInt.h"
int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv) { int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv) {
STEnv *pEnv; TENV *pEnv;
int dsize; int dsize;
int zsize; int zsize;
u8 *pPtr; u8 *pPtr;
int ret; int ret;
*ppEnv = NULL; *ppEnv = NULL;
dsize = strlen(rootDir); dsize = strlen(rootDir);
zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
pPtr = (uint8_t *)calloc(1, zsize); pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) { if (pPtr == NULL) {
return -1; return -1;
} }
pEnv = (STEnv *)pPtr; pEnv = (TENV *)pPtr;
pPtr += sizeof(*pEnv); pPtr += sizeof(*pEnv);
// pEnv->rootDir // pEnv->rootDir
pEnv->rootDir = pPtr; pEnv->rootDir = pPtr;
@ -59,12 +59,12 @@ int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv)
return 0; return 0;
} }
int tdbEnvClose(STEnv *pEnv) { int tdbEnvClose(TENV *pEnv) {
// TODO // TODO
return 0; return 0;
} }
SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname) { SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
// TODO // TODO
return NULL; return NULL;
} }

98
source/libs/tdb/src/db/tdbOs.c Normal file
View File

@ -0,0 +1,98 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
#ifndef TDB_FOR_TDENGINE
// tdbOsRead
i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes) {
i64 nRead = 0;
i64 iRead = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = read(fd, pBuf, nBytes);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsPRead
i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset) {
i64 nRead = 0;
i64 iRead = 0;
i64 iOffset = offset;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iRead = pread(fd, pBuf, nBytes, iOffset);
if (iRead < 0) {
if (errno == EINTR) {
continue;
} else {
return -1;
}
} else if (iRead == 0) {
break;
}
nRead += iRead;
pBuf += iRead;
iOffset += iRead;
nBytes -= iRead;
}
return nRead;
}
// tdbOsWrite
i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes) {
i64 nWrite = 0;
i64 iWrite = 0;
u8 *pBuf = (u8 *)pData;
while (nBytes > 0) {
iWrite = write(fd, pBuf, nBytes);
if (iWrite < 0) {
if (errno == EINTR) {
continue;
}
return -1;
}
nWrite += iWrite;
pBuf += iWrite;
nBytes -= iWrite;
}
return nWrite;
}
#endif

34
source/libs/tdb/src/db/tdbPCache.c
View File

@ -15,16 +15,16 @@
#include "tdbInt.h" #include "tdbInt.h"
struct SPCache { struct SPCache {
int pageSize; int pageSize;
int cacheSize; int cacheSize;
pthread_mutex_t mutex; tdb_mutex_t mutex;
int nFree; int nFree;
SPage *pFree; SPage *pFree;
int nPage; int nPage;
int nHash; int nHash;
SPage **pgHash; SPage **pgHash;
int nRecyclable; int nRecyclable;
SPage lru; SPage lru;
}; };
#define PCACHE_PAGE_HASH(pPgid) \ #define PCACHE_PAGE_HASH(pPgid) \
@ -63,7 +63,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
void *pPtr; void *pPtr;
SPage *pPgHdr; SPage *pPgHdr;
pCache = (SPCache *)calloc(1, sizeof(*pCache)); pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache));
if (pCache == NULL) { if (pCache == NULL) {
return -1; return -1;
} }
@ -72,7 +72,7 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
pCache->cacheSize = cacheSize; pCache->cacheSize = cacheSize;
if (tdbPCacheOpenImpl(pCache) < 0) { if (tdbPCacheOpenImpl(pCache) < 0) {
free(pCache); tdbOsFree(pCache);
return -1; return -1;
} }
@ -116,13 +116,13 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
} }
} }
static void tdbPCacheInitLock(SPCache *pCache) { pthread_mutex_init(&(pCache->mutex), NULL); } static void tdbPCacheInitLock(SPCache *pCache) { tdbMutexInit(&(pCache->mutex), NULL); }
static void tdbPCacheClearLock(SPCache *pCache) { pthread_mutex_destroy(&(pCache->mutex)); } static void tdbPCacheClearLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); }
static void tdbPCacheLock(SPCache *pCache) { pthread_mutex_lock(&(pCache->mutex)); } static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
static void tdbPCacheUnlock(SPCache *pCache) { pthread_mutex_unlock(&(pCache->mutex)); } static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); }
static bool tdbPCacheLocked(SPCache *pCache) { static bool tdbPCacheLocked(SPCache *pCache) {
assert(0); assert(0);
@ -268,7 +268,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
// Open the hash table // Open the hash table
pCache->nPage = 0; pCache->nPage = 0;
pCache->nHash = pCache->cacheSize; pCache->nHash = pCache->cacheSize;
pCache->pgHash = (SPage **)calloc(pCache->nHash, sizeof(SPage *)); pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
if (pCache->pgHash == NULL) { if (pCache->pgHash == NULL) {
// TODO // TODO
return -1; return -1;

114
source/libs/tdb/src/db/tdbPager.c
View File

@ -20,14 +20,12 @@ struct SPager {
char *jFileName; char *jFileName;
int pageSize; int pageSize;
uint8_t fid[TDB_FILE_ID_LEN]; uint8_t fid[TDB_FILE_ID_LEN];
int fd; tdb_fd_t fd;
int jfd; tdb_fd_t jfd;
SPCache *pCache; SPCache *pCache;
SPgno dbFileSize; SPgno dbFileSize;
SPgno dbOrigSize; SPgno dbOrigSize;
int nDirty;
SPage *pDirty; SPage *pDirty;
SPage *pDirtyTail;
u8 inTran; u8 inTran;
}; };
@ -46,6 +44,8 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct")
static int tdbPagerReadPage(SPager *pPager, SPage *pPage); static int tdbPagerReadPage(SPager *pPager, SPage *pPage);
static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg); static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg);
static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
uint8_t *pPtr; uint8_t *pPtr;
@ -60,7 +60,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
zsize = sizeof(*pPager) /* SPager */ zsize = sizeof(*pPager) /* SPager */
+ fsize + 1 /* dbFileName */ + fsize + 1 /* dbFileName */
+ fsize + 8 + 1; /* jFileName */ + fsize + 8 + 1; /* jFileName */
pPtr = (uint8_t *)calloc(1, zsize); pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) { if (pPtr == NULL) {
return -1; return -1;
} }
@ -80,7 +80,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
// pPager->pCache // pPager->pCache
pPager->pCache = pCache; pPager->pCache = pCache;
pPager->fd = open(pPager->dbFileName, O_RDWR | O_CREAT, 0755); pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (pPager->fd < 0) { if (pPager->fd < 0) {
return -1; return -1;
} }
@ -90,7 +90,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
return -1; return -1;
} }
pPager->jfd = -1; // pPager->jfd = -1;
pPager->pageSize = tdbPCacheGetPageSize(pCache); pPager->pageSize = tdbPCacheGetPageSize(pCache);
*ppPager = pPager; *ppPager = pPager;
@ -140,14 +140,25 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
} }
} }
if (pPage->isDirty == 0) { if (pPage->isDirty) return 0;
pPage->isDirty = 1;
// TODO: add the page to the dirty list
// TODO: write the page to the journal // Set page as dirty
if (1 /*actually load from the file*/) { pPage->isDirty = 1;
// Add page to dirty list
// TODO: sort the list according to the page number
pPage->pDirtyNext = pPager->pDirty;
pPager->pDirty = pPage;
// Write page to journal
if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
ret = tdbPagerWritePageToJournal(pPager, pPage);
if (ret < 0) {
ASSERT(0);
return -1;
} }
} }
return 0; return 0;
} }
@ -157,7 +168,7 @@ int tdbPagerBegin(SPager *pPager) {
} }
// Open the journal // Open the journal
pPager->jfd = open(pPager->jFileName, O_RDWR | O_CREAT, 0755); pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
if (pPager->jfd < 0) { if (pPager->jfd < 0) {
return -1; return -1;
} }
@ -170,7 +181,37 @@ int tdbPagerBegin(SPager *pPager) {
} }
int tdbPagerCommit(SPager *pPager) { int tdbPagerCommit(SPager *pPager) {
// TODO SPage *pPage;
int ret;
// Begin commit
{
// TODO: Sync the journal file (Here or when write ?)
}
for (;;) {
pPage = pPager->pDirty;
if (pPage == NULL) break;
ret = tdbPagerWritePageToDB(pPager, pPage);
if (ret < 0) {
ASSERT(0);
return -1;
}
pPager->pDirty = pPage->pDirtyNext;
pPage->pDirtyNext = NULL;
// TODO: release the page
}
tdbOsFSync(pPager->fd);
tdbOsClose(pPager->jfd);
tdbOsRemove(pPager->jFileName);
// pPager->jfd = -1;
return 0; return 0;
} }
@ -181,7 +222,7 @@ static int tdbPagerReadPage(SPager *pPager, SPage *pPage) {
ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0); ASSERT(memcmp(pPager->fid, pPage->pgid.fileid, TDB_FILE_ID_LEN) == 0);
offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize); offset = (pPage->pgid.pgno - 1) * (i64)(pPager->pageSize);
ret = tdbPRead(pPager->fd, pPage->pData, pPager->pageSize, offset); ret = tdbOsPRead(pPager->fd, pPage->pData, pPager->pageSize, offset);
if (ret < 0) { if (ret < 0) {
// TODO: handle error // TODO: handle error
return -1; return -1;
@ -255,9 +296,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
return 0; return 0;
} }
void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { tdbPCacheRelease(pPager->pCache, pPage); }
tdbPCacheRelease(pPager->pCache, pPage);
}
static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) {
// TODO: Allocate a page from the free list // TODO: Allocate a page from the free list
@ -328,5 +367,44 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
return -1; return -1;
} }
return 0;
}
// ---------------------------- Journal manipulation
static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
int ret;
SPgno pgno;
pgno = TDB_PAGE_PGNO(pPage);
ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno));
if (ret < 0) {
return -1;
}
ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize);
if (ret < 0) {
return -1;
}
return 0;
}
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
i64 offset;
int ret;
offset = pPage->pageSize * TDB_PAGE_PGNO(pPage);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
ASSERT(0);
return -1;
}
ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize);
if (ret < 0) {
ASSERT(0);
return -1;
}
return 0; return 0;
} }

31
source/libs/tdb/src/db/tdbTxn.c Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
int tdbTxnBegin(TENV *pEnv) {
// TODO
return 0;
}
int tdbTxnCommit(TENV *pEnv) {
// TODO
return 0;
}
int tdbTxnRollback(TENV *pEnv) {
// TODO
return 0;
}

58
source/libs/tdb/src/db/tdbUtil.c
View File

@ -31,62 +31,4 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
} }
return 0; return 0;
}
// int tdbCheckFileAccess(const char *pathname, int mode) {
// int flags = 0;
// if (mode & TDB_F_OK) {
// flags |= F_OK;
// }
// if (mode & TDB_R_OK) {
// flags |= R_OK;
// }
// if (mode & TDB_W_OK) {
// flags |= W_OK;
// }
// return access(pathname, flags);
// }
int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize) {
struct stat st;
int ret;
int64_t file_size = 0;
ret = taosStatFile(fname, &file_size, NULL);
if (ret != 0) {
return -1;
}
ASSERT(file_size % pgSize == 0);
*pSize = file_size / pgSize;
return 0;
}
int tdbPRead(int fd, void *pData, int count, i64 offset) {
void *pBuf;
int nbytes;
i64 ioffset;
int iread;
pBuf = pData;
nbytes = count;
ioffset = offset;
while (nbytes > 0) {
iread = pread(fd, pBuf, nbytes, ioffset);
if (iread < 0) {
/* TODO */
} else if (iread == 0) {
return (count - iread);
}
nbytes = nbytes - iread;
pBuf = (void *)((u8 *)pBuf + iread);
ioffset += iread;
}
return count;
} }

24
source/libs/tdb/src/inc/tdbDb.h
View File

@ -20,20 +20,20 @@
extern "C" { extern "C" {
#endif #endif
typedef struct STDB STDB; typedef struct STDB TDB;
typedef struct STDBC STDBC; typedef struct STDBC TDBC;
// STDB // TDB
int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, TENV *pEnv, TDB **ppDb);
int tdbDbClose(STDB *pDb); int tdbDbClose(TDB *pDb);
int tdbDbDrop(STDB *pDb); int tdbDbDrop(TDB *pDb);
int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen);
int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen);
// STDBC // TDBC
int tdbDbcOpen(STDB *pDb, STDBC **ppDbc); int tdbDbcOpen(TDB *pDb, TDBC **ppDbc);
int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int tdbDbcClose(STDBC *pDbc); int tdbDbcClose(TDBC *pDbc);
#ifdef __cplusplus #ifdef __cplusplus
} }

12
source/libs/tdb/src/inc/tdbEnv.h
View File

@ -21,16 +21,16 @@ extern "C" {
#endif #endif
typedef struct STEnv { typedef struct STEnv {
char * rootDir; char *rootDir;
char * jfname; char *jfname;
int jfd; int jfd;
SPCache *pCache; SPCache *pCache;
} STEnv; } TENV;
int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, STEnv **ppEnv); int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv);
int tdbEnvClose(STEnv *pEnv); int tdbEnvClose(TENV *pEnv);
SPager *tdbEnvGetPager(STEnv *pEnv, const char *fname); SPager *tdbEnvGetPager(TENV *pEnv, const char *fname);
#ifdef __cplusplus #ifdef __cplusplus
} }

38
source/libs/tdb/src/inc/tdbInt.h
View File

@ -16,10 +16,9 @@
#ifndef _TD_TDB_INTERNAL_H_ #ifndef _TD_TDB_INTERNAL_H_
#define _TD_TDB_INTERNAL_H_ #define _TD_TDB_INTERNAL_H_
#include "tlist.h" #include "os.h"
#include "tlockfree.h"
// #include "tdb.h" #include "tdb.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -51,18 +50,18 @@ typedef u32 SPgno;
// fileid // fileid
#define TDB_FILE_ID_LEN 24 #define TDB_FILE_ID_LEN 24
// pgid_t // SPgid
typedef struct { typedef struct {
uint8_t fileid[TDB_FILE_ID_LEN]; uint8_t fileid[TDB_FILE_ID_LEN];
SPgno pgno; SPgno pgno;
} pgid_t, SPgid; } SPgid;
#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; #define TDB_IVLD_PGID (SPgid){0, TDB_IVLD_PGNO};
static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
pgid_t *pgid1 = (pgid_t *)p1; SPgid *pgid1 = (SPgid *)p1;
pgid_t *pgid2 = (pgid_t *)p2; SPgid *pgid2 = (SPgid *)p2;
int rcode; int rcode;
rcode = memcmp(pgid1->fileid, pgid2->fileid, TDB_FILE_ID_LEN); rcode = memcmp(pgid1->fileid, pgid2->fileid, TDB_FILE_ID_LEN);
if (rcode) { if (rcode) {
@ -95,10 +94,6 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
// tdb_log // tdb_log
#define tdbError(var) #define tdbError(var)
typedef TD_DLIST(STDB) STDbList;
typedef TD_DLIST(SPgFile) SPgFileList;
typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
#define TERR_A(val, op, flag) \ #define TERR_A(val, op, flag) \
do { \ do { \
if (((val) = (op)) != 0) { \ if (((val) = (op)) != 0) { \
@ -115,19 +110,6 @@ typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
#define TDB_VARIANT_LEN ((int)-1) #define TDB_VARIANT_LEN ((int)-1)
// page payload format
// <keyLen> + <valLen> + [key] + [value]
#define TDB_DECODE_PAYLOAD(pPayload, keyLen, pKey, valLen, pVal) \
do { \
if ((keyLen) == TDB_VARIANT_LEN) { \
/* TODO: decode the keyLen */ \
} \
if ((valLen) == TDB_VARIANT_LEN) { \
/* TODO: decode the valLen */ \
} \
/* TODO */ \
} while (0)
typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
#define TDB_JOURNAL_NAME "tdb.journal" #define TDB_JOURNAL_NAME "tdb.journal"
@ -148,6 +130,8 @@ typedef struct SPager SPager;
typedef struct SPCache SPCache; typedef struct SPCache SPCache;
typedef struct SPage SPage; typedef struct SPage SPage;
#include "tdbOs.h"
#include "tdbUtil.h" #include "tdbUtil.h"
#include "tdbPCache.h" #include "tdbPCache.h"
@ -162,6 +146,8 @@ typedef struct SPage SPage;
#include "tdbPage.h" #include "tdbPage.h"
#include "tdbTxn.h"
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

136
source/libs/tdb/src/inc/tdbOs.h Normal file
View File

@ -0,0 +1,136 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TDB_OS_H_
#define _TDB_OS_H_
#ifdef __cplusplus
extern "C" {
#endif
// TODO: use cmake to control the option
#define TDB_FOR_TDENGINE
#ifdef TDB_FOR_TDENGINE
// For memory -----------------
#define tdbOsMalloc taosMemoryMalloc
#define tdbOsCalloc taosMemoryCalloc
#define tdbOsRealloc taosMemoryRealloc
#define tdbOsFree taosMemoryFree
// For file and directory -----------------
/* file */
typedef TdFilePtr tdb_fd_t;
#define TDB_O_CREAT TD_FILE_CTEATE
#define TDB_O_WRITE TD_FILE_WRITE
#define TDB_O_READ TD_FILE_READ
#define TDB_O_TRUNC TD_FILE_TRUNC
#define TDB_O_APPEND TD_FILE_APPEND
#define TDB_O_RDWR (TD_FILE_WRITE) | (TD_FILE_READ)
#define tdbOsOpen(PATH, OPTION, MODE) taosOpenFile((PATH), (OPTION))
#define tdbOsClose(FD) taosCloseFile(&(FD))
#define tdbOsRead taosReadFile
#define tdbOsPRead taosPReadFile
#define tdbOsWrite taosWriteFile
#define tdbOsFSync taosFsyncFile
#define tdbOsLSeek taosLSeekFile
#define tdbOsRemove remove
/* directory */
#define tdbOsMkdir taosMkDir
#define tdbOsRmdir taosRemoveDir
// For threads and lock -----------------
/* spin lock */
typedef TdThreadSpinlock tdb_spinlock_t;
#define tdbSpinlockInit taosThreadSpinInit
#define tdbSpinlockDestroy taosThreadSpinDestroy
#define tdbSpinlockLock taosThreadSpinLock
#define tdbSpinlockUnlock taosThreadSpinUnlock
#define tdbSpinlockTrylock pthread_spin_trylock
/* mutex lock */
typedef TdThreadMutex tdb_mutex_t;
#define tdbMutexInit taosThreadMutexInit
#define tdbMutexDestroy taosThreadMutexDestroy
#define tdbMutexLock taosThreadMutexLock
#define tdbMutexUnlock taosThreadMutexUnlock
#else
// For memory -----------------
#define tdbOsMalloc malloc
#define tdbOsCalloc calloc
#define tdbOsRealloc realloc
#define tdbOsFree free
// For file and directory -----------------
/* file */
typedef int tdb_fd_t;
#define TDB_O_CREAT O_CREAT
#define TDB_O_WRITE O_WRONLY
#define TDB_O_READ O_RDONLY
#define TDB_O_TRUNC O_TRUNC
#define TDB_O_APPEND O_APPEND
#define TDB_O_RDWR O_RDWR
#define tdbOsOpen(PATH, OPTION, MODE) open((PATH), (OPTION), (MODE))
#define tdbOsClose close
i64 tdbOsRead(tdb_fd_t fd, void *pData, i64 nBytes);
i64 tdbOsPRead(tdb_fd_t fd, void *pData, i64 nBytes, i64 offset);
i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes);
#define tdbOsFSync fsync
#define tdbOsLSeek lseek
#define tdbOsRemove remove
/* directory */
#define tdbOsMkdir mkdir
#define tdbOsRmdir rmdir
// For threads and lock -----------------
/* spin lock */
typedef pthread_spinlock_t tdb_spinlock_t;
#define tdbSpinlockInit pthread_spin_init
#define tdbSpinlockDestroy pthread_spin_destroy
#define tdbSpinlockLock pthread_spin_lock
#define tdbSpinlockUnlock pthread_spin_unlock
#define tdbSpinlockTrylock pthread_spin_trylock
/* mutex lock */
typedef pthread_mutex_t tdb_mutex_t;
#define tdbMutexInit pthread_mutex_init
#define tdbMutexDestroy pthread_mutex_destroy
#define tdbMutexLock pthread_mutex_lock
#define tdbMutexUnlock pthread_mutex_unlock
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TDB_OS_H_*/

38
source/libs/tdb/src/inc/tdbPage.h
View File

@ -53,10 +53,10 @@ typedef struct __attribute__((__packed__)) {
} SPageFtr; } SPageFtr;
struct SPage { struct SPage {
pthread_spinlock_t lock; tdb_spinlock_t lock;
int pageSize; int pageSize;
u8 *pData; u8 *pData;
SPageMethods *pPageMethods; SPageMethods *pPageMethods;
// Fields below used by pager and am // Fields below used by pager and am
u8 *pPageHdr; u8 *pPageHdr;
u8 *pCellIdx; u8 *pCellIdx;
@ -80,21 +80,21 @@ struct SPage {
#define P_LOCK_BUSY 1 #define P_LOCK_BUSY 1
#define P_LOCK_FAIL -1 #define P_LOCK_FAIL -1
#define TDB_INIT_PAGE_LOCK(pPage) pthread_spin_init(&((pPage)->lock), 0) #define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0)
#define TDB_DESTROY_PAGE_LOCK(pPage) pthread_spin_destroy(&((pPage)->lock)) #define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock))
#define TDB_LOCK_PAGE(pPage) pthread_spin_lock(&((pPage)->lock)) #define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock))
#define TDB_UNLOCK_PAGE(pPage) pthread_spin_unlock(&((pPage)->lock)) #define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock))
#define TDB_TRY_LOCK_PAGE(pPage) \ #define TDB_TRY_LOCK_PAGE(pPage) \
({ \ ({ \
int ret; \ int ret; \
if (pthread_spin_trylock(&((pPage)->lock)) == 0) { \ if (tdbSpinlockTrylock(&((pPage)->lock)) == 0) { \
ret = P_LOCK_SUCC; \ ret = P_LOCK_SUCC; \
} else if (errno == EBUSY) { \ } else if (errno == EBUSY) { \
ret = P_LOCK_BUSY; \ ret = P_LOCK_BUSY; \
} else { \ } else { \
ret = P_LOCK_FAIL; \ ret = P_LOCK_FAIL; \
} \ } \
ret; \ ret; \
}) })
// APIs // APIs

39
source/libs/tdb/src/inc/tdbTxn.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TDB_TXN_H_
#define _TDB_TXN_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef struct STxn STXN;
struct STxn {
u64 txnId;
void *(*xMalloc)(void *, int);
void *xArg;
};
int tdbTxnBegin(TENV *pEnv);
int tdbTxnCommit(TENV *pEnv);
int tdbTxnRollback(TENV *pEnv);
#ifdef __cplusplus
}
#endif
#endif /*_TDB_TXN_H_*/

51
source/libs/tdb/src/inc/tdbUtil.h
View File

@ -30,46 +30,37 @@ extern "C" {
int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
// #define TDB_F_OK 0x1 #define TDB_REALLOC(PTR, SIZE) \
// #define TDB_R_OK 0x2 ({ \
// #define TDB_W_OK 0x4 void *nPtr; \
// int tdbCheckFileAccess(const char *pathname, int mode); if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \
nPtr = tdbOsRealloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \
int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); if (nPtr) { \
((int *)nPtr)[0] = (SIZE); \
int tdbPRead(int fd, void *pData, int count, i64 offset); nPtr = (char *)nPtr + sizeof(int); \
} \
#define TDB_REALLOC(PTR, SIZE) \ } else { \
({ \ nPtr = (PTR); \
void *nPtr; \ } \
if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ nPtr; \
nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \
if (nPtr) { \
((int *)nPtr)[0] = (SIZE); \
nPtr = (char *)nPtr + sizeof(int); \
} \
} else { \
nPtr = (PTR); \
} \
nPtr; \
}) })
#define TDB_FREE(PTR) \ #define TDB_FREE(PTR) \
do { \ do { \
if (PTR) { \ if (PTR) { \
free((char *)(PTR) - sizeof(int)); \ tdbOsFree((char *)(PTR) - sizeof(int)); \
} \ } \
} while (0) } while (0)
static inline void *tdbOsMalloc(void *arg, size_t size) { static inline void *tdbDefaultMalloc(void *arg, size_t size) {
void *ptr; void *ptr;
ptr = malloc(size); ptr = tdbOsMalloc(size);
return ptr; return ptr;
} }
static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); } static inline void tdbDefaultFree(void *arg, void *ptr) { tdbOsFree(ptr); }
static inline int tdbPutVarInt(u8 *p, int v) { static inline int tdbPutVarInt(u8 *p, int v) {
int n = 0; int n = 0;

10
source/libs/tdb/src/page/tdbPage.c
View File

@ -48,7 +48,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
*ppPage = NULL; *ppPage = NULL;
size = pageSize + sizeof(*pPage); size = pageSize + sizeof(*pPage);
if (xMalloc == NULL) { if (xMalloc == NULL) {
xMalloc = tdbOsMalloc; xMalloc = tdbDefaultMalloc;
} }
ptr = (u8 *)((*xMalloc)(arg, size)); ptr = (u8 *)((*xMalloc)(arg, size));
@ -76,7 +76,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg)
u8 *ptr; u8 *ptr;
if (!xFree) { if (!xFree) {
xFree = tdbOsFree; xFree = tdbDefaultFree;
} }
ptr = pPage->pData; ptr = pPage->pData;
@ -144,7 +144,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
} }
// TODO: here has memory leak // TODO: here has memory leak
pNewCell = (SCell *)malloc(szCell); pNewCell = (SCell *)tdbOsMalloc(szCell);
memcpy(pNewCell, pCell, szCell); memcpy(pNewCell, pCell, szCell);
pPage->apOvfl[iOvfl] = pNewCell; pPage->apOvfl[iOvfl] = pNewCell;
@ -372,11 +372,11 @@ static int tdbPageDefragment(SPage *pPage) {
int idx; int idx;
int iCell; int iCell;
ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree);
nFree = TDB_PAGE_NFREE(pPage); nFree = TDB_PAGE_NFREE(pPage);
nCells = TDB_PAGE_NCELLS(pPage); nCells = TDB_PAGE_NCELLS(pPage);
ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree);
// Loop to compact the page content // Loop to compact the page content
// Here we use an O(n^2) algorithm to do the job since // Here we use an O(n^2) algorithm to do the job since
// this is a low frequency job. // this is a low frequency job.

51
source/libs/tdb/test/tdbTest.cpp
View File

@ -11,7 +11,7 @@ typedef struct SPoolMem {
} SPoolMem; } SPoolMem;
static SPoolMem *openPool() { static SPoolMem *openPool() {
SPoolMem *pPool = (SPoolMem *)malloc(sizeof(*pPool)); SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool));
pPool->prev = pPool->next = pPool; pPool->prev = pPool->next = pPool;
pPool->size = 0; pPool->size = 0;
@ -31,20 +31,22 @@ static void closePool(SPoolMem *pPool) {
pMem->prev->next = pMem->next; pMem->prev->next = pMem->next;
pPool->size -= pMem->size; pPool->size -= pMem->size;
free(pMem); tdbOsFree(pMem);
} while (1); } while (1);
assert(pPool->size == 0); assert(pPool->size == 0);
free(pPool); tdbOsFree(pPool);
} }
#define clearPool closePool
static void *poolMalloc(void *arg, int size) { static void *poolMalloc(void *arg, int size) {
void *ptr = NULL; void *ptr = NULL;
SPoolMem *pPool = (SPoolMem *)arg; SPoolMem *pPool = (SPoolMem *)arg;
SPoolMem *pMem; SPoolMem *pMem;
pMem = (SPoolMem *)malloc(sizeof(*pMem) + size); pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size);
if (pMem == NULL) { if (pMem == NULL) {
assert(0); assert(0);
} }
@ -71,7 +73,7 @@ static void poolFree(void *arg, void *ptr) {
pMem->prev->next = pMem->next; pMem->prev->next = pMem->next;
pPool->size -= pMem->size; pPool->size -= pMem->size;
free(pMem); tdbOsFree(pMem);
} }
static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
@ -113,10 +115,10 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
TEST(tdb_test, simple_test) { TEST(tdb_test, simple_test) {
int ret; int ret;
STEnv *pEnv; TENV *pEnv;
STDB *pDb; TDB *pDb;
FKeyComparator compFunc; FKeyComparator compFunc;
int nData = 10000000; int nData = 1000000;
// Open Env // Open Env
ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv);
@ -132,13 +134,34 @@ TEST(tdb_test, simple_test) {
char val[64]; char val[64];
{ // Insert some data { // Insert some data
int i = 1;
SPoolMem *pPool;
int memPoolCapacity = 16 * 1024;
pPool = openPool();
tdbTxnBegin(pEnv);
for (;;) {
if (i > nData) break;
for (int i = 1; i <= nData; i++) {
sprintf(key, "key%d", i); sprintf(key, "key%d", i);
sprintf(val, "value%d", i); sprintf(val, "value%d", i);
ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
if (pPool->size >= memPoolCapacity) {
tdbTxnCommit(pEnv);
clearPool(pPool);
tdbTxnBegin(pEnv);
}
i++;
} }
closePool(pPool);
} }
{ // Query the data { // Query the data
@ -160,11 +183,11 @@ TEST(tdb_test, simple_test) {
} }
{ // Iterate to query the DB data { // Iterate to query the DB data
STDBC *pDBC; TDBC *pDBC;
void *pKey = NULL; void *pKey = NULL;
void *pVal = NULL; void *pVal = NULL;
int vLen, kLen; int vLen, kLen;
int count = 0; int count = 0;
ret = tdbDbcOpen(pDb, &pDBC); ret = tdbDbcOpen(pDb, &pDBC);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);

12
source/libs/transport/test/transportTests.cpp
View File

@ -156,14 +156,14 @@ TEST_F(TransCtxEnv, mergeTest) {
STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx));
transCtxInit(src); transCtxInit(src);
{ {
STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = taosMemoryFree};
val1.val = taosMemoryMalloc(12); val1.val = taosMemoryMalloc(12);
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++; key++;
} }
{ {
STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = taosMemoryFree};
val1.val = taosMemoryMalloc(12); val1.val = taosMemoryMalloc(12);
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++; key++;
@ -176,14 +176,14 @@ TEST_F(TransCtxEnv, mergeTest) {
STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx));
transCtxInit(src); transCtxInit(src);
{ {
STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = taosMemoryFree};
val1.val = taosMemoryMalloc(12); val1.val = taosMemoryMalloc(12);
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++; key++;
} }
{ {
STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = taosMemoryFree};
val1.val = taosMemoryMalloc(12); val1.val = taosMemoryMalloc(12);
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));
key++; key++;
@ -198,7 +198,7 @@ TEST_F(TransCtxEnv, mergeTest) {
STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx));
transCtxInit(src); transCtxInit(src);
{ {
STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = taosMemoryFree};
val1.val = taosMemoryCalloc(1, 11); val1.val = taosMemoryCalloc(1, 11);
memcpy(val1.val, val.c_str(), val.size()); memcpy(val1.val, val.c_str(), val.size());
@ -206,7 +206,7 @@ TEST_F(TransCtxEnv, mergeTest) {
key++; key++;
} }
{ {
STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = taosMemoryFree};
val1.val = taosMemoryCalloc(1, 11); val1.val = taosMemoryCalloc(1, 11);
memcpy(val1.val, val.c_str(), val.size()); memcpy(val1.val, val.c_str(), val.size());
taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1));

3
source/os/CMakeLists.txt
View File

@ -13,6 +13,9 @@ find_path(IconvApiIncludes iconv.h PATHS)
if(NOT IconvApiIncludes) if(NOT IconvApiIncludes)
add_definitions(-DDISALLOW_NCHAR_WITHOUT_ICONV) add_definitions(-DDISALLOW_NCHAR_WITHOUT_ICONV)
endif () endif ()
if(USE_TD_MEMORY)
add_definitions(-DUSE_TD_MEMORY)
endif ()
target_link_libraries( target_link_libraries(
os pthread dl rt m os pthread dl rt m
) )

29
source/os/src/osMemory.c
View File

@ -14,17 +14,17 @@
*/ */
#define ALLOW_FORBID_FUNC #define ALLOW_FORBID_FUNC
#include <malloc.h>
#include "os.h" #include "os.h"
#define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S') #define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S')
#define TD_MEMORY_STACK_TRACE_DEPTH 10 #define TD_MEMORY_STACK_TRACE_DEPTH 10
typedef struct TdMemoryInfo typedef struct TdMemoryInfo {
{
int32_t symbol; int32_t symbol;
int32_t memorySize;
void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH]; // gdb: disassemble /m 0xXXX void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH]; // gdb: disassemble /m 0xXXX
int32_t memorySize;
} *TdMemoryInfoPtr , TdMemoryInfo; } *TdMemoryInfoPtr , TdMemoryInfo;
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@ -32,6 +32,7 @@ typedef struct TdMemoryInfo
#else #else
#include<execinfo.h> #include<execinfo.h>
#define STACKCALL __attribute__((regparm(1), noinline)) #define STACKCALL __attribute__((regparm(1), noinline))
void **STACKCALL taosGetEbp(void) { void **STACKCALL taosGetEbp(void) {
void **ebp = NULL; void **ebp = NULL;
@ -41,6 +42,7 @@ void **STACKCALL taosGetEbp(void) {
: "memory"); /* not affect register */ : "memory"); /* not affect register */
return (void **)(*ebp); return (void **)(*ebp);
} }
int32_t taosBackTrace(void **buffer, int32_t size) { int32_t taosBackTrace(void **buffer, int32_t size) {
int32_t frame = 0; int32_t frame = 0;
void **ebp; void **ebp;
@ -59,6 +61,7 @@ int32_t taosBackTrace(void **buffer, int32_t size) {
} }
return frame; return frame;
} }
#endif #endif
// char **taosBackTraceSymbols(int32_t *size) { // char **taosBackTraceSymbols(int32_t *size) {
@ -68,6 +71,7 @@ int32_t taosBackTrace(void **buffer, int32_t size) {
// } // }
void *taosMemoryMalloc(int32_t size) { void *taosMemoryMalloc(int32_t size) {
#ifdef USE_TD_MEMORY
void *tmp = malloc(size + sizeof(TdMemoryInfo)); void *tmp = malloc(size + sizeof(TdMemoryInfo));
if (tmp == NULL) return NULL; if (tmp == NULL) return NULL;
@ -77,9 +81,13 @@ void *taosMemoryMalloc(int32_t size) {
taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH);
return (char*)tmp + sizeof(TdMemoryInfo); return (char*)tmp + sizeof(TdMemoryInfo);
#else
return malloc(size);
#endif
} }
void *taosMemoryCalloc(int32_t num, int32_t size) { void *taosMemoryCalloc(int32_t num, int32_t size) {
#ifdef USE_TD_MEMORY
int32_t memorySize = num * size; int32_t memorySize = num * size;
char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1);
if (tmp == NULL) return NULL; if (tmp == NULL) return NULL;
@ -90,9 +98,13 @@ void *taosMemoryCalloc(int32_t num, int32_t size) {
taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH);
return (char*)tmp + sizeof(TdMemoryInfo); return (char*)tmp + sizeof(TdMemoryInfo);
#else
return calloc(num, size);
#endif
} }
void *taosMemoryRealloc(void *ptr, int32_t size) { void *taosMemoryRealloc(void *ptr, int32_t size) {
#ifdef USE_TD_MEMORY
if (ptr == NULL) return taosMemoryMalloc(size); if (ptr == NULL) return taosMemoryMalloc(size);
TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo));
@ -108,9 +120,13 @@ void *taosMemoryRealloc(void *ptr, int32_t size) {
((TdMemoryInfoPtr)tmp)->memorySize = size; ((TdMemoryInfoPtr)tmp)->memorySize = size;
return (char*)tmp + sizeof(TdMemoryInfo); return (char*)tmp + sizeof(TdMemoryInfo);
#else
return realloc(ptr, size);
#endif
} }
void taosMemoryFree(const void *ptr) { void taosMemoryFree(const void *ptr) {
#ifdef USE_TD_MEMORY
if (ptr == NULL) return; if (ptr == NULL) return;
TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo));
@ -121,13 +137,20 @@ void taosMemoryFree(const void *ptr) {
} else { } else {
free((void*)ptr); free((void*)ptr);
} }
#else
return free((void*)ptr);
#endif
} }
int32_t taosMemorySize(void *ptr) { int32_t taosMemorySize(void *ptr) {
#ifdef USE_TD_MEMORY
if (ptr == NULL) return 0; if (ptr == NULL) return 0;
TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo));
assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL);
return pTdMemoryInfo->memorySize; return pTdMemoryInfo->memorySize;
#else
return malloc_usable_size(ptr);
#endif
} }

22
source/util/src/tprocess.c
View File

@ -264,7 +264,8 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea
taosThreadMutexUnlock(pQueue->mutex); taosThreadMutexUnlock(pQueue->mutex);
tsem_post(&pQueue->sem); tsem_post(&pQueue->sem);
uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, headLen, pBody, bodyLen, pQueue); uTrace("proc:%s, push msg to queue:%p remains:%d, head:%d:%p body:%d:%p", pQueue->name, pQueue, pQueue->items,
headLen, pHead, bodyLen, pBody);
return 0; return 0;
} }
@ -277,7 +278,7 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea
taosThreadMutexUnlock(pQueue->mutex); taosThreadMutexUnlock(pQueue->mutex);
tsem_post(&pQueue->sem); tsem_post(&pQueue->sem);
terrno = TSDB_CODE_OUT_OF_SHM_MEM; terrno = TSDB_CODE_OUT_OF_SHM_MEM;
return -1; return 0;
} }
int32_t headLen = 0; int32_t headLen = 0;
@ -341,8 +342,9 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea
*pHeadLen = headLen; *pHeadLen = headLen;
*pBodyLen = bodyLen; *pBodyLen = bodyLen;
uTrace("proc:%s, get msg:%p:%d cont:%p:%d from queue:%p", pQueue->name, pHead, headLen, pBody, bodyLen, pQueue); uTrace("proc:%s, pop msg from queue:%p remains:%d, head:%d:%p body:%d:%p", pQueue->name, pQueue, pQueue->items,
return 0; headLen, pHead, bodyLen, pBody);
return 1;
} }
SProcObj *taosProcInit(const SProcCfg *pCfg) { SProcObj *taosProcInit(const SProcCfg *pCfg) {
@ -396,15 +398,15 @@ static void taosProcThreadLoop(SProcQueue *pQueue) {
void *pHead, *pBody; void *pHead, *pBody;
int32_t headLen, bodyLen; int32_t headLen, bodyLen;
uDebug("proc:%s, start to get message from queue:%p", pQueue->name, pQueue); uDebug("proc:%s, start to get msg from queue:%p", pQueue->name, pQueue);
while (1) { while (1) {
int32_t code = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen); int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen);
if (code < 0) { if (numOfMsgs == 0) {
uDebug("proc:%s, get no message from queue:%p and exiting", pQueue->name, pQueue); uDebug("proc:%s, get no msg from queue:%p and exit the proc thread", pQueue->name, pQueue);
break; break;
} else if (code == 0) { } else if (numOfMsgs < 0) {
uTrace("proc:%s, get no message from queue:%p since %s", pQueue->name, pQueue, terrstr()); uTrace("proc:%s, get no msg from queue:%p since %s", pQueue->name, pQueue, terrstr());
taosMsleep(1); taosMsleep(1);
continue; continue;
} else { } else {

4
source/util/src/tqueue.c
View File

@ -146,7 +146,7 @@ void taosFreeQitem(void *pItem) {
taosMemoryFree(temp); taosMemoryFree(temp);
} }
int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { void taosWriteQitem(STaosQueue *queue, void *pItem) {
STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode)); STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode));
pNode->next = NULL; pNode->next = NULL;
@ -167,8 +167,6 @@ int32_t taosWriteQitem(STaosQueue *queue, void *pItem) {
taosThreadMutexUnlock(&queue->mutex); taosThreadMutexUnlock(&queue->mutex);
if (queue->qset) tsem_post(&queue->qset->sem); if (queue->qset) tsem_post(&queue->qset->sem);
return 0;
} }
int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { int32_t taosReadQitem(STaosQueue *queue, void **ppItem) {

6
source/util/src/tworker.c
View File

@ -287,8 +287,8 @@ void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) {
int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) { int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) {
SQWorkerPool *pPool = &pWorker->pool; SQWorkerPool *pPool = &pWorker->pool;
pPool->name = pCfg->name; pPool->name = pCfg->name;
pPool->min = pCfg->minNum; pPool->min = pCfg->min;
pPool->max = pCfg->maxNum; pPool->max = pCfg->max;
if (tQWorkerInit(pPool) != 0) { if (tQWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
@ -316,7 +316,7 @@ void tSingleWorkerCleanup(SSingleWorker *pWorker) {
int32_t tMultiWorkerInit(SMultiWorker *pWorker, const SMultiWorkerCfg *pCfg) { int32_t tMultiWorkerInit(SMultiWorker *pWorker, const SMultiWorkerCfg *pCfg) {
SWWorkerPool *pPool = &pWorker->pool; SWWorkerPool *pPool = &pWorker->pool;
pPool->name = pCfg->name; pPool->name = pCfg->name;
pPool->max = pCfg->maxNum; pPool->max = pCfg->max;
if (tWWorkerInit(pPool) != 0) { if (tWWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;

2
tests/script/jenkins/basic.txt
View File

@ -6,6 +6,8 @@
# ---- db # ---- db
./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic1.sim
./test.sh -f tsim/db/basic2.sim
./test.sh -f tsim/db/basic3.sim
./test.sh -f tsim/db/basic6.sim ./test.sh -f tsim/db/basic6.sim
./test.sh -f tsim/db/basic7.sim ./test.sh -f tsim/db/basic7.sim
./test.sh -f tsim/db/error1.sim ./test.sh -f tsim/db/error1.sim

2
tests/script/sh/massiveTable/compileVersion.sh
View File

@ -75,10 +75,12 @@ rm -f /usr/bin/taos
rm -f /usr/bin/taosd rm -f /usr/bin/taosd
rm -f /usr/bin/create_table rm -f /usr/bin/create_table
rm -f /usr/bin/tmq_demo rm -f /usr/bin/tmq_demo
rm -f /usr/bin/tmq_sim
ln -s $taos_dir/taos /usr/bin/taos ln -s $taos_dir/taos /usr/bin/taos
ln -s $taosd_dir/taosd /usr/bin/taosd ln -s $taosd_dir/taosd /usr/bin/taosd
ln -s $exec_process_dir/create_table /usr/bin/create_table ln -s $exec_process_dir/create_table /usr/bin/create_table
ln -s $exec_process_dir/tmq_demo /usr/bin/tmq_demo ln -s $exec_process_dir/tmq_demo /usr/bin/tmq_demo
ln -s $exec_process_dir/tmq_sim /usr/bin/tmq_sim

34
tests/script/general/db/basic2.sim → tests/script/tsim/db/basic2.sim
View File

@ -1,7 +1,24 @@
system sh/stop_dnodes.sh system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sleep 2000
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect sql connect
print =============== create database d1 print =============== create database d1
@ -13,7 +30,10 @@ sql create table t3 (ts timestamp, i int);
sql create table t4 (ts timestamp, i int); sql create table t4 (ts timestamp, i int);
sql show databases sql show databases
if $rows != 1 then print rows: $rows
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
if $rows != 2 then
return -1 return -1
endi endi
@ -21,13 +41,13 @@ if $data00 != d1 then
return -1 return -1
endi endi
if $data02 != 4 then if $data02 != 2 then # vgroups
return -1 return -1
endi endi
if $data03 != 1 then #if $data03 != 4 then # ntables
return -1 # return -1
endi #endi
sql show tables sql show tables
if $rows != 4 then if $rows != 4 then
@ -42,7 +62,7 @@ sql create table t2 (ts timestamp, i int);
sql create table t3 (ts timestamp, i int); sql create table t3 (ts timestamp, i int);
sql show databases sql show databases
if $rows != 2 then if $rows != 3 then
return -1 return -1
endi endi

Some files were not shown because too many files have changed in this diff Show More