xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/data_format

This commit is contained in:
Hongze Cheng 2022-05-16 01:20:46 +00:00
parent 6e3a0bb79d 4adba6ff23
commit e0b3e97de6
91 changed files with 5590 additions and 1212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/common/taosdef.h
View File

@ -37,7 +37,8 @@ typedef enum {
TSDB_STREAM_TABLE = 4, // table created from stream computing TSDB_STREAM_TABLE = 4, // table created from stream computing
TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_TEMP_TABLE = 5, // temp table created by nest query
TSDB_SYSTEM_TABLE = 6, TSDB_SYSTEM_TABLE = 6,
TSDB_TABLE_MAX = 7 TSDB_TSMA_TABLE = 7, // time-range-wise sma
TSDB_TABLE_MAX = 8
} ETableType; } ETableType;
typedef enum { typedef enum {

163
include/common/tmsg.h
View File

@ -2163,26 +2163,23 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
typedef struct { typedef struct {
int8_t version; // for compatibility(default 0) int8_t version; // for compatibility(default 0)
int8_t intervalUnit; // MACRO: TIME_UNIT_XXX int8_t intervalUnit; // MACRO: TIME_UNIT_XXX
int8_t slidingUnit; // MACRO: TIME_UNIT_XXX int8_t slidingUnit; // MACRO: TIME_UNIT_XXX
int8_t timezoneInt; // sma data expired if timezone changes. int8_t timezoneInt; // sma data expired if timezone changes.
char indexName[TSDB_INDEX_NAME_LEN]; char indexName[TSDB_INDEX_NAME_LEN];
int32_t exprLen; int32_t exprLen;
int32_t tagsFilterLen; int32_t tagsFilterLen;
int64_t indexUid; int64_t indexUid;
tb_uid_t tableUid; // super/child/common table uid tb_uid_t tableUid; // super/child/common table uid
int64_t interval; int64_t interval;
int64_t offset; // use unit by precision of DB int64_t offset; // use unit by precision of DB
int64_t sliding; int64_t sliding;
char* expr; // sma expression const char* expr; // sma expression
char* tagsFilter; const char* tagsFilter;
} STSma; // Time-range-wise SMA } STSma; // Time-range-wise SMA
typedef struct { typedef STSma SVCreateTSmaReq;
int64_t ver; // use a general definition
STSma tSma;
} SVCreateTSmaReq;
typedef struct { typedef struct {
int8_t type; // 0 status report, 1 update data int8_t type; // 0 status report, 1 update data
@ -2191,7 +2188,6 @@ typedef struct {
} STSmaMsg; } STSmaMsg;
typedef struct { typedef struct {
int64_t ver; // use a general definition
int64_t indexUid; int64_t indexUid;
char indexName[TSDB_INDEX_NAME_LEN]; char indexName[TSDB_INDEX_NAME_LEN];
} SVDropTSmaReq; } SVDropTSmaReq;
@ -2200,28 +2196,21 @@ typedef struct {
int tmp; // TODO: to avoid compile error int tmp; // TODO: to avoid compile error
} SVCreateTSmaRsp, SVDropTSmaRsp; } SVCreateTSmaRsp, SVDropTSmaRsp;
#if 0
int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq); int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq);
void* tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq); void* tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq);
int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq); int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq); void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
#endif
// RSma: Rollup SMA int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
typedef struct { int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
int64_t interval; int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq);
int32_t retention; // unit: day int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
uint16_t days; // unit: day
int8_t intervalUnit;
} SSmaParams;
typedef struct { typedef struct {
STSma tsma; int32_t number;
float xFilesFactor; STSma* tSma;
SArray* smaParams; // SSmaParams
} SRSma;
typedef struct {
uint32_t number;
STSma* tSma;
} STSmaWrapper; } STSmaWrapper;
static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
@ -2248,96 +2237,26 @@ static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
return NULL; return NULL;
} }
static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
int32_t tlen = 0; int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
tlen += taosEncodeFixedI8(buf, pSma->version); int32_t tEncodeTSma(SEncoder* pCoder, const STSma* pSma);
tlen += taosEncodeFixedI8(buf, pSma->intervalUnit); int32_t tDecodeTSma(SDecoder* pCoder, STSma* pSma);
tlen += taosEncodeFixedI8(buf, pSma->slidingUnit);
tlen += taosEncodeFixedI8(buf, pSma->timezoneInt);
tlen += taosEncodeString(buf, pSma->indexName);
tlen += taosEncodeFixedI32(buf, pSma->exprLen);
tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen);
tlen += taosEncodeFixedI64(buf, pSma->indexUid);
tlen += taosEncodeFixedI64(buf, pSma->tableUid);
tlen += taosEncodeFixedI64(buf, pSma->interval);
tlen += taosEncodeFixedI64(buf, pSma->offset);
tlen += taosEncodeFixedI64(buf, pSma->sliding);
if (pSma->exprLen > 0) { static int32_t tEncodeTSmaWrapper(SEncoder* pEncoder, const STSmaWrapper* pReq) {
tlen += taosEncodeString(buf, pSma->expr); if (tEncodeI32(pEncoder, pReq->number) < 0) return -1;
for (int32_t i = 0; i < pReq->number; ++i) {
tEncodeTSma(pEncoder, pReq->tSma + i);
} }
return 0;
if (pSma->tagsFilterLen > 0) {
tlen += taosEncodeString(buf, pSma->tagsFilter);
}
return tlen;
} }
static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) { static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) {
int32_t tlen = 0; if (tDecodeI32(pDecoder, &pReq->number) < 0) return -1;
for (int32_t i = 0; i < pReq->number; ++i) {
tlen += taosEncodeFixedU32(buf, pSW->number); tDecodeTSma(pDecoder, pReq->tSma + i);
for (uint32_t i = 0; i < pSW->number; ++i) {
tlen += tEncodeTSma(buf, pSW->tSma + i);
} }
return tlen; return 0;
}
static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
buf = taosDecodeFixedI8(buf, &pSma->version);
buf = taosDecodeFixedI8(buf, &pSma->intervalUnit);
buf = taosDecodeFixedI8(buf, &pSma->slidingUnit);
buf = taosDecodeFixedI8(buf, &pSma->timezoneInt);
buf = taosDecodeStringTo(buf, pSma->indexName);
buf = taosDecodeFixedI32(buf, &pSma->exprLen);
buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen);
buf = taosDecodeFixedI64(buf, &pSma->indexUid);
buf = taosDecodeFixedI64(buf, &pSma->tableUid);
buf = taosDecodeFixedI64(buf, &pSma->interval);
buf = taosDecodeFixedI64(buf, &pSma->offset);
buf = taosDecodeFixedI64(buf, &pSma->sliding);
if (pSma->exprLen > 0) {
if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
tdDestroyTSma(pSma);
return NULL;
}
} else {
pSma->expr = NULL;
}
if (pSma->tagsFilterLen > 0) {
if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
tdDestroyTSma(pSma);
return NULL;
}
} else {
pSma->tagsFilter = NULL;
}
return buf;
}
static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
buf = taosDecodeFixedU32(buf, &pSW->number);
pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma));
if (pSW->tSma == NULL) {
return NULL;
}
for (uint32_t i = 0; i < pSW->number; ++i) {
if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) {
for (uint32_t j = i; j >= 0; --i) {
tdDestroyTSma(pSW->tSma + j);
}
taosMemoryFree(pSW->tSma);
return NULL;
}
}
return buf;
} }
typedef struct { typedef struct {
@ -2577,6 +2496,14 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
} }
typedef struct {
void* data;
} SStreamDispatchReq;
typedef struct {
int8_t status;
} SStreamDispatchRsp;
#define TD_AUTO_CREATE_TABLE 0x1 #define TD_AUTO_CREATE_TABLE 0x1
typedef struct { typedef struct {
int64_t suid; int64_t suid;

15
include/common/ttime.h
View File

@ -59,10 +59,11 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) {
* precision == TSDB_TIME_PRECISION_NANO, it returns timestamp in nanosecond. * precision == TSDB_TIME_PRECISION_NANO, it returns timestamp in nanosecond.
*/ */
static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 : int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000
(precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
time_t t = taosTime(NULL); : 1000000000;
struct tm * tm= taosLocalTime(&t, NULL); time_t t = taosTime(NULL);
struct tm* tm = taosLocalTime(&t, NULL);
tm->tm_hour = 0; tm->tm_hour = 0;
tm->tm_min = 0; tm->tm_min = 0;
tm->tm_sec = 0; tm->tm_sec = 0;
@ -79,13 +80,13 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth); int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
void deltaToUtcInitOnce(); void deltaToUtcInitOnce();
char getPrecisionUnit(int32_t precision); char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit); int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal); int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision); void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t time, int32_t precision);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/libs/function/function.h
View File

@ -309,7 +309,7 @@ void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
/** /**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
* @return error code * @return error code
*/ */
int32_t udfcOpen(); int32_t udfcOpen();

34
include/libs/function/tudf.h
View File

@ -39,16 +39,6 @@ extern "C" {
//====================================================================================== //======================================================================================
//begin API to taosd and qworker //begin API to taosd and qworker
typedef void *UdfcFuncHandle;
/**
* setup udf
* @param udf, in
* @param handle, out
* @return error code
*/
int32_t setupUdf(char udfName[], UdfcFuncHandle *handle);
typedef struct SUdfColumnMeta { typedef struct SUdfColumnMeta {
int16_t type; int16_t type;
int32_t bytes; int32_t bytes;
@ -95,32 +85,44 @@ typedef struct SUdfInterBuf {
char* buf; char* buf;
int8_t numOfResult; //zero or one int8_t numOfResult; //zero or one
} SUdfInterBuf; } SUdfInterBuf;
typedef void *UdfcFuncHandle;
/**
* setup udf
* @param udf, in
* @param funcHandle, out
* @return error code
*/
int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle);
// output: interBuf // output: interBuf
int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf);
// input: block, state // input: block, state
// output: newState // output: newState
int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
// input: interBuf // input: interBuf
// output: resultData // output: resultData
int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
// input: interbuf1, interbuf2 // input: interbuf1, interbuf2
// output: resultBuf // output: resultBuf
int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
// input: block // input: block
// output: resultData // output: resultData
int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output); int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
/** /**
* tearn down udf * tearn down udf
* @param handle * @param handle
* @return * @return
*/ */
int32_t teardownUdf(UdfcFuncHandle handle); int32_t doTeardownUdf(UdfcFuncHandle handle);
bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
int32_t teardownUdfs();
// end API to taosd and qworker // end API to taosd and qworker
//============================================================================================================================= //=============================================================================================================================
// begin API to UDF writer. // begin API to UDF writer.

119
include/libs/qcom/query.h
View File

@ -51,14 +51,12 @@ typedef struct STableComInfo {
} STableComInfo; } STableComInfo;
typedef struct SIndexMeta { typedef struct SIndexMeta {
#ifdef WINDOWS #ifdef WINDOWS
size_t avoidCompilationErrors; size_t avoidCompilationErrors;
#endif #endif
} SIndexMeta; } SIndexMeta;
/* /*
* ASSERT(sizeof(SCTableMeta) == 24) * ASSERT(sizeof(SCTableMeta) == 24)
* ASSERT(tableType == TSDB_CHILD_TABLE) * ASSERT(tableType == TSDB_CHILD_TABLE)
@ -95,7 +93,7 @@ typedef struct SDBVgInfo {
int32_t vgVersion; int32_t vgVersion;
int8_t hashMethod; int8_t hashMethod;
int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT
SHashObj *vgHash; //key:vgId, value:SVgroupInfo SHashObj* vgHash; // key:vgId, value:SVgroupInfo
} SDBVgInfo; } SDBVgInfo;
typedef struct SUseDbOutput { typedef struct SUseDbOutput {
@ -135,7 +133,7 @@ typedef struct SMsgSendInfo {
} SMsgSendInfo; } SMsgSendInfo;
typedef struct SQueryNodeStat { typedef struct SQueryNodeStat {
int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
} SQueryNodeStat; } SQueryNodeStat;
int32_t initTaskQueue(); int32_t initTaskQueue();
@ -172,7 +170,7 @@ const SSchema* tGetTbnameColumnSchema();
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
char *jobTaskStatusStr(int32_t status); char* jobTaskStatusStr(int32_t status);
SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
@ -184,62 +182,87 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) #define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) #define NEED_CLIENT_HANDLE_ERROR(_code) \
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
#define NEED_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) #define NEED_SCHEDULER_RETRY_ERROR(_code) \
((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
#define REQUEST_MAX_TRY_TIMES 5 #define REQUEST_MAX_TRY_TIMES 5
#define qFatal(...) \ #define qFatal(...) \
do { \ do { \
if (qDebugFlag & DEBUG_FATAL) { \ if (qDebugFlag & DEBUG_FATAL) { \
taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY FATAL ", DEBUG_FATAL, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qError(...) \ #define qError(...) \
do { \ do { \
if (qDebugFlag & DEBUG_ERROR) { \ if (qDebugFlag & DEBUG_ERROR) { \
taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qWarn(...) \ #define qWarn(...) \
do { \ do { \
if (qDebugFlag & DEBUG_WARN) { \ if (qDebugFlag & DEBUG_WARN) { \
taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qInfo(...) \ #define qInfo(...) \
do { \ do { \
if (qDebugFlag & DEBUG_INFO) { \ if (qDebugFlag & DEBUG_INFO) { \
taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qDebug(...) \ #define qDebug(...) \
do { \ do { \
if (qDebugFlag & DEBUG_DEBUG) { \ if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qTrace(...) \ #define qTrace(...) \
do { \ do { \
if (qDebugFlag & DEBUG_TRACE) { \ if (qDebugFlag & DEBUG_TRACE) { \
taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qDebugL(...) \ #define qDebugL(...) \
do { \ do { \
if (qDebugFlag & DEBUG_DEBUG) { \ if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define QRY_ERR_RET(c) \
#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) do { \
#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
return _code; \
} \
} while (0)
#define QRY_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
} \
return _code; \
} while (0)
#define QRY_ERR_JRET(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }

102
include/libs/stream/tstream.h
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tmsg.h" #include "tmsg.h"
#include "tmsgcb.h" #include "tmsgcb.h"
@ -29,8 +30,23 @@ extern "C" {
typedef struct SStreamTask SStreamTask; typedef struct SStreamTask SStreamTask;
enum { enum {
STREAM_TASK_STATUS__RUNNING = 1, TASK_STATUS__IDLE = 1,
STREAM_TASK_STATUS__STOP, TASK_STATUS__EXECUTING,
TASK_STATUS__CLOSING,
};
enum {
TASK_INPUT_STATUS__NORMAL = 1,
TASK_INPUT_STATUS__BLOCKED,
TASK_INPUT_STATUS__RECOVER,
TASK_INPUT_STATUS__STOP,
TASK_INPUT_STATUS__FAILED,
};
enum {
TASK_OUTPUT_STATUS__NORMAL = 1,
TASK_OUTPUT_STATUS__WAIT,
TASK_OUTPUT_STATUS__BLOCKED,
}; };
enum { enum {
@ -38,10 +54,64 @@ enum {
STREAM_CREATED_BY__SMA, STREAM_CREATED_BY__SMA,
}; };
enum {
STREAM_INPUT__DATA_SUBMIT = 1,
STREAM_INPUT__DATA_BLOCK,
STREAM_INPUT__CHECKPOINT,
};
typedef struct { typedef struct {
int32_t nodeId; // 0 for snode int8_t type;
SEpSet epSet;
} SStreamTaskEp; int32_t sourceVg;
int64_t sourceVer;
int32_t* dataRef;
SSubmitReq* data;
} SStreamDataSubmit;
typedef struct {
int8_t type;
int32_t sourceVg;
int64_t sourceVer;
SArray* blocks; // SArray<SSDataBlock*>
} SStreamDataBlock;
typedef struct {
int8_t type;
} SStreamCheckpoint;
static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) {
SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosMemoryCalloc(1, sizeof(SStreamDataSubmit));
if (pDataSubmit == NULL) return NULL;
pDataSubmit->data = pReq;
pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t));
if (pDataSubmit->data == NULL) goto FAIL;
*pDataSubmit->dataRef = 1;
return pDataSubmit;
FAIL:
taosMemoryFree(pDataSubmit);
return NULL;
}
static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) {
//
atomic_add_fetch_32(pDataSubmit->dataRef, 1);
}
static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) {
int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1);
ASSERT(ref >= 0);
if (ref == 0) {
taosMemoryFree(pDataSubmit->data);
taosMemoryFree(pDataSubmit->dataRef);
}
}
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
typedef struct { typedef struct {
void* inputHandle; void* inputHandle;
@ -122,9 +192,15 @@ enum {
TASK_SINK__FETCH, TASK_SINK__FETCH,
}; };
enum {
TASK_INPUT_TYPE__SUMBIT_BLOCK = 1,
TASK_INPUT_TYPE__DATA_BLOCK,
};
struct SStreamTask { struct SStreamTask {
int64_t streamId; int64_t streamId;
int32_t taskId; int32_t taskId;
int8_t inputType;
int8_t status; int8_t status;
int8_t sourceType; int8_t sourceType;
@ -155,9 +231,13 @@ struct SStreamTask {
STaskDispatcherShuffle shuffleDispatcher; STaskDispatcherShuffle shuffleDispatcher;
}; };
// msg buffer int8_t inputStatus;
int32_t memUsed; int8_t outputStatus;
STaosQueue* inputQ; STaosQueue* inputQ;
STaosQall* inputQAll;
STaosQueue* outputQ;
STaosQall* outputQAll;
// application storage // application storage
void* ahandle; void* ahandle;
@ -199,10 +279,16 @@ typedef struct {
SArray* res; // SArray<SSDataBlock> SArray* res; // SArray<SSDataBlock>
} SStreamSinkReq; } SStreamSinkReq;
int32_t streamEnqueueData(SStreamTask* pTask, const void* input, int32_t inputType); int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
int32_t streamTaskRun(SStreamTask* pTask);
int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

90
include/os/osAtomic.h
View File

@ -23,92 +23,92 @@ extern "C" {
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section. // When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID
#define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID
#define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID
#define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID #define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID
#define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID
#define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID
#define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID
#define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID
#define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID
#define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID #define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID
#define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID
#define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID
#define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID
#define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID
#endif #endif
int8_t atomic_load_8(int8_t volatile *ptr); int8_t atomic_load_8(int8_t volatile *ptr);
int16_t atomic_load_16(int16_t volatile *ptr); int16_t atomic_load_16(int16_t volatile *ptr);
int32_t atomic_load_32(int32_t volatile *ptr); int32_t atomic_load_32(int32_t volatile *ptr);
int64_t atomic_load_64(int64_t volatile *ptr); int64_t atomic_load_64(int64_t volatile *ptr);
void* atomic_load_ptr(void *ptr); void *atomic_load_ptr(void *ptr);
void atomic_store_8(int8_t volatile *ptr, int8_t val); void atomic_store_8(int8_t volatile *ptr, int8_t val);
void atomic_store_16(int16_t volatile *ptr, int16_t val); void atomic_store_16(int16_t volatile *ptr, int16_t val);
void atomic_store_32(int32_t volatile *ptr, int32_t val); void atomic_store_32(int32_t volatile *ptr, int32_t val);
void atomic_store_64(int64_t volatile *ptr, int64_t val); void atomic_store_64(int64_t volatile *ptr, int64_t val);
void atomic_store_ptr(void *ptr, void *val); void atomic_store_ptr(void *ptr, void *val);
int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
void* atomic_exchange_ptr(void *ptr, void *val); void *atomic_exchange_ptr(void *ptr, void *val);
int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval); int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval);
int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval); int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval);
void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval); void *atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval);
int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val); int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val); int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val); int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_add_fetch_ptr(void *ptr, void *val); void *atomic_add_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_add_ptr(void *ptr, void *val); void *atomic_fetch_add_ptr(void *ptr, void *val);
int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val); int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_sub_fetch_ptr(void *ptr, void *val); void *atomic_sub_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_sub_ptr(void *ptr, void *val); void *atomic_fetch_sub_ptr(void *ptr, void *val);
int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val); int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_and_fetch_ptr(void *ptr, void *val); void *atomic_and_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val); int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val); int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val); int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val); int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_and_ptr(void *ptr, void *val); void *atomic_fetch_and_ptr(void *ptr, void *val);
int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val); int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val); int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val); int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val); int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_or_fetch_ptr(void *ptr, void *val); void *atomic_or_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val); int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val); int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val); int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val); int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_or_ptr(void *ptr, void *val); void *atomic_fetch_or_ptr(void *ptr, void *val);
int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val); int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val); int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val); int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val); int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_xor_fetch_ptr(void *ptr, void *val); void *atomic_xor_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val); int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val); int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val); int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val); int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_xor_ptr(void *ptr, void *val); void *atomic_fetch_xor_ptr(void *ptr, void *val);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
include/util/taoserror.h
View File

@ -354,7 +354,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A) #define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C) #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C)
#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x062D) #define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x061D)
#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x061E)
// query // query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)
@ -636,6 +637,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_PERMISSION_DENIED TAOS_DEF_ERROR_CODE(0, 0x2644) #define TSDB_CODE_PAR_PERMISSION_DENIED TAOS_DEF_ERROR_CODE(0, 0x2644)
#define TSDB_CODE_PAR_INVALID_STREAM_QUERY TAOS_DEF_ERROR_CODE(0, 0x2645) #define TSDB_CODE_PAR_INVALID_STREAM_QUERY TAOS_DEF_ERROR_CODE(0, 0x2645)
#define TSDB_CODE_PAR_INVALID_INTERNAL_PK TAOS_DEF_ERROR_CODE(0, 0x2646) #define TSDB_CODE_PAR_INVALID_INTERNAL_PK TAOS_DEF_ERROR_CODE(0, 0x2646)
#define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC TAOS_DEF_ERROR_CODE(0, 0x2647)
#define TSDB_CODE_PAR_INVALID_PASSWD TAOS_DEF_ERROR_CODE(0, 0x2648)
//planner //planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
@ -657,6 +660,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) #define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905)
#define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906) #define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906)
#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907) #define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907)
#define TSDB_CODE_UDF_NO_FUNC_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2908)
#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000)
#define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) #define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001)

1
include/util/tlog.h
View File

@ -61,6 +61,7 @@ extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag; extern int32_t fsDebugFlag;
extern int32_t metaDebugFlag; extern int32_t metaDebugFlag;
extern int32_t fnDebugFlag; extern int32_t fnDebugFlag;
extern int32_t smaDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles); int32_t taosInitLog(const char *logName, int32_t maxFiles);
void taosCloseLog(); void taosCloseLog();

1
packaging/release.sh
View File

@ -67,7 +67,6 @@ bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compi
cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/ cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/
cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/
cp ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" cp ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries"
cp -rf ${compile_dir}/build/lib/pkgconfig ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory" cp -rf ${compile_dir}/build/lib/pkgconfig ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory"

6
packaging/tools/install.sh
View File

@ -215,15 +215,9 @@ function install_lib() {
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib_link_dir}/libtdb.so.1
${csudo} ln -s ${lib_link_dir}/libtdb.so.1 ${lib_link_dir}/libtdb.so
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib64_link_dir}/libtdb.so.1 || :
${csudo} ln -s ${lib64_link_dir}/libtdb.so.1 ${lib64_link_dir}/libtdb.so || :
fi fi
${csudo} ldconfig ${csudo} ldconfig

6
source/client/src/clientImpl.c
View File

@ -172,7 +172,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pTransporter = pTscObj->pAppInfo->pTransporter, .pTransporter = pTscObj->pAppInfo->pTransporter,
.pStmtCb = pStmtCb, .pStmtCb = pStmtCb,
.pUser = pTscObj->user}; .pUser = pTscObj->user,
.isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER))};
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
@ -947,8 +948,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
// TODO handle the compressed case // TODO handle the compressed case
pResultInfo->totalRows += pResultInfo->numOfRows; pResultInfo->totalRows += pResultInfo->numOfRows;
return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4);
convertUcs4);
} }
TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) { TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) {

282
source/client/src/clientSml.c
View File

@ -9,7 +9,6 @@
#include "tdef.h" #include "tdef.h"
#include "tlog.h" #include "tlog.h"
#include "tmsg.h" #include "tmsg.h"
#include "tstrbuild.h"
#include "ttime.h" #include "ttime.h"
#include "ttypes.h" #include "ttypes.h"
#include "tcommon.h" #include "tcommon.h"
@ -26,6 +25,35 @@
#define SLASH '\\' #define SLASH '\\'
#define tsMaxSQLStringLen (1024*1024) #define tsMaxSQLStringLen (1024*1024)
#define JUMP_SPACE(sql) while (*sql != '\0'){if(*sql == SPACE) sql++;else break;}
// comma ,
#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql) - 1) == SLASH)
#define IS_COMMA(sql) (*(sql) == COMMA && *((sql) - 1) != SLASH)
// space
#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql) - 1) == SLASH)
#define IS_SPACE(sql) (*(sql) == SPACE && *((sql) - 1) != SLASH)
// equal =
#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql) - 1) == SLASH)
#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql) - 1) != SLASH)
// quote "
#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql) - 1) == SLASH)
#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql) - 1) != SLASH)
// SLASH
#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql) - 1) == SLASH)
#define IS_SLASH_LETTER(sql) (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql))
#define MOVE_FORWARD_ONE(sql,len) (memmove((void*)((sql) - 1), (sql), len))
#define PROCESS_SLASH(key,keyLen) \
for (int i = 1; i < keyLen; ++i) { \
if(IS_SLASH_LETTER(key+i)){ \
MOVE_FORWARD_ONE(key+i, keyLen-i); \
i--; \
keyLen--; \
} \
}
#define OTD_MAX_FIELDS_NUM 2 #define OTD_MAX_FIELDS_NUM 2
#define OTD_JSON_SUB_FIELDS_NUM 2 #define OTD_JSON_SUB_FIELDS_NUM 2
#define OTD_JSON_FIELDS_NUM 4 #define OTD_JSON_FIELDS_NUM 4
@ -42,6 +70,7 @@
#define BINARY_ADD_LEN 2 // "binary" 2 means " " #define BINARY_ADD_LEN 2 // "binary" 2 means " "
#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " #define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" "
#define CHAR_SAVE_LENGTH 8
//================================================================================================= //=================================================================================================
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
@ -231,7 +260,7 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi
char tname[TSDB_TABLE_NAME_LEN] = {0}; char tname[TSDB_TABLE_NAME_LEN] = {0};
memcpy(tname, field->key, field->keyLen); memcpy(tname, field->key, field->keyLen);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
int32_t bytes = field->length; // todo int32_t bytes = field->length > CHAR_SAVE_LENGTH ? (2*field->length) : CHAR_SAVE_LENGTH;
int out = snprintf(buf, bufSize,"`%s` %s(%d)", int out = snprintf(buf, bufSize,"`%s` %s(%d)",
tname, tDataTypes[field->type].name, bytes); tname, tDataTypes[field->type].name, bytes);
*outBytes = out; *outBytes = out;
@ -431,7 +460,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
size_t superTableLen = 0; size_t superTableLen = 0;
void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); // todo escape void *superTable = taosHashGetKey(tableMetaSml, &superTableLen);
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
strcpy(pName.dbname, info->pRequest->pDb); strcpy(pName.dbname, info->pRequest->pDb);
memcpy(pName.tname, superTable, superTableLen); memcpy(pName.tname, superTable, superTableLen);
@ -760,7 +789,7 @@ static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t le
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL);
return -1; return -1;
} }
if(!data){ if(len == 0){
return smlGetTimeNow(tsType); return smlGetTimeNow(tsType);
} }
@ -850,66 +879,56 @@ static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
static int32_t smlParseInfluxString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){ static int32_t smlParseInfluxString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){
if(!sql) return TSDB_CODE_SML_INVALID_DATA; if(!sql) return TSDB_CODE_SML_INVALID_DATA;
while (*sql != '\0') { // jump the space at the begining JUMP_SPACE(sql)
if(*sql != SPACE) { if(*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA;
elements->measure = sql; elements->measure = sql;
break;
}
sql++;
}
if (!elements->measure || *sql == COMMA) {
smlBuildInvalidDataMsg(msg, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
// parse measure and tag // parse measure
while (*sql != '\0') { while (*sql != '\0') {
if (elements->measureLen == 0 && *sql == COMMA && *(sql - 1) != SLASH) { // find the first comma if((sql != elements->measure) && IS_SLASH_LETTER(sql)){
elements->measureLen = sql - elements->measure; MOVE_FORWARD_ONE(sql,strlen(sql) + 1);
sql++;
elements->tags = sql;
continue; continue;
} }
if(IS_COMMA(sql)){
if (*sql == SPACE && *(sql - 1) != SLASH) { // find the first space
if (elements->measureLen == 0) {
elements->measureLen = sql - elements->measure;
elements->tags = sql;
}
elements->tagsLen = sql - elements->tags;
elements->measureTagsLen = sql - elements->measure;
break; break;
} }
if(IS_SPACE(sql)){
break;
}
sql++; sql++;
} }
if(elements->tagsLen == 0){ // measure, cols1=a measure cols1=a elements->measureLen = sql - elements->measure;
elements->measureTagsLen = elements->measureLen;
}
if(elements->measureLen == 0) { if(elements->measureLen == 0) {
smlBuildInvalidDataMsg(msg, "invalid measure", elements->measure); smlBuildInvalidDataMsg(msg, "measure is empty", NULL);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
// parse tag
if(*sql == SPACE){
elements->tagsLen = 0;
}else{
if(*sql == COMMA) sql++;
elements->tags = sql;
while (*sql != '\0') {
if(IS_SPACE(sql)){
break;
}
sql++;
}
elements->tagsLen = sql - elements->tags;
}
elements->measureTagsLen = sql - elements->measure;
// parse cols // parse cols
while (*sql != '\0') { JUMP_SPACE(sql)
if(*sql != SPACE) { elements->cols = sql;
elements->cols = sql;
break;
}
sql++;
}
if(!elements->cols) {
smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols);
return TSDB_CODE_SML_INVALID_DATA;
}
bool isInQuote = false; bool isInQuote = false;
while (*sql != '\0') { while (*sql != '\0') {
if(*sql == QUOTE && *(sql - 1) != SLASH){ if(IS_QUOTE(sql)){
isInQuote = !isInQuote; isInQuote = !isInQuote;
} }
if(!isInQuote && *sql == SPACE && *(sql - 1) != SLASH) { if(!isInQuote && IS_SPACE(sql)){
break; break;
} }
sql++; sql++;
@ -919,20 +938,21 @@ static int32_t smlParseInfluxString(const char* sql, SSmlLineInfo *elements, SSm
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
elements->colsLen = sql - elements->cols; elements->colsLen = sql - elements->cols;
if(elements->colsLen == 0) {
smlBuildInvalidDataMsg(msg, "cols is empty", NULL);
return TSDB_CODE_SML_INVALID_DATA;
}
// parse ts,ts can be empty // parse timestamp
JUMP_SPACE(sql)
elements->timestamp = sql;
while (*sql != '\0') { while (*sql != '\0') {
if(*sql != SPACE && elements->timestamp == NULL) { if(*sql == SPACE){
elements->timestamp = sql;
}
if(*sql == SPACE && elements->timestamp != NULL){
break; break;
} }
sql++; sql++;
} }
if(elements->timestamp){ elements->timestampLen = sql - elements->timestamp;
elements->timestampLen = sql - elements->timestamp;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -949,38 +969,58 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t *
} }
} }
static int32_t smlParseTelnetTags(const char* data, int32_t len, SArray *cols, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ static int32_t smlParseTelnetTags(const char* data, SArray *cols, SHashObj *dumplicateKey, SSmlMsgBuf *msg){
for(int i = 0; i < len; i++){ const char *sql = data;
// parse key while(*sql != '\0'){
const char *key = data + i; JUMP_SPACE(sql)
if(*sql == '\0') break;
const char *key = sql;
int32_t keyLen = 0; int32_t keyLen = 0;
while(i < len){
if(data[i] == EQUAL){ // parse key
keyLen = data + i - key; while(*sql != '\0'){
if(*sql == SPACE) {
smlBuildInvalidDataMsg(msg, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
if(*sql == EQUAL) {
keyLen = sql - key;
sql++;
break; break;
} }
i++; sql++;
} }
if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){
smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
if(smlCheckDuplicateKey(key, keyLen, dumplicateKey)){ if(smlCheckDuplicateKey(key, keyLen, dumplicateKey)){
smlBuildInvalidDataMsg(msg, "dumplicate key", key); smlBuildInvalidDataMsg(msg, "dumplicate key", key);
return TSDB_CODE_TSC_DUP_TAG_NAMES; return TSDB_CODE_TSC_DUP_TAG_NAMES;
} }
// parse value // parse value
i++; const char *value = sql;
const char *value = data + i; int32_t valueLen = 0;
while(i < len){ while(*sql != '\0') {
if(data[i] == SPACE){ // parse value
if (*sql == SPACE) {
valueLen = sql - value;
sql++;
break; break;
} }
i++; if (*sql == EQUAL) {
smlBuildInvalidDataMsg(msg, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
sql++;
} }
int32_t valueLen = data + i - value; if(valueLen == 0){
valueLen = sql - value;
}
if(valueLen == 0){ if(valueLen == 0){
smlBuildInvalidDataMsg(msg, "invalid value", value); smlBuildInvalidDataMsg(msg, "invalid value", value);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
@ -993,13 +1033,14 @@ static int32_t smlParseTelnetTags(const char* data, int32_t len, SArray *cols, S
kv->keyLen = keyLen; kv->keyLen = keyLen;
kv->value = value; kv->value = value;
kv->length = valueLen; kv->length = valueLen;
kv->type = TSDB_DATA_TYPE_NCHAR; //todo kv->type = TSDB_DATA_TYPE_NCHAR;
if(cols) taosArrayPush(cols, &kv); if(cols) taosArrayPush(cols, &kv);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>] // format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTableInfo *tinfo, SArray *cols){ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTableInfo *tinfo, SArray *cols){
if(!sql) return TSDB_CODE_SML_INVALID_DATA; if(!sql) return TSDB_CODE_SML_INVALID_DATA;
@ -1048,10 +1089,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable
} }
// parse tags // parse tags
while(*sql == SPACE){ ret = smlParseTelnetTags(sql, tinfo->tags, info->dumplicateKey, &info->msgBuf);
sql++;
}
ret = smlParseTelnetTags(sql, strlen(sql), tinfo->tags, info->dumplicateKey, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
@ -1073,49 +1111,67 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
for(int i = 0; i < len; i++){ const char *sql = data;
// parse key while(sql < data + len){
const char *key = data + i; const char *key = sql;
int32_t keyLen = 0; int32_t keyLen = 0;
while(i < len){
if(data[i] == EQUAL && i > 0 && data[i-1] != SLASH){ while(sql < data + len){
keyLen = data + i - key; // parse key
if(IS_COMMA(sql)) {
smlBuildInvalidDataMsg(msg, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
if(IS_EQUAL(sql)) {
keyLen = sql - key;
sql++;
break; break;
} }
i++; sql++;
} }
if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){
smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
if(smlCheckDuplicateKey(key, keyLen, dumplicateKey)){ if(smlCheckDuplicateKey(key, keyLen, dumplicateKey)){
smlBuildInvalidDataMsg(msg, "dumplicate key", key); smlBuildInvalidDataMsg(msg, "dumplicate key", key);
return TSDB_CODE_TSC_DUP_TAG_NAMES; return TSDB_CODE_TSC_DUP_TAG_NAMES;
} }
// parse value // parse value
i++; const char *value = sql;
const char *value = data + i; int32_t valueLen = 0;
bool isInQuote = false; bool isInQuote = false;
while(i < len){ while(sql < data + len) {
if(!isTag && data[i] == QUOTE && data[i-1] != SLASH){ // parse value
if(!isTag && IS_QUOTE(sql)){
isInQuote = !isInQuote; isInQuote = !isInQuote;
sql++;
continue;
} }
if(!isInQuote && data[i] == COMMA && i > 0 && data[i-1] != SLASH){ if (!isInQuote && IS_COMMA(sql)) {
break; break;
} }
i++; if (!isInQuote && IS_EQUAL(sql)) {
smlBuildInvalidDataMsg(msg, "invalid data", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
sql++;
} }
if(!isTag && isInQuote){ valueLen = sql - value;
sql++;
if(isInQuote){
smlBuildInvalidDataMsg(msg, "only one quote", value); smlBuildInvalidDataMsg(msg, "only one quote", value);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
int32_t valueLen = data + i - value;
if(valueLen == 0){ if(valueLen == 0){
smlBuildInvalidDataMsg(msg, "invalid value", value); smlBuildInvalidDataMsg(msg, "invalid value", value);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
PROCESS_SLASH(key, keyLen)
PROCESS_SLASH(value, valueLen)
// add kv to SSmlKv // add kv to SSmlKv
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
@ -1138,49 +1194,6 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
//static int32_t parseSmlCols(const char* data, SArray *cols){
// while(*data != '\0'){
// if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA;
// const char *key = data;
// int32_t keyLen = 0;
// while(*data != '\0'){
// if(*data == EQUAL && *(data-1) != SLASH){
// keyLen = data - key;
// data ++;
// break;
// }
// data++;
// }
// if(keyLen == 0){
// return TSDB_CODE_SML_INVALID_DATA;
// }
//
// if(*data == COMMA) return TSDB_CODE_SML_INVALID_DATA;
// const char *value = data;
// int32_t valueLen = 0;
// while(*data != '\0'){
// if(*data == COMMA && *(data-1) != SLASH){
// valueLen = data - value;
// data ++;
// break;
// }
// data++;
// }
// if(valueLen == 0){
// return TSDB_CODE_SML_INVALID_DATA;
// }
//
// TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1);
// kv->key = key;
// kv->keyLen = keyLen;
// kv->value = value;
// kv->valueLen = valueLen;
// kv->type = TSDB_DATA_TYPE_NCHAR;
// if(cols) taosArrayPush(cols, &kv);
// }
// return TSDB_CODE_SUCCESS;
//}
static bool smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg){ static bool smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg){
for (int i = 0; i < taosArrayGetSize(cols); ++i) { //jump timestamp for (int i = 0; i < taosArrayGetSize(cols); ++i) { //jump timestamp
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
@ -1298,7 +1311,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col
} }
for(size_t i = 0; i < taosArrayGetSize(cols); i++){ for(size_t i = 0; i < taosArrayGetSize(cols); i++){
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); // todo key need escape, like \=, because find by schema name later taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
} }
taosArrayPush(oneTable->cols, &kvHash); taosArrayPush(oneTable->cols, &kvHash);
@ -1346,6 +1359,7 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta){
taosArrayDestroy(meta->tags); taosArrayDestroy(meta->tags);
taosArrayDestroy(meta->cols); taosArrayDestroy(meta->cols);
taosMemoryFree(meta->tableMeta); taosMemoryFree(meta->tableMeta);
taosMemoryFree(meta);
} }
static void smlDestroyCols(SArray *cols) { static void smlDestroyCols(SArray *cols) {

180
source/client/test/smlTest.cpp
View File

@ -41,12 +41,14 @@ TEST(testCase, smlParseInfluxString_Test) {
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
// case 1 // case 1
char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3";
char *sql = (char*)taosMemoryCalloc(256, 1);
memcpy(sql, tmp, strlen(tmp) + 1);
int ret = smlParseInfluxString(sql, &elements, &msgBuf); int ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
ASSERT_EQ(elements.measureLen, strlen("st")); ASSERT_EQ(elements.measureLen, strlen(",st"));
ASSERT_EQ(elements.measureTagsLen, strlen("st,t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3"));
ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); ASSERT_EQ(elements.tags, sql + elements.measureLen + 1);
ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3"));
@ -58,76 +60,79 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
// case 2 false // case 2 false
sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
// case 3 false // case 3 false
sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1);
ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3"));
// case 4 tag is null // case 4 tag is null
sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
ASSERT_EQ(elements.measureLen, strlen("st")); ASSERT_EQ(elements.measureLen, strlen("st"));
ASSERT_EQ(elements.measureTagsLen, strlen("st")); ASSERT_EQ(elements.measureTagsLen, strlen("st,"));
ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); ASSERT_EQ(elements.tags, sql + elements.measureTagsLen);
ASSERT_EQ(elements.tagsLen, 0); ASSERT_EQ(elements.tagsLen, 0);
ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1);
ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64"));
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 2 + elements.colsLen + 1); ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
// case 5 tag is null // case 5 tag is null
sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
sql++;
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql + 1);
ASSERT_EQ(elements.measureLen, strlen("st")); ASSERT_EQ(elements.measureLen, strlen("st"));
ASSERT_EQ(elements.measureTagsLen, strlen("st")); ASSERT_EQ(elements.measureTagsLen, strlen("st"));
ASSERT_EQ(elements.tags, sql + elements.measureLen);
ASSERT_EQ(elements.tagsLen, 0); ASSERT_EQ(elements.tagsLen, 0);
ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); ASSERT_EQ(elements.cols, sql + 1 + elements.measureTagsLen + 3);
ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64"));
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
// case 6 // case 6
sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
// case 7 // case 7
sql = " st , "; tmp = " st , ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
sql++;
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3);
ASSERT_EQ(elements.colsLen, strlen(","));
// case 8 false // case 8 false
sql = ", st , "; tmp = ", st , ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(sql, &elements, &msgBuf); ret = smlParseInfluxString(sql, &elements, &msgBuf);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
taosMemoryFree(sql);
} }
TEST(testCase, smlParseCols_Error_Test) { TEST(testCase, smlParseCols_Error_Test) {
@ -188,7 +193,8 @@ TEST(testCase, smlParseCols_Error_Test) {
"c=-3.402823466e+39u64", "c=-3.402823466e+39u64",
"c=-339u64", "c=-339u64",
"c=18446744073709551616u64", "c=18446744073709551616u64",
"c=1,c=2" "c=1,c=2",
"c=1=2"
}; };
SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@ -198,9 +204,18 @@ TEST(testCase, smlParseCols_Error_Test) {
msgBuf.buf = msg; msgBuf.buf = msg;
msgBuf.len = 256; msgBuf.len = 256;
int32_t len = strlen(data[i]); int32_t len = strlen(data[i]);
int32_t ret = smlParseCols(data[i], len, NULL, false, dumplicateKey, &msgBuf); char *sql = (char*)taosMemoryCalloc(256, 1);
memcpy(sql, data[i], len + 1);
SArray *cols = taosArrayInit(8, POINTER_BYTES);
int32_t ret = smlParseCols(sql, len, cols, false, dumplicateKey, &msgBuf);
ASSERT_NE(ret, TSDB_CODE_SUCCESS); ASSERT_NE(ret, TSDB_CODE_SUCCESS);
taosHashClear(dumplicateKey); taosHashClear(dumplicateKey);
taosMemoryFree(sql);
for(int j = 0; j < taosArrayGetSize(cols); j++){
void *kv = taosArrayGetP(cols, j);
taosMemoryFree(kv);
}
taosArrayDestroy(cols);
} }
taosHashCleanup(dumplicateKey); taosHashCleanup(dumplicateKey);
} }
@ -216,7 +231,7 @@ TEST(testCase, smlParseCols_tag_Test) {
SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
const char *data = const char *data =
"cbin=\"passit helloc=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; "cbin=\"passit helloc\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\"";
int32_t len = strlen(data); int32_t len = strlen(data);
int32_t ret = smlParseCols(data, len, cols, true, dumplicateKey, &msgBuf); int32_t ret = smlParseCols(data, len, cols, true, dumplicateKey, &msgBuf);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS); ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
@ -228,9 +243,8 @@ TEST(testCase, smlParseCols_tag_Test) {
ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 17); ASSERT_EQ(kv->length, 15);
ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0);
taosMemoryFree(kv);
// nchar // nchar
kv = (SSmlKv *)taosArrayGetP(cols, 3); kv = (SSmlKv *)taosArrayGetP(cols, 3);
@ -239,11 +253,13 @@ TEST(testCase, smlParseCols_tag_Test) {
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 7); ASSERT_EQ(kv->length, 7);
ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0);
taosMemoryFree(kv);
for(int i = 0; i < size; i++){
void *tmp = taosArrayGetP(cols, i);
taosMemoryFree(tmp);
}
taosArrayClear(cols); taosArrayClear(cols);
// test tag is null // test tag is null
data = "t=3e"; data = "t=3e";
len = 0; len = 0;
@ -278,20 +294,22 @@ TEST(testCase, smlParseCols_Test) {
SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; const char *data = "cb\\=in=\"pass\\,it hello,c=2\",cnch=L\"ii\\=sdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\"";
int32_t len = strlen(data); int32_t len = strlen(data);
int32_t ret = smlParseCols(data, len, cols, false, dumplicateKey, &msgBuf); char *sql = (char*)taosMemoryCalloc(1024, 1);
memcpy(sql, data, len + 1);
int32_t ret = smlParseCols(sql, len, cols, false, dumplicateKey, &msgBuf);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS); ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
int32_t size = taosArrayGetSize(cols); int32_t size = taosArrayGetSize(cols);
ASSERT_EQ(size, 19); ASSERT_EQ(size, 19);
// binary // binary
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0);
ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
ASSERT_EQ(kv->length, 16); ASSERT_EQ(kv->length, 17);
ASSERT_EQ(strncasecmp(kv->value, "passit", 6), 0); ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0);
taosMemoryFree(kv); taosMemoryFree(kv);
// nchar // nchar
@ -299,8 +317,8 @@ TEST(testCase, smlParseCols_Test) {
ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 7); ASSERT_EQ(kv->length, 8);
ASSERT_EQ(strncasecmp(kv->value, "iisd", 4), 0); ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0);
taosMemoryFree(kv); taosMemoryFree(kv);
// bool // bool
@ -463,6 +481,7 @@ TEST(testCase, smlParseCols_Test) {
taosArrayDestroy(cols); taosArrayDestroy(cols);
taosHashCleanup(dumplicateKey); taosHashCleanup(dumplicateKey);
taosMemoryFree(sql);
} }
TEST(testCase, smlProcess_influx_Test) { TEST(testCase, smlProcess_influx_Test) {
@ -481,7 +500,7 @@ TEST(testCase, smlProcess_influx_Test) {
SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true);
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
const char *sql[11] = { const char *sql[] = {
"readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606400000000000", "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0 1451606400000000000",
"readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607400000000000", "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607400000000000",
"readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,heading=221,grade=0,fuel_consumption=25 1451608400000000000", "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,heading=221,grade=0,fuel_consumption=25 1451608400000000000",
@ -492,20 +511,22 @@ TEST(testCase, smlProcess_influx_Test) {
"readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609400000000000", "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609400000000000",
"readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629400000000000", "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629400000000000",
"stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=3,c4=4 1451629500000000000", "stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=3,c4=4 1451629500000000000",
"stable,t2=t2,t1=t1,t3=t3 c1=1,c3=3,c4=4 1451629600000000000" "stable,t2=t2,t1=t1,t3=t3 c1=1,c3=3,c4=4 1451629600000000000",
}; };
smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
ASSERT_EQ(ret, 0);
TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d"); TAOS_RES *res = taos_query(taos, "select * from t_6885c584b98481584ee13dac399e173d");
ASSERT_NE(res, nullptr); ASSERT_NE(res, nullptr);
int fieldNum = taos_field_count(res); // int fieldNum = taos_field_count(res);
ASSERT_EQ(fieldNum, 5); // ASSERT_EQ(fieldNum, 5);
int rowNum = taos_affected_rows(res); // int rowNum = taos_affected_rows(res);
ASSERT_EQ(rowNum, 2); // ASSERT_EQ(rowNum, 2);
for (int i = 0; i < rowNum; ++i) { // for (int i = 0; i < rowNum; ++i) {
TAOS_ROW rows = taos_fetch_row(res); // TAOS_ROW rows = taos_fetch_row(res);
} // }
taos_free_result(res); taos_free_result(res);
destroyRequest(request);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -526,7 +547,7 @@ TEST(testCase, smlParseLine_error_Test) {
SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true);
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
const char *sql[2] = { const char *sql[] = {
"measure,t1=3 c1=8", "measure,t1=3 c1=8",
"measure,t2=3 c1=8u8" "measure,t2=3 c1=8u8"
}; };
@ -575,37 +596,37 @@ TEST(testCase, smlProcess_telnet_Test) {
SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true);
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
const char *sql[5] = { const char *sql[] = {
"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", "sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0",
"sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01", "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ",
"sys.if.bytes.out 1479496102 1.3E3 network=tcp", "sys.if.bytes.out 1479496102 1.3E3 network=tcp",
"sys.procs.running 1479496100 42 host=web01 ", " sys.procs.running 1479496100 42 host=web01 "
" sys.procs.running 1479496200 42 host=web01=4"
}; };
int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a"); TAOS_RES *res = taos_query(taos, "select * from t_8c30283b3c4131a071d1e16cf6d7094a");
ASSERT_NE(res, nullptr); ASSERT_NE(res, nullptr);
int fieldNum = taos_field_count(res); // int fieldNum = taos_field_count(res);
ASSERT_EQ(fieldNum, 2); // ASSERT_EQ(fieldNum, 2);
int rowNum = taos_affected_rows(res); // int rowNum = taos_affected_rows(res);
ASSERT_EQ(rowNum, 1); // ASSERT_EQ(rowNum, 1);
for (int i = 0; i < rowNum; ++i) { // for (int i = 0; i < rowNum; ++i) {
TAOS_ROW rows = taos_fetch_row(res); // TAOS_ROW rows = taos_fetch_row(res);
} // }
taos_free_result(pRes); taos_free_result(res);
res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961"); // res = taos_query(taos, "select * from t_6931529054e5637ca92c78a1ad441961");
ASSERT_NE(res, nullptr); // ASSERT_NE(res, nullptr);
fieldNum = taos_field_count(res); // fieldNum = taos_field_count(res);
ASSERT_EQ(fieldNum, 2); // ASSERT_EQ(fieldNum, 2);
rowNum = taos_affected_rows(res); // rowNum = taos_affected_rows(res);
ASSERT_EQ(rowNum, 2); // ASSERT_EQ(rowNum, 2);
for (int i = 0; i < rowNum; ++i) { // for (int i = 0; i < rowNum; ++i) {
TAOS_ROW rows = taos_fetch_row(res); // TAOS_ROW rows = taos_fetch_row(res);
} // }
taos_free_result(pRes); // taos_free_result(res);
destroyRequest(request);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -658,7 +679,8 @@ TEST(testCase, smlProcess_json1_Test) {
// for (int i = 0; i < rowNum; ++i) { // for (int i = 0; i < rowNum; ++i) {
// TAOS_ROW rows = taos_fetch_row(res); // TAOS_ROW rows = taos_fetch_row(res);
// } // }
taos_free_result(pRes); taos_free_result(res);
destroyRequest(request);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -702,7 +724,7 @@ TEST(testCase, smlProcess_json2_Test) {
"}"; "}";
int32_t ret = smlProcess(info, (char **)(&sql), -1); int32_t ret = smlProcess(info, (char **)(&sql), -1);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
taos_free_result(pRes); destroyRequest(request);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -774,7 +796,7 @@ TEST(testCase, smlProcess_json3_Test) {
"}"; "}";
int32_t ret = smlProcess(info, (char **)(&sql), -1); int32_t ret = smlProcess(info, (char **)(&sql), -1);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
taos_free_result(pRes); destroyRequest(request);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -836,7 +858,7 @@ TEST(testCase, smlProcess_json4_Test) {
"}"; "}";
int32_t ret = smlProcess(info, (char**)(&sql), -1); int32_t ret = smlProcess(info, (char**)(&sql), -1);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
taos_free_result(pRes); destroyRequest(request);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -857,7 +879,7 @@ TEST(testCase, smlParseTelnetLine_error_Test) {
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
int32_t ret = 0; int32_t ret = 0;
const char *sql[19] = { const char *sql[] = {
"sys.procs.running 14794961040 42 host=web01", "sys.procs.running 14794961040 42 host=web01",
"sys.procs.running 14791040 42 host=web01", "sys.procs.running 14791040 42 host=web01",
"sys.procs.running erere 42 host=web01", "sys.procs.running erere 42 host=web01",
@ -877,6 +899,8 @@ TEST(testCase, smlParseTelnetLine_error_Test) {
"sys.procs.running 1479496100 42 host=web01 cpu= ", "sys.procs.running 1479496100 42 host=web01 cpu= ",
"sys.procs.running 1479496100 42 host=web01 host=w2", "sys.procs.running 1479496100 42 host=web01 host=w2",
"sys.procs.running 1479496100 42 host=web01 host", "sys.procs.running 1479496100 42 host=web01 host",
"sys.procs.running 1479496100 42 host=web01=er",
"sys.procs.running 1479496100 42 host= web01",
}; };
for(int i = 0; i < sizeof(sql)/sizeof(sql[0]); i++){ for(int i = 0; i < sizeof(sql)/sizeof(sql[0]); i++){
ret = smlParseTelnetLine(info, (void*)sql[i]); ret = smlParseTelnetLine(info, (void*)sql[i]);

2
source/common/src/tglobal.c
View File

@ -299,6 +299,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1; if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1;
return 0; return 0;
} }
@ -480,6 +481,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32;
smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32;
} }
static int32_t taosSetClientCfg(SConfig *pCfg) { static int32_t taosSetClientCfg(SConfig *pCfg) {

107
source/common/src/tmsg.c
View File

@ -3562,39 +3562,92 @@ int tDecodeSVCreateTbBatchRsp(SDecoder *pCoder, SVCreateTbBatchRsp *pRsp) {
return 0; return 0;
} }
int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) { int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) {
int32_t tlen = 0; if (tEncodeI8(pCoder, pSma->version) < 0) return -1;
if (tEncodeI8(pCoder, pSma->intervalUnit) < 0) return -1;
tlen += taosEncodeFixedI64(buf, pReq->ver); if (tEncodeI8(pCoder, pSma->slidingUnit) < 0) return -1;
tlen += tEncodeTSma(buf, &pReq->tSma); if (tEncodeI8(pCoder, pSma->timezoneInt) < 0) return -1;
if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1;
return tlen; if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1;
} if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1;
if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1;
void *tDeserializeSVCreateTSmaReq(void *buf, SVCreateTSmaReq *pReq) { if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1;
buf = taosDecodeFixedI64(buf, &(pReq->ver)); if (tEncodeI64(pCoder, pSma->interval) < 0) return -1;
if (tEncodeI64(pCoder, pSma->offset) < 0) return -1;
if ((buf = tDecodeTSma(buf, &pReq->tSma)) == NULL) { if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1;
tdDestroyTSma(&pReq->tSma); if (pSma->exprLen > 0) {
if (tEncodeCStr(pCoder, pSma->expr) < 0) return -1;
} }
return buf; if (pSma->tagsFilterLen > 0) {
if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1;
}
return 0;
} }
int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) { int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) {
int32_t tlen = 0; if (tDecodeI8(pCoder, &pSma->version) < 0) return -1;
if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1;
if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1;
if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1;
if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1;
if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1;
if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1;
if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1;
if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1;
if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1;
if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1;
if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1;
if (pSma->exprLen > 0) {
if (tDecodeCStr(pCoder, &pSma->expr) < 0) return -1;
} else {
pSma->expr = NULL;
}
if (pSma->tagsFilterLen > 0) {
if (tDecodeCStr(pCoder, &pSma->tagsFilter) < 0) return -1;
} else {
pSma->tagsFilter = NULL;
}
tlen += taosEncodeFixedI64(buf, pReq->ver); return 0;
tlen += taosEncodeFixedI64(buf, pReq->indexUid);
tlen += taosEncodeString(buf, pReq->indexName);
return tlen;
} }
void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
buf = taosDecodeFixedI64(buf, &(pReq->ver));
buf = taosDecodeFixedI64(buf, &(pReq->indexUid));
buf = taosDecodeStringTo(buf, pReq->indexName);
return buf; int32_t tEncodeSVCreateTSmaReq(SEncoder *pCoder, const SVCreateTSmaReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
tEncodeTSma(pCoder, pReq);
tEndEncode(pCoder);
return 0;
}
int32_t tDecodeSVCreateTSmaReq(SDecoder *pCoder, SVCreateTSmaReq *pReq) {
if (tStartDecode(pCoder) < 0) return -1;
tDecodeTSma(pCoder, pReq);
tEndDecode(pCoder);
return 0;
}
int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1;
if (tEncodeCStr(pCoder, pReq->indexName) < 0) return -1;
tEndEncode(pCoder);
return 0;
}
int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
tEndDecode(pCoder);
return 0;
} }
int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) { int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) {

282
source/dnode/mnode/impl/src/mndDef.c
View File

@ -17,6 +17,147 @@
#include "mndDef.h" #include "mndDef.h"
#include "mndConsumer.h" #include "mndConsumer.h"
int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
int32_t sz = 0;
/*int32_t outputNameSz = 0;*/
if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
/*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
// TODO encode tasks
if (pObj->tasks) {
sz = taosArrayGetSize(pObj->tasks);
}
if (tEncodeI32(pEncoder, sz) < 0) return -1;
for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGetP(pObj->tasks, i);
int32_t innerSz = taosArrayGetSize(pArray);
if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGetP(pArray, j);
if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1;
}
}
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
#if 0
if (pObj->ColAlias != NULL) {
outputNameSz = taosArrayGetSize(pObj->ColAlias);
}
if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1;
for (int32_t i = 0; i < outputNameSz; i++) {
char *name = taosArrayGetP(pObj->ColAlias, i);
if (tEncodeCStr(pEncoder, name) < 0) return -1;
}
#endif
return pEncoder->pos;
}
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
/*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
pObj->tasks = NULL;
int32_t sz;
if (tDecodeI32(pDecoder, &sz) < 0) return -1;
if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(void *));
for (int32_t i = 0; i < sz; i++) {
int32_t innerSz;
if (tDecodeI32(pDecoder, &innerSz) < 0) return -1;
SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) return -1;
if (tDecodeSStreamTask(pDecoder, pTask) < 0) return -1;
taosArrayPush(pArray, &pTask);
}
taosArrayPush(pObj->tasks, &pArray);
}
}
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
#if 0
int32_t outputNameSz;
if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1;
if (outputNameSz != 0) {
pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *));
if (pObj->ColAlias == NULL) {
return -1;
}
}
for (int32_t i = 0; i < outputNameSz; i++) {
char *name;
if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1;
taosArrayPush(pObj->ColAlias, &name);
}
#endif
return 0;
}
SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
if (pVgEpNew == NULL) return NULL;
pVgEpNew->vgId = pVgEp->vgId;
pVgEpNew->qmsg = strdup(pVgEp->qmsg);
pVgEpNew->epSet = pVgEp->epSet;
return pVgEpNew;
}
void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg);
}
int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
tlen += taosEncodeString(buf, pVgEp->qmsg);
tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
return tlen;
}
void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
buf = taosDecodeString(buf, &pVgEp->qmsg);
buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
return (void *)buf;
}
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) { SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) {
SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
if (pConsumer == NULL) { if (pConsumer == NULL) {
@ -187,34 +328,6 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
return (void *)buf; return (void *)buf;
} }
SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
if (pVgEpNew == NULL) return NULL;
pVgEpNew->vgId = pVgEp->vgId;
pVgEpNew->qmsg = strdup(pVgEp->qmsg);
pVgEpNew->epSet = pVgEp->epSet;
return pVgEpNew;
}
void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg);
}
int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
tlen += taosEncodeString(buf, pVgEp->qmsg);
tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
return tlen;
}
void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
buf = taosDecodeString(buf, &pVgEp->qmsg);
buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
return (void *)buf;
}
SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) { SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp)); SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
if (pConsumerEpNew == NULL) return NULL; if (pConsumerEpNew == NULL) return NULL;
@ -413,119 +526,6 @@ void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
return (void *)buf; return (void *)buf;
} }
int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
int32_t sz = 0;
/*int32_t outputNameSz = 0;*/
if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
/*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
// TODO encode tasks
if (pObj->tasks) {
sz = taosArrayGetSize(pObj->tasks);
}
if (tEncodeI32(pEncoder, sz) < 0) return -1;
for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGetP(pObj->tasks, i);
int32_t innerSz = taosArrayGetSize(pArray);
if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGetP(pArray, j);
if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1;
}
}
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
#if 0
if (pObj->ColAlias != NULL) {
outputNameSz = taosArrayGetSize(pObj->ColAlias);
}
if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1;
for (int32_t i = 0; i < outputNameSz; i++) {
char *name = taosArrayGetP(pObj->ColAlias, i);
if (tEncodeCStr(pEncoder, name) < 0) return -1;
}
#endif
return pEncoder->pos;
}
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
/*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
pObj->tasks = NULL;
int32_t sz;
if (tDecodeI32(pDecoder, &sz) < 0) return -1;
if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(void *));
for (int32_t i = 0; i < sz; i++) {
int32_t innerSz;
if (tDecodeI32(pDecoder, &innerSz) < 0) return -1;
SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) return -1;
if (tDecodeSStreamTask(pDecoder, pTask) < 0) return -1;
taosArrayPush(pArray, &pTask);
}
taosArrayPush(pObj->tasks, &pArray);
}
}
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
#if 0
int32_t outputNameSz;
if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1;
if (outputNameSz != 0) {
pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *));
if (pObj->ColAlias == NULL) {
return -1;
}
}
for (int32_t i = 0; i < outputNameSz; i++) {
char *name;
if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1;
taosArrayPush(pObj->ColAlias, &name);
}
#endif
return 0;
}
int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeString(buf, pOffset->key); tlen += taosEncodeString(buf, pOffset->key);

5
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -194,6 +194,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p
// source // source
pTask->sourceType = TASK_SOURCE__MERGE; pTask->sourceType = TASK_SOURCE__MERGE;
pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// exec // exec
pTask->execType = TASK_EXEC__NONE; pTask->execType = TASK_EXEC__NONE;
@ -235,6 +236,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr
pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
// source // source
pTask->sourceType = TASK_SOURCE__MERGE; pTask->sourceType = TASK_SOURCE__MERGE;
pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// exec // exec
pTask->execType = TASK_EXEC__NONE; pTask->execType = TASK_EXEC__NONE;
@ -309,6 +311,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SStreamTask* pTask = tNewSStreamTask(pStream->uid); SStreamTask* pTask = tNewSStreamTask(pStream->uid);
// source part // source part
pTask->sourceType = TASK_SOURCE__SCAN; pTask->sourceType = TASK_SOURCE__SCAN;
pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
// sink part // sink part
if (level == 0) { if (level == 0) {
@ -372,6 +375,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
// source part, currently only support multi source // source part, currently only support multi source
pTask->sourceType = TASK_SOURCE__PIPE; pTask->sourceType = TASK_SOURCE__PIPE;
pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// sink part // sink part
pTask->sinkType = TASK_SINK__NONE; pTask->sinkType = TASK_SINK__NONE;
@ -459,6 +463,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
// source part // source part
pTask->sourceType = TASK_SOURCE__MERGE; pTask->sourceType = TASK_SOURCE__MERGE;
pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// sink part // sink part
pTask->sinkType = TASK_SINK__NONE; pTask->sinkType = TASK_SINK__NONE;

72
source/dnode/mnode/impl/src/mndSma.c
View File

@ -242,26 +242,35 @@ SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) {
} }
static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) {
SName name = {0}; SEncoder encoder = {0};
int32_t contLen = 0;
SName name = {0};
tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
SVCreateTSmaReq req = {0}; SVCreateTSmaReq req = {0};
req.tSma.version = 0; req.version = 0;
req.tSma.intervalUnit = pSma->intervalUnit; req.intervalUnit = pSma->intervalUnit;
req.tSma.slidingUnit = pSma->slidingUnit; req.slidingUnit = pSma->slidingUnit;
req.tSma.timezoneInt = pSma->timezone; req.timezoneInt = pSma->timezone;
tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
req.tSma.exprLen = pSma->exprLen; req.exprLen = pSma->exprLen;
req.tSma.tagsFilterLen = pSma->tagsFilterLen; req.tagsFilterLen = pSma->tagsFilterLen;
req.tSma.indexUid = pSma->uid; req.indexUid = pSma->uid;
req.tSma.tableUid = pSma->stbUid; req.tableUid = pSma->stbUid;
req.tSma.interval = pSma->interval; req.interval = pSma->interval;
req.tSma.offset = pSma->offset; req.offset = pSma->offset;
req.tSma.sliding = pSma->sliding; req.sliding = pSma->sliding;
req.tSma.expr = pSma->expr; req.expr = pSma->expr;
req.tSma.tagsFilter = pSma->tagsFilter; req.tagsFilter = pSma->tagsFilter;
// get length
int32_t ret = 0;
tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret);
if (ret < 0) {
return NULL;
}
contLen += sizeof(SMsgHead);
int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead);
SMsgHead *pHead = taosMemoryMalloc(contLen); SMsgHead *pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) { if (pHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -272,22 +281,38 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm
pHead->vgId = htonl(pVgroup->vgId); pHead->vgId = htonl(pVgroup->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tSerializeSVCreateTSmaReq(&pBuf, &req); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
if (tEncodeSVCreateTSmaReq(&encoder, &req) < 0) {
taosMemoryFreeClear(pHead);
tEncoderClear(&encoder);
return NULL;
}
tEncoderClear(&encoder);
*pContLen = contLen; *pContLen = contLen;
return pHead; return pHead;
} }
static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) { static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) {
SEncoder encoder = {0};
int32_t contLen;
SName name = {0}; SName name = {0};
tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
SVDropTSmaReq req = {0}; SVDropTSmaReq req = {0};
req.ver = 0;
req.indexUid = pSma->uid; req.indexUid = pSma->uid;
tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead); // get length
int32_t ret = 0;
tEncodeSize(tEncodeSVDropTSmaReq, &req, contLen, ret);
if (ret < 0) {
return NULL;
}
contLen += sizeof(SMsgHead);
SMsgHead *pHead = taosMemoryMalloc(contLen); SMsgHead *pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) { if (pHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -298,7 +323,14 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma,
pHead->vgId = htonl(pVgroup->vgId); pHead->vgId = htonl(pVgroup->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tDeserializeSVDropTSmaReq(&pBuf, &req); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
if (tEncodeSVDropTSmaReq(&encoder, &req) < 0) {
taosMemoryFreeClear(pHead);
tEncoderClear(&encoder);
return NULL;
}
tEncoderClear(&encoder);
*pContLen = contLen; *pContLen = contLen;
return pHead; return pHead;

4
source/dnode/mnode/impl/src/mndStb.c
View File

@ -425,6 +425,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead)); tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
if (tEncodeSVCreateStbReq(&encoder, &req) < 0) { if (tEncodeSVCreateStbReq(&encoder, &req) < 0) {
taosMemoryFreeClear(pHead);
taosMemoryFreeClear(req.pRSmaParam.qmsg1);
taosMemoryFreeClear(req.pRSmaParam.qmsg2);
tEncoderClear(&encoder);
return NULL; return NULL;
} }
tEncoderClear(&encoder); tEncoderClear(&encoder);

2
source/dnode/mnode/impl/test/trans/trans2.cpp
View File

@ -510,4 +510,4 @@ TEST_F(MndTestTrans2, 04_Conflict) {
ASSERT_EQ(pUser, nullptr); ASSERT_EQ(pUser, nullptr);
mndReleaseUser(pMnode, pUser); mndReleaseUser(pMnode, pUser);
} }
} }

13
source/dnode/vnode/CMakeLists.txt
View File

@ -18,12 +18,21 @@ target_sources(
"src/meta/metaOpen.c" "src/meta/metaOpen.c"
"src/meta/metaIdx.c" "src/meta/metaIdx.c"
"src/meta/metaTable.c" "src/meta/metaTable.c"
"src/meta/metaSma.c"
"src/meta/metaQuery.c" "src/meta/metaQuery.c"
"src/meta/metaCommit.c" "src/meta/metaCommit.c"
"src/meta/metaEntry.c" "src/meta/metaEntry.c"
# sma
"src/sma/sma.c"
"src/sma/smaTDBImpl.c"
"src/sma/smaEnv.c"
"src/sma/smaOpen.c"
"src/sma/smaRollup.c"
"src/sma/smaTimeRange.c"
# tsdb # tsdb
"src/tsdb/tsdbTDBImpl.c" # "src/tsdb/tsdbTDBImpl.c"
"src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit.c"
"src/tsdb/tsdbCommit2.c" "src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbFile.c" "src/tsdb/tsdbFile.c"
@ -33,7 +42,7 @@ target_sources(
"src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbMemTable2.c"
"src/tsdb/tsdbRead.c" "src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbSma.c" # "src/tsdb/tsdbSma.c"
"src/tsdb/tsdbWrite.c" "src/tsdb/tsdbWrite.c"
# tq # tq

3
source/dnode/vnode/inc/vnode.h
View File

@ -191,6 +191,9 @@ struct SMetaEntry {
int32_t ttlDays; int32_t ttlDays;
SSchemaWrapper schema; SSchemaWrapper schema;
} ntbEntry; } ntbEntry;
struct {
STSma *tsma;
} smaEntry;
}; };
}; };

8
source/dnode/vnode/src/inc/meta.h
View File

@ -73,6 +73,7 @@ struct SMeta {
TDB* pCtbIdx; TDB* pCtbIdx;
TDB* pTagIdx; TDB* pTagIdx;
TDB* pTtlIdx; TDB* pTtlIdx;
TDB* pSmaIdx;
SMetaIdx* pIdx; SMetaIdx* pIdx;
}; };
@ -108,6 +109,11 @@ typedef struct {
tb_uid_t uid; tb_uid_t uid;
} STtlIdxKey; } STtlIdxKey;
typedef struct {
tb_uid_t uid;
int64_t smaUid;
} SSmaIdxKey;
#if 1 #if 1
SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
@ -118,7 +124,7 @@ int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur);
// SMetaDB // SMetaDB
int metaOpenDB(SMeta* pMeta); int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta);
// int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);

225
source/dnode/vnode/src/inc/sma.h Normal file
View File

@ -0,0 +1,225 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_SMA_H_
#define _TD_VNODE_SMA_H_
#include "vnodeInt.h"
#ifdef __cplusplus
extern "C" {
#endif
// smaDebug ================
// clang-format off
#define smaFatal(...) do { if (smaDebugFlag & DEBUG_FATAL) { taosPrintLog("SMA FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
#define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
#define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
typedef struct SSmaEnv SSmaEnv;
typedef struct SSmaStat SSmaStat;
typedef struct SSmaStatItem SSmaStatItem;
typedef struct SSmaKey SSmaKey;
typedef struct SRSmaInfo SRSmaInfo;
#define SMA_IVLD_FID INT_MIN
struct SSmaEnv {
TdThreadRwlock lock;
int8_t type;
TXN txn;
void *pPool; // SPoolMem
SDiskID did;
TENV *dbEnv; // TODO: If it's better to put it in smaIndex level?
char *path; // relative path
SSmaStat *pStat;
};
#define SMA_ENV_LOCK(env) ((env)->lock)
#define SMA_ENV_TYPE(env) ((env)->type)
#define SMA_ENV_DID(env) ((env)->did)
#define SMA_ENV_ENV(env) ((env)->dbEnv)
#define SMA_ENV_PATH(env) ((env)->path)
#define SMA_ENV_STAT(env) ((env)->pStat)
#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
struct SSmaStatItem {
/**
* @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service.
* - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from
* Streaming Module or TSDB local persistence.
* - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
* without information about its previous state.
* - TSDB_SMA_STAT_DROPPED: 1)sma dropped
* N.B. only applicable to tsma
*/
int8_t state; // ETsdbSmaStat
SHashObj *expiredWindows; // key: skey of time window, value: N/A
STSma *pTSma; // cache schema
};
struct SSmaStat {
union {
SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem for tsma
SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo;
};
T_REF_DECLARE()
};
#define SMA_STAT_ITEMS(s) ((s)->smaStatItems)
#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash)
struct SSmaKey {
TSKEY skey;
int64_t groupId;
};
typedef struct SDBFile SDBFile;
struct SDBFile {
int32_t fid;
TDB *pDB;
char *path;
};
int32_t tdSmaBeginCommit(SSmaEnv *pEnv);
int32_t tdSmaEndCommit(SSmaEnv *pEnv);
int32_t smaOpenDBEnv(TENV **ppEnv, const char *path);
int32_t smaCloseDBEnv(TENV *pEnv);
int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF);
int32_t smaCloseDBF(SDBFile *pDBF);
int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
void tdDestroySmaEnv(SSmaEnv *pSmaEnv);
void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
#if 0
int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result);
int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin);
#endif
static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
int32_t len = 0;
len += taosEncodeFixedI64(pData, tsKey);
len += taosEncodeFixedI64(pData, groupId);
return len;
}
int32_t tdInitSma(SSma *pSma);
int32_t tdDropTSma(SSma *pSma, char *pMsg);
int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid);
int32_t tdInsertRSmaData(SSma *pSma, char *msg);
int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType);
int32_t tdLockSma(SSma *pSma);
int32_t tdUnLockSma(SSma *pSma);
int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); }
static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); }
static FORCE_INLINE int32_t tdRLockSmaEnv(SSmaEnv *pEnv) {
int code = taosThreadRwlockRdlock(&(pEnv->lock));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
static FORCE_INLINE int32_t tdWLockSmaEnv(SSmaEnv *pEnv) {
int code = taosThreadRwlockWrlock(&(pEnv->lock));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) {
int code = taosThreadRwlockUnlock(&(pEnv->lock));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) {
if (pStatItem) {
return atomic_load_8(&pStatItem->state);
}
return TSDB_SMA_STAT_UNKNOWN;
}
static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) {
if (!pStatItem) {
return false;
}
if (state) {
*state = atomic_load_8(&pStatItem->state);
return *state == TSDB_SMA_STAT_OK;
}
return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK;
}
static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) {
return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true;
}
static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) {
return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true;
}
static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) {
if (pStatItem) {
atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK);
}
}
static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) {
if (pStatItem) {
atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED);
}
}
static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) {
if (pStatItem) {
atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED);
}
}
static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType);
void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem);
static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did);
static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv);
void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_SMA_H_*/

80
source/dnode/vnode/src/inc/tsdb.h
View File

@ -60,31 +60,22 @@ typedef struct {
TSKEY minKey; TSKEY minKey;
} SRtn; } SRtn;
struct SSmaEnvs { #define TSDB_DATA_DIR_LEN 6
int16_t nTSma;
int16_t nRSma;
SSmaEnv *pTSmaEnv;
SSmaEnv *pRSmaEnv;
};
struct STsdb { struct STsdb {
char *path; char *path;
SVnode *pVnode; SVnode *pVnode;
TdThreadMutex mutex; TdThreadMutex mutex;
char dir[TSDB_DATA_DIR_LEN];
bool repoLocked; bool repoLocked;
int8_t level; // retention level
STsdbKeepCfg keepCfg; STsdbKeepCfg keepCfg;
STsdbMemTable *mem; STsdbMemTable *mem;
STsdbMemTable *imem; STsdbMemTable *imem;
SRtn rtn; SRtn rtn;
STsdbFS *fs; STsdbFS *fs;
SSmaEnvs smaEnvs;
}; };
#if 1 // ====================================== #if 1 // ======================================
typedef struct SSmaStat SSmaStat;
struct STable { struct STable {
uint64_t tid; uint64_t tid;
uint64_t uid; uint64_t uid;
@ -95,10 +86,6 @@ struct STable {
#define TABLE_UID(t) (t)->uid #define TABLE_UID(t) (t)->uid
int tsdbPrepareCommit(STsdb *pTsdb); int tsdbPrepareCommit(STsdb *pTsdb);
int32_t tsdbInitSma(STsdb *pTsdb);
int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
typedef enum { typedef enum {
TSDB_FILE_HEAD = 0, // .head TSDB_FILE_HEAD = 0, // .head
TSDB_FILE_DATA, // .data TSDB_FILE_DATA, // .data
@ -107,8 +94,6 @@ typedef enum {
TSDB_FILE_SMAL, // .smal(Block-wise SMA) TSDB_FILE_SMAL, // .smal(Block-wise SMA)
TSDB_FILE_MAX, // TSDB_FILE_MAX, //
TSDB_FILE_META, // meta TSDB_FILE_META, // meta
TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA
TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
} E_TSDB_FILE_T; } E_TSDB_FILE_T;
typedef struct { typedef struct {
@ -186,15 +171,10 @@ struct STsdbFS {
#define REPO_ID(r) TD_VID((r)->pVnode) #define REPO_ID(r) TD_VID((r)->pVnode)
#define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) #define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg)
#define REPO_KEEP_CFG(r) (&(r)->keepCfg) #define REPO_KEEP_CFG(r) (&(r)->keepCfg)
#define REPO_LEVEL(r) ((r)->level)
#define REPO_FS(r) ((r)->fs) #define REPO_FS(r) ((r)->fs)
#define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_META(r) ((r)->pVnode->pMeta)
#define REPO_TFS(r) ((r)->pVnode->pTfs) #define REPO_TFS(r) ((r)->pVnode->pTfs)
#define IS_REPO_LOCKED(r) ((r)->repoLocked) #define IS_REPO_LOCKED(r) ((r)->repoLocked)
#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma)
#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma)
#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv)
#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv)
int tsdbLockRepo(STsdb *pTsdb); int tsdbLockRepo(STsdb *pTsdb);
int tsdbUnlockRepo(STsdb *pTsdb); int tsdbUnlockRepo(STsdb *pTsdb);
@ -794,25 +774,6 @@ typedef struct {
} SFSHeader; } SFSHeader;
// ================== TSDB File System Meta // ================== TSDB File System Meta
/**
* @brief Directory structure of .tsma data files.
*
* /vnode2/tsdb $ tree tsma/
* tsma/
* ├── v2f100.index_name_1
* ├── v2f101.index_name_1
* ├── v2f102.index_name_1
* ├── v2f1900.index_name_3
* ├── v2f1901.index_name_3
* ├── v2f1902.index_name_3
* ├── v2f200.index_name_2
* ├── v2f201.index_name_2
* └── v2f202.index_name_2
*
* 0 directories, 9 files
*/
#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus) #define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
#define FS_NEW_STATUS(pfs) ((pfs)->nstatus) #define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
#define FS_IN_TXN(pfs) (pfs)->intxn #define FS_IN_TXN(pfs) (pfs)->intxn
@ -874,43 +835,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
return 0; return 0;
} }
typedef struct SSmaKey SSmaKey;
struct SSmaKey {
TSKEY skey;
int64_t groupId;
};
typedef struct SDBFile SDBFile;
struct SDBFile {
int32_t fid;
TDB *pDB;
char *path;
};
int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path);
int32_t tsdbCloseDBEnv(TENV *pEnv);
int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF);
int32_t tsdbCloseDBF(SDBFile *pDBF);
int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
#if 0
int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
#endif
// internal func
static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
int32_t len = 0;
len += taosEncodeFixedI64(pData, tsKey);
len += taosEncodeFixedI64(pData, groupId);
return len;
}
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus

80
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -47,13 +47,15 @@
extern "C" { extern "C" {
#endif #endif
typedef struct SVnodeInfo SVnodeInfo; typedef struct SVnodeInfo SVnodeInfo;
typedef struct SMeta SMeta; typedef struct SMeta SMeta;
typedef struct STsdb STsdb; typedef struct SSma SSma;
typedef struct STQ STQ; typedef struct STsdb STsdb;
typedef struct SVState SVState; typedef struct STQ STQ;
typedef struct SVBufPool SVBufPool; typedef struct SVState SVState;
typedef struct SQWorker SQHandle; typedef struct SVBufPool SVBufPool;
typedef struct SQWorker SQHandle;
typedef struct STsdbKeepCfg STsdbKeepCfg;
#define VNODE_META_DIR "meta" #define VNODE_META_DIR "meta"
#define VNODE_TSDB_DIR "tsdb" #define VNODE_TSDB_DIR "tsdb"
@ -90,17 +92,14 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup); SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup);
void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode); void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid); STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
// tsdb // tsdb
int tsdbOpen(SVnode* pVnode, int8_t type); int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg *pKeepCfg);
int tsdbClose(STsdb* pTsdb); int tsdbClose(STsdb** pTsdb);
int tsdbBegin(STsdb* pTsdb); int tsdbBegin(STsdb* pTsdb);
int tsdbCommit(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb);
int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);
tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
@ -121,13 +120,31 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
// sma // sma
int32_t smaOpen(SVnode* pVnode);
int32_t smaClose(SSma* pSma);
int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version);
int32_t tdProcessTSmaCreate(SSma* pSma, char* pMsg);
int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
int32_t tdProcessRSmaCreate(SSma* pSma, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);
int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
void tdUidStoreDestory(STbUidStore* pStore);
void* tdUidStoreFree(STbUidStore* pStore);
#if 0
int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb); int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
void tsdbUidStoreDestory(STbUidStore* pStore); void tsdbUidStoreDestory(STbUidStore* pStore);
void* tsdbUidStoreFree(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore);
int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType);
#endif
typedef struct { typedef struct {
int8_t streamType; // sma or other int8_t streamType; // sma or other
@ -164,13 +181,13 @@ typedef enum {
TSDB_TYPE_RSMA_L2 = 4, // RSMA Level 2 TSDB_TYPE_RSMA_L2 = 4, // RSMA Level 2
} ETsdbType; } ETsdbType;
typedef struct { struct STsdbKeepCfg{
int8_t precision; // precision always be used with below keep cfgs int8_t precision; // precision always be used with below keep cfgs
int32_t days; int32_t days;
int32_t keep0; int32_t keep0;
int32_t keep1; int32_t keep1;
int32_t keep2; int32_t keep2;
} STsdbKeepCfg; };
struct SVnode { struct SVnode {
char* path; char* path;
@ -183,9 +200,8 @@ struct SVnode {
SVBufPool* onCommit; SVBufPool* onCommit;
SVBufPool* onRecycle; SVBufPool* onRecycle;
SMeta* pMeta; SMeta* pMeta;
SSma* pSma;
STsdb* pTsdb; STsdb* pTsdb;
STsdb* pRSma1;
STsdb* pRSma2;
SWal* pWal; SWal* pWal;
STQ* pTq; STQ* pTq;
SSink* pSink; SSink* pSink;
@ -194,10 +210,12 @@ struct SVnode {
SQHandle* pQuery; SQHandle* pQuery;
}; };
#define TD_VID(PVNODE) (PVNODE)->config.vgId
#define VND_TSDB(vnd) ((vnd)->pTsdb) #define VND_TSDB(vnd) ((vnd)->pTsdb)
#define VND_RSMA0(vnd) ((vnd)->pTsdb) #define VND_RSMA0(vnd) ((vnd)->pTsdb)
#define VND_RSMA1(vnd) ((vnd)->pRSma1) #define VND_RSMA1(vnd) ((vnd)->pSma->pRSmaTsdb1)
#define VND_RSMA2(vnd) ((vnd)->pRSma2) #define VND_RSMA2(vnd) ((vnd)->pSma->pRSmaTsdb2)
#define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions) #define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions)
struct STbUidStore { struct STbUidStore {
@ -207,7 +225,29 @@ struct STbUidStore {
SHashObj* uidHash; SHashObj* uidHash;
}; };
#define TD_VID(PVNODE) (PVNODE)->config.vgId struct SSma {
int16_t nTSma;
bool locked;
TdThreadMutex mutex;
SVnode* pVnode;
STsdb* pRSmaTsdb1;
STsdb* pRSmaTsdb2;
void* pTSmaEnv;
void* pRSmaEnv;
};
#define SMA_CFG(s) (&(s)->pVnode->config)
#define SMA_TSDB_CFG(s) (&(s)->pVnode->config.tsdbCfg)
#define SMA_LOCKED(s) ((s)->locked)
#define SMA_META(s) ((s)->pVnode->pMeta)
#define SMA_VID(s) TD_VID((s)->pVnode)
#define SMA_TFS(s) ((s)->pVnode->pTfs)
#define SMA_TSMA_NUM(s) ((s)->nTSma)
#define SMA_TSMA_ENV(s) ((s)->pTSmaEnv)
#define SMA_RSMA_ENV(s) ((s)->pRSmaEnv)
#define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb)
#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1)
#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2)
static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) {
SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]);

6
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -35,6 +35,8 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
} else if (pME->type == TSDB_TSMA_TABLE) {
if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
} else { } else {
ASSERT(0); ASSERT(0);
} }
@ -64,7 +66,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
} else { } else if (pME->type == TSDB_TSMA_TABLE) {
if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
} else {
ASSERT(0); ASSERT(0);
} }

31
source/dnode/vnode/src/meta/metaIdx.c
View File

@ -112,35 +112,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
#endif #endif
// TODO // TODO
return 0; return 0;
}
int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
// TODO: Validate the cfg
// The table uid should exists and be super table or common table.
// Check other cfg value
// TODO: add atomicity
#ifdef META_REFACT
#else
if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) {
// TODO: handle error
return -1;
}
#endif
return TSDB_CODE_SUCCESS;
}
int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
// TODO: Validate the cfg
// TODO: add atomicity
#ifdef META_REFACT
#else
if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
// TODO: handle error
return -1;
}
#endif
return TSDB_CODE_SUCCESS;
} }

33
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -21,6 +21,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); } static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); }
static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); } static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); }
@ -104,6 +105,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
goto _err; goto _err;
} }
// open pSmaIdx
ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx);
if (ret < 0) {
metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open index // open index
if (metaOpenIdx(pMeta) < 0) { if (metaOpenIdx(pMeta) < 0) {
metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
@ -117,11 +125,12 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
_err: _err:
if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
@ -133,11 +142,12 @@ _err:
int metaClose(SMeta *pMeta) { int metaClose(SMeta *pMeta) {
if (pMeta) { if (pMeta) {
if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx); if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
@ -295,3 +305,22 @@ static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
return 0; return 0;
} }
static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
SSmaIdxKey *pSmaIdxKey1 = (SSmaIdxKey *)pKey1;
SSmaIdxKey *pSmaIdxKey2 = (SSmaIdxKey *)pKey2;
if (pSmaIdxKey1->uid > pSmaIdxKey2->uid) {
return 1;
} else if (pSmaIdxKey1->uid < pSmaIdxKey2->uid) {
return -1;
}
if (pSmaIdxKey1->smaUid > pSmaIdxKey2->smaUid) {
return 1;
} else if (pSmaIdxKey1->smaUid < pSmaIdxKey2->smaUid) {
return -1;
}
return 0;
}

208
source/dnode/vnode/src/meta/metaSma.c Normal file
View File

@ -0,0 +1,208 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME);
int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
// TODO: Validate the cfg
// The table uid should exists and be super table or normal table.
// Check other cfg value
SMetaEntry me = {0};
int kLen = 0;
int vLen = 0;
const void *pKey = NULL;
const void *pVal = NULL;
void *pBuf = NULL;
int32_t szBuf = 0;
void *p = NULL;
SMetaReader mr = {0};
// validate req
metaReaderInit(&mr, pMeta, 0);
if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) {
// TODO: just for pass case
#if 1
terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST;
metaReaderClear(&mr);
return -1;
#else
metaReaderClear(&mr);
return 0;
#endif
}
metaReaderClear(&mr);
// set structs
me.version = version;
me.type = TSDB_TSMA_TABLE;
me.uid = pCfg->indexUid;
me.name = pCfg->indexName;
me.smaEntry.tsma = pCfg;
if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err;
metaDebug("vgId:%d tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid);
return 0;
_err:
metaError("vgId:%d failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName,
pCfg->indexUid, tstrerror(terrno));
return -1;
}
int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
// TODO: Validate the cfg
// TODO: add atomicity
#ifdef META_REFACT
#else
if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
// TODO: handle error
return -1;
}
#endif
return TSDB_CODE_SUCCESS;
}
// static int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
// int32_t ret = 0;
// void *pBuf = NULL, *qBuf = NULL;
// void *key = {0}, *val = {0};
// // save sma info
// int32_t len = tEncodeTSma(NULL, pSmaCfg);
// pBuf = taosMemoryCalloc(1, len);
// if (pBuf == NULL) {
// terrno = TSDB_CODE_OUT_OF_MEMORY;
// return -1;
// }
// key = (void *)&pSmaCfg->indexUid;
// qBuf = pBuf;
// tEncodeTSma(&qBuf, pSmaCfg);
// val = pBuf;
// int32_t kLen = sizeof(pSmaCfg->indexUid);
// int32_t vLen = POINTER_DISTANCE(qBuf, pBuf);
// ret = tdbDbInsert(pMeta->pTbDb, key, kLen, val, vLen, &pMeta->txn);
// if (ret < 0) {
// taosMemoryFreeClear(pBuf);
// return -1;
// }
// // add sma idx
// SSmaIdxKey smaIdxKey;
// smaIdxKey.uid = pSmaCfg->tableUid;
// smaIdxKey.smaUid = pSmaCfg->indexUid;
// key = &smaIdxKey;
// kLen = sizeof(smaIdxKey);
// val = NULL;
// vLen = 0;
// ret = tdbDbInsert(pMeta->pSmaIdx, key, kLen, val, vLen, &pMeta->txn);
// if (ret < 0) {
// taosMemoryFreeClear(pBuf);
// return -1;
// }
// // release
// taosMemoryFreeClear(pBuf);
// return 0;
// }
static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
STbDbKey tbDbKey;
void *pKey = NULL;
void *pVal = NULL;
int kLen = 0;
int vLen = 0;
SEncoder coder = {0};
// set key and value
tbDbKey.version = pME->version;
tbDbKey.uid = pME->uid;
pKey = &tbDbKey;
kLen = sizeof(tbDbKey);
int32_t ret = 0;
tEncodeSize(metaEncodeEntry, pME, vLen, ret);
if (ret < 0) {
goto _err;
}
pVal = taosMemoryMalloc(vLen);
if (pVal == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
tEncoderInit(&coder, pVal, vLen);
if (metaEncodeEntry(&coder, pME) < 0) {
goto _err;
}
tEncoderClear(&coder);
// write to table.db
if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
goto _err;
}
taosMemoryFree(pVal);
return 0;
_err:
taosMemoryFree(pVal);
return -1;
}
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
}
static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) {
SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid};
return tdbDbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn);
}
static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) {
metaWLock(pMeta);
// save to table.db
if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err;
// // update uid.idx
if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err;
if (metaUpdateSmaIdx(pMeta, pME) < 0) goto _err;
metaULock(pMeta);
return 0;
_err:
metaULock(pMeta);
return -1;
}

33
source/dnode/vnode/src/inc/tsdbSma.h → source/dnode/vnode/src/sma/sma.c
View File

@ -13,35 +13,18 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef _TD_VNODE_TSDB_SMA_H_ #include "sma.h"
#define _TD_VNODE_TSDB_SMA_H_
#include "tsdb.h"
#ifdef __cplusplus // TODO: Who is responsible for resource allocate and release?
extern "C" { int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) {
#endif int32_t code = TSDB_CODE_SUCCESS;
// typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2); if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) {
smaWarn("vgId:%d insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
// struct STbDdlH {
// void *ahandle;
// void *result;
// __tb_ddl_fn_t fp;
// };
static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
ASSERT(*pStore == NULL);
*pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
if (*pStore == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
} }
return TSDB_CODE_SUCCESS; // TODO: destroy SSDataBlocks(msg)
return code;
} }
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_TSDB_SMA_H_*/

463
source/dnode/vnode/src/sma/smaEnv.c Normal file
View File

@ -0,0 +1,463 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sma.h"
typedef struct SSmaStat SSmaStat;
static const char *TSDB_SMA_DNAME[] = {
"", // TSDB_SMA_TYPE_BLOCK
"tsma", // TSDB_SMA_TYPE_TIME_RANGE
"rsma", // TSDB_SMA_TYPE_ROLLUP
};
#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test
#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test
#define SMA_STATE_HASH_SLOT 4
#define RSMA_TASK_INFO_HASH_SLOT 8
typedef struct SPoolMem {
int64_t size;
struct SPoolMem *prev;
struct SPoolMem *next;
} SPoolMem;
// declaration of static functions
// insert data
static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]);
// Pool Memory
static SPoolMem *openPool();
static void clearPool(SPoolMem *pPool);
static void closePool(SPoolMem *pPool);
static void *poolMalloc(void *arg, size_t size);
static void poolFree(void *arg, void *ptr);
// implementation
static SPoolMem *openPool() {
SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool));
pPool->prev = pPool->next = pPool;
pPool->size = 0;
return pPool;
}
static void clearPool(SPoolMem *pPool) {
if (!pPool) return;
SPoolMem *pMem;
do {
pMem = pPool->next;
if (pMem == pPool) break;
pMem->next->prev = pMem->prev;
pMem->prev->next = pMem->next;
pPool->size -= pMem->size;
taosMemoryFree(pMem);
} while (1);
assert(pPool->size == 0);
}
static void closePool(SPoolMem *pPool) {
if (pPool) {
clearPool(pPool);
taosMemoryFree(pPool);
}
}
static void *poolMalloc(void *arg, size_t size) {
void *ptr = NULL;
SPoolMem *pPool = (SPoolMem *)arg;
SPoolMem *pMem;
pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size);
if (!pMem) {
assert(0);
}
pMem->size = sizeof(*pMem) + size;
pMem->next = pPool->next;
pMem->prev = pPool;
pPool->next->prev = pMem;
pPool->next = pMem;
pPool->size += pMem->size;
ptr = (void *)(&pMem[1]);
return ptr;
}
static void poolFree(void *arg, void *ptr) {
SPoolMem *pPool = (SPoolMem *)arg;
SPoolMem *pMem;
pMem = &(((SPoolMem *)ptr)[-1]);
pMem->next->prev = pMem->prev;
pMem->prev->next = pMem->next;
pPool->size -= pMem->size;
taosMemoryFree(pMem);
}
int32_t tdInitSma(SSma *pSma) {
// tSma
int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma), false));
if (numOfTSma > 0) {
atomic_store_16(&SMA_TSMA_NUM(pSma), (int16_t)numOfTSma);
}
// TODO: rSma
return TSDB_CODE_SUCCESS;
}
static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]);
}
static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did) {
SSmaEnv *pEnv = NULL;
pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv));
if (!pEnv) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
SMA_ENV_TYPE(pEnv) = smaType;
int code = taosThreadRwlockInit(&(pEnv->lock), NULL);
if (code) {
terrno = TAOS_SYSTEM_ERROR(code);
taosMemoryFree(pEnv);
return NULL;
}
ASSERT(path && (strlen(path) > 0));
SMA_ENV_PATH(pEnv) = strdup(path);
if (!SMA_ENV_PATH(pEnv)) {
tdFreeSmaEnv(pEnv);
return NULL;
}
SMA_ENV_DID(pEnv) = did;
if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) {
tdFreeSmaEnv(pEnv);
return NULL;
}
char aname[TSDB_FILENAME_LEN] = {0};
tfsAbsoluteName(SMA_TFS(pSma), did, path, aname);
if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
tdFreeSmaEnv(pEnv);
return NULL;
}
if (!(pEnv->pPool = openPool())) {
tdFreeSmaEnv(pEnv);
return NULL;
}
return pEnv;
}
static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) {
if (!pEnv) {
terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_FAILED;
}
if (!(*pEnv)) {
if (!(*pEnv = tdNewSmaEnv(pSma, smaType, path, did))) {
return TSDB_CODE_FAILED;
}
}
return TSDB_CODE_SUCCESS;
}
/**
* @brief Release resources allocated for its member fields, not including itself.
*
* @param pSmaEnv
* @return int32_t
*/
void tdDestroySmaEnv(SSmaEnv *pSmaEnv) {
if (pSmaEnv) {
tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv));
taosMemoryFreeClear(pSmaEnv->pStat);
taosMemoryFreeClear(pSmaEnv->path);
taosThreadRwlockDestroy(&(pSmaEnv->lock));
smaCloseDBEnv(pSmaEnv->dbEnv);
closePool(pSmaEnv->pPool);
}
}
void *tdFreeSmaEnv(SSmaEnv *pSmaEnv) {
tdDestroySmaEnv(pSmaEnv);
taosMemoryFreeClear(pSmaEnv);
return NULL;
}
int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) {
if (!pStat) return 0;
int ref = T_REF_INC(pStat);
smaDebug("vgId:%d ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
return 0;
}
int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
if (!pStat) return 0;
int ref = T_REF_DEC(pStat);
smaDebug("vgId:%d unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
return 0;
}
static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
ASSERT(pSmaStat != NULL);
if (*pSmaStat) { // no lock
return TSDB_CODE_SUCCESS;
}
/**
* 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew).
* 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if
* tdInitSmaStat invoked in other multithread environment later.
*/
if (!(*pSmaStat)) {
*pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat));
if (!(*pSmaStat)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
if (smaType == TSDB_SMA_TYPE_ROLLUP) {
SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit(
RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
if (!SMA_STAT_INFO_HASH(*pSmaStat)) {
taosMemoryFreeClear(*pSmaStat);
return TSDB_CODE_FAILED;
}
} else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
SMA_STAT_ITEMS(*pSmaStat) =
taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (!SMA_STAT_ITEMS(*pSmaStat)) {
taosMemoryFreeClear(*pSmaStat);
return TSDB_CODE_FAILED;
}
} else {
ASSERT(0);
}
}
return TSDB_CODE_SUCCESS;
}
void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) {
if (pSmaStatItem) {
tdDestroyTSma(pSmaStatItem->pTSma);
taosMemoryFreeClear(pSmaStatItem->pTSma);
taosHashCleanup(pSmaStatItem->expiredWindows);
taosMemoryFreeClear(pSmaStatItem);
}
return NULL;
}
/**
* @brief Release resources allocated for its member fields, not including itself.
*
* @param pSmaStat
* @return int32_t
*/
int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
if (pSmaStat) {
// TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL);
while (item) {
SSmaStatItem *pItem = *(SSmaStatItem **)item;
tdFreeSmaStatItem(pItem);
item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item);
}
taosHashCleanup(SMA_STAT_ITEMS(pSmaStat));
} else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL);
while (infoHash) {
SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash;
tdFreeRSmaInfo(pInfoHash);
infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash);
}
taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat));
} else {
ASSERT(0);
}
}
return TSDB_CODE_SUCCESS;
}
int32_t tdLockSma(SSma *pSma) {
int code = taosThreadMutexLock(&pSma->mutex);
if (code != 0) {
smaError("vgId:%d failed to lock td since %s", SMA_VID(pSma), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
pSma->locked = true;
return 0;
}
int32_t tdUnLockSma(SSma *pSma) {
ASSERT(SMA_LOCKED(pSma));
pSma->locked = false;
int code = taosThreadMutexUnlock(&pSma->mutex);
if (code != 0) {
smaError("vgId:%d failed to unlock td since %s", SMA_VID(pSma), strerror(errno));
terrno = TAOS_SYSTEM_ERROR(code);
return -1;
}
return 0;
}
int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
SSmaEnv *pEnv = NULL;
// return if already init
switch (smaType) {
case TSDB_SMA_TYPE_TIME_RANGE:
if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_TSMA_ENV(pSma)))) {
return TSDB_CODE_SUCCESS;
}
break;
case TSDB_SMA_TYPE_ROLLUP:
if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_RSMA_ENV(pSma)))) {
return TSDB_CODE_SUCCESS;
}
break;
default:
TASSERT(0);
return TSDB_CODE_FAILED;
}
// init sma env
tdLockSma(pSma);
pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&SMA_TSMA_ENV(pSma))
: atomic_load_ptr(&SMA_RSMA_ENV(pSma));
if (!pEnv) {
char rname[TSDB_FILENAME_LEN] = {0};
SDiskID did = {0};
if (tfsAllocDisk(SMA_TFS(pSma), TFS_PRIMARY_LEVEL, &did) < 0) {
tdUnLockSma(pSma);
return TSDB_CODE_FAILED;
}
if (did.level < 0 || did.id < 0) {
tdUnLockSma(pSma);
smaError("vgId:%d init sma env failed since invalid did(%d,%d)", SMA_VID(pSma), did.level, did.id);
return TSDB_CODE_FAILED;
}
tdGetSmaDir(SMA_VID(pSma), smaType, rname);
if (tfsMkdirRecurAt(SMA_TFS(pSma), rname, did) < 0) {
tdUnLockSma(pSma);
return TSDB_CODE_FAILED;
}
if (tdInitSmaEnv(pSma, smaType, rname, did, &pEnv) < 0) {
tdUnLockSma(pSma);
return TSDB_CODE_FAILED;
}
(smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), pEnv)
: atomic_store_ptr(&SMA_RSMA_ENV(pSma), pEnv);
}
tdUnLockSma(pSma);
return TSDB_CODE_SUCCESS;
};
int32_t tdSmaBeginCommit(SSmaEnv *pEnv) {
TXN *pTxn = &pEnv->txn;
// start a new txn
tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
if (tdbBegin(pEnv->dbEnv, pTxn) != 0) {
smaWarn("tdSma tdb begin commit fail");
return -1;
}
return 0;
}
int32_t tdSmaEndCommit(SSmaEnv *pEnv) {
TXN *pTxn = &pEnv->txn;
// Commit current txn
if (tdbCommit(pEnv->dbEnv, pTxn) != 0) {
smaWarn("tdSma tdb end commit fail");
return -1;
}
tdbTxnClose(pTxn);
clearPool(pEnv->pPool);
return 0;
}
#if 0
/**
* @brief Get the start TS key of the last data block of one interval/sliding.
*
* @param pSma
* @param param
* @param result
* @return int32_t
* 1) Return 0 and fill the result if the check procedure is normal;
* 2) Return -1 if error occurs during the check procedure.
*/
int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) {
const char *procedure = "";
if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) {
return -1;
}
// fill the result
return TSDB_CODE_SUCCESS;
}
/**
* @brief Remove the tSma data files related to param between pWin.
*
* @param pSma
* @param param
* @param pWin
* @return int32_t
*/
int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) {
// for ("tSmaFiles of param-interval-sliding between pWin") {
// // remove the tSmaFile
// }
return TSDB_CODE_SUCCESS;
}
#endif

137
source/dnode/vnode/src/sma/smaOpen.c Normal file
View File

@ -0,0 +1,137 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sma.h"
#include "tsdb.h"
static int32_t smaEvalDays(SRetention *r, int8_t precision);
static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
#define SMA_SET_KEEP_CFG(l) \
do { \
SRetention *r = &pCfg->retentions[l]; \
pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
pKeepCfg->keep0 = pKeepCfg->keep2; \
pKeepCfg->keep1 = pKeepCfg->keep2; \
pKeepCfg->days = smaEvalDays(r, pCfg->precision); \
} while (0)
#define SMA_OPEN_RSMA_IMPL(v, l) \
do { \
SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \
if (!RETENTION_VALID(r)) { \
if (l == 0) { \
goto _err; \
} \
break; \
} \
smaSetKeepCfg(&keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \
if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg) < 0) { \
goto _err; \
} \
} while (0)
#define RETENTION_DAYS_SPLIT_RATIO 10
#define RETENTION_DAYS_SPLIT_MIN 1
#define RETENTION_DAYS_SPLIT_MAX 30
static int32_t smaEvalDays(SRetention *r, int8_t precision) {
int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY);
int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY);
int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO;
if (days <= RETENTION_DAYS_SPLIT_MIN) {
days = RETENTION_DAYS_SPLIT_MIN;
if (days < freqDays) {
days = freqDays + 1;
}
} else {
if (days > RETENTION_DAYS_SPLIT_MAX) {
days = RETENTION_DAYS_SPLIT_MAX;
}
if (days < freqDays) {
days = freqDays + 1;
}
}
return days * 1440;
}
int smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) {
pKeepCfg->precision = pCfg->precision;
switch (type) {
case TSDB_TYPE_TSMA:
ASSERT(0);
break;
case TSDB_TYPE_RSMA_L0:
SMA_SET_KEEP_CFG(0);
break;
case TSDB_TYPE_RSMA_L1:
SMA_SET_KEEP_CFG(1);
break;
case TSDB_TYPE_RSMA_L2:
SMA_SET_KEEP_CFG(2);
break;
default:
ASSERT(0);
break;
}
return 0;
}
int32_t smaOpen(SVnode *pVnode) {
STsdbCfg *pCfg = &pVnode->config.tsdbCfg;
ASSERT(!pVnode->pSma);
SSma *pSma = taosMemoryCalloc(1, sizeof(SSma));
if (!pSma) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pSma->pVnode = pVnode;
taosThreadMutexInit(&pSma->mutex, NULL);
pSma->locked = false;
if (vnodeIsRollup(pVnode)) {
STsdbKeepCfg keepCfg = {0};
for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
if (i == TSDB_RETENTION_L0) {
SMA_OPEN_RSMA_IMPL(pVnode, 0);
} else if (i == TSDB_RETENTION_L1) {
SMA_OPEN_RSMA_IMPL(pVnode, 1);
} else if (i == TSDB_RETENTION_L2) {
SMA_OPEN_RSMA_IMPL(pVnode, 2);
} else {
ASSERT(0);
}
}
}
pVnode->pSma = pSma;
return 0;
_err:
taosMemoryFreeClear(pSma);
return -1;
}
int32_t smaClose(SSma *pSma) {
if (pSma) {
taosThreadMutexDestroy(&pSma->mutex);
if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));
}
return 0;
}

484
source/dnode/vnode/src/sma/smaRollup.c Normal file
View File

@ -0,0 +1,484 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sma.h"
static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo,
STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level);
struct SRSmaInfo {
void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t
};
static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) {
// Note: free/kill may in RC
qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
qDestroyTask(otaskHandle);
}
}
void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
if (pInfo->taskInfo[i]) {
tdFreeTaskHandle(pInfo->taskInfo[i]);
}
}
return NULL;
}
static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
ASSERT(*pStore == NULL);
*pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
if (*pStore == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
if (!suid || !tbUids) {
terrno = TSDB_CODE_INVALID_PTR;
smaError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
return TSDB_CODE_FAILED;
}
pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
smaError("vgId:%d failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
return TSDB_CODE_FAILED;
}
if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) {
smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
return TSDB_CODE_FAILED;
} else {
smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0));
}
if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) {
smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
return TSDB_CODE_FAILED;
} else {
smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0));
}
return TSDB_CODE_SUCCESS;
}
int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) {
if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) {
return TSDB_CODE_SUCCESS;
}
if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
void *pIter = taosHashIterate(pStore->uidHash, NULL);
while (pIter) {
tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
SArray *pTbUids = *(SArray **)pIter;
if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) {
taosHashCancelIterate(pStore->uidHash, pIter);
return TSDB_CODE_FAILED;
}
pIter = taosHashIterate(pStore->uidHash, pIter);
}
return TSDB_CODE_SUCCESS;
}
/**
* @brief fetch suid/uids when create child tables of rollup SMA
*
* @param pTsdb
* @param ppStore
* @param suid
* @param uid
* @return int32_t
*/
int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) {
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
// only applicable to rollup SMA ctables
if (!pEnv) {
return TSDB_CODE_SUCCESS;
}
SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SHashObj *infoHash = NULL;
if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) {
terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
return TSDB_CODE_FAILED;
}
// info cached when create rsma stable and return directly for non-rsma ctables
if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) {
return TSDB_CODE_SUCCESS;
}
ASSERT(ppStore != NULL);
if (!(*ppStore)) {
if (tdUidStoreInit(ppStore) != 0) {
return TSDB_CODE_FAILED;
}
}
if (tdUidStorePut(*ppStore, suid, &uid) != 0) {
*ppStore = tdUidStoreFree(*ppStore);
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
/**
* @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
*
* @param pTsdb
* @param pMeta
* @param pReq
* @return int32_t
*/
int32_t tdProcessRSmaCreate(SSma *pSma, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) {
if (!pReq->rollup) {
smaTrace("vgId:%d return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
return TSDB_CODE_SUCCESS;
}
SRSmaParam *param = &pReq->pRSmaParam;
if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) {
smaWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
return TSDB_CODE_SUCCESS;
}
if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INIT_FAILED;
return TSDB_CODE_FAILED;
}
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
if (pRSmaInfo) {
smaWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
return TSDB_CODE_SUCCESS;
}
pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo));
if (!pRSmaInfo) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta);
if (!pReadHandle) {
taosMemoryFree(pRSmaInfo);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
SReadHandle handle = {
.reader = pReadHandle,
.meta = pMeta,
.pMsgCb = pMsgCb,
};
if (param->qmsg1) {
pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle);
if (!pRSmaInfo->taskInfo[0]) {
taosMemoryFree(pRSmaInfo);
taosMemoryFree(pReadHandle);
return TSDB_CODE_FAILED;
}
}
if (param->qmsg2) {
pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle);
if (!pRSmaInfo->taskInfo[1]) {
taosMemoryFree(pRSmaInfo);
taosMemoryFree(pReadHandle);
return TSDB_CODE_FAILED;
}
}
if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) !=
TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
} else {
smaDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid);
}
return TSDB_CODE_SUCCESS;
}
/**
* @brief store suid/[uids], prefer to use array and then hash
*
* @param pStore
* @param suid
* @param uid
* @return int32_t
*/
static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) {
// prefer to store suid/uids in array
if ((suid == pStore->suid) || (pStore->suid == 0)) {
if (pStore->suid == 0) {
pStore->suid = suid;
}
if (uid) {
if (!pStore->tbUids) {
if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
}
if (!taosArrayPush(pStore->tbUids, uid)) {
return TSDB_CODE_FAILED;
}
}
} else {
// store other suid/uids in hash when multiple stable/table included in 1 batch of request
if (!pStore->uidHash) {
pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (!pStore->uidHash) {
return TSDB_CODE_FAILED;
}
}
if (uid) {
SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t));
if (uidArray && ((uidArray = *(SArray **)uidArray))) {
taosArrayPush(uidArray, uid);
} else {
SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t));
if (!pUidArray) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
if (!taosArrayPush(pUidArray, uid)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) {
return TSDB_CODE_FAILED;
}
}
} else {
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) {
return TSDB_CODE_FAILED;
}
}
}
return TSDB_CODE_SUCCESS;
}
void tdUidStoreDestory(STbUidStore *pStore) {
if (pStore) {
if (pStore->uidHash) {
if (pStore->tbUids) {
// When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys.
void *pIter = taosHashIterate(pStore->uidHash, NULL);
while (pIter) {
SArray *arr = *(SArray **)pIter;
taosArrayDestroy(arr);
pIter = taosHashIterate(pStore->uidHash, pIter);
}
}
taosHashCleanup(pStore->uidHash);
}
taosArrayDestroy(pStore->tbUids);
}
}
void *tdUidStoreFree(STbUidStore *pStore) {
if (pStore) {
tdUidStoreDestory(pStore);
taosMemoryFree(pStore);
}
return NULL;
}
static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
if (!pReq) {
terrno = TSDB_CODE_INVALID_PTR;
return TSDB_CODE_FAILED;
}
SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) {
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
ASSERT(pMsg != NULL);
SSubmitMsgIter msgIter = {0};
SSubmitBlk *pBlock = NULL;
SSubmitBlkIter blkIter = {0};
STSRow *row = NULL;
terrno = TSDB_CODE_SUCCESS;
if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
while (true) {
if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
if (!pBlock) break;
tdUidStorePut(pStore, msgIter.suid, NULL);
pStore->uid = msgIter.uid; // TODO: remove, just for debugging
}
if (terrno != TSDB_CODE_SUCCESS) return -1;
return 0;
}
static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo,
STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level) {
SArray *pResult = NULL;
if (!taskInfo) {
smaDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
return TSDB_CODE_SUCCESS;
}
smaDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, taskInfo, suid);
qSetStreamInput(taskInfo, pMsg, inputType);
while (1) {
SSDataBlock *output = NULL;
uint64_t ts;
if (qExecTask(taskInfo, &output, &ts) < 0) {
ASSERT(false);
}
if (!output) {
break;
}
if (!pResult) {
pResult = taosArrayInit(0, sizeof(SSDataBlock));
if (!pResult) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
}
taosArrayPush(pResult, output);
}
if (taosArrayGetSize(pResult) > 0) {
blockDebugShowData(pResult);
STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
SSubmitReq *pReq = NULL;
if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), uid, suid) != 0) {
taosArrayDestroy(pResult);
return TSDB_CODE_FAILED;
}
if (tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) {
taosArrayDestroy(pResult);
taosMemoryFreeClear(pReq);
return TSDB_CODE_FAILED;
}
taosMemoryFreeClear(pReq);
} else {
smaWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno));
}
taosArrayDestroy(pResult);
return TSDB_CODE_SUCCESS;
}
static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) {
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
if (!pEnv) {
// only applicable when rsma env exists
return TSDB_CODE_SUCCESS;
}
ASSERT(uid != 0); // TODO: remove later
SSmaStat *pStat = SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL;
pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
smaDebug("vgId:%d no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
return TSDB_CODE_SUCCESS;
}
if (!pRSmaInfo->taskInfo[0]) {
smaDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
return TSDB_CODE_SUCCESS;
}
if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
// TODO: use the proper schema instead of 0, and cache STSchema in cache
STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 0);
if (!pTSchema) {
terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
return TSDB_CODE_FAILED;
}
tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1);
tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2);
taosMemoryFree(pTSchema);
}
return TSDB_CODE_SUCCESS;
}
int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
if (!pEnv) {
// only applicable when rsma env exists
return TSDB_CODE_SUCCESS;
}
if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
STbUidStore uidStore = {0};
tdFetchSubmitReqSuids(pMsg, &uidStore);
if (uidStore.suid != 0) {
tdExecuteRSma(pSma, pMsg, inputType, uidStore.suid, uidStore.uid);
void *pIter = taosHashIterate(uidStore.uidHash, NULL);
while (pIter) {
tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
tdExecuteRSma(pSma, pMsg, inputType, *pTbSuid, 0);
pIter = taosHashIterate(uidStore.uidHash, pIter);
}
tdUidStoreDestory(&uidStore);
}
}
return TSDB_CODE_SUCCESS;
}

128
source/dnode/vnode/src/sma/smaTDBImpl.c Normal file
View File

@ -0,0 +1,128 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "sma.h"
int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) {
int ret = 0;
if (path == NULL) return -1;
ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param
if (ret != 0) {
smaError("failed to create tsdb db env, ret = %d", ret);
return -1;
}
return 0;
}
int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); }
static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
const SSmaKey *pKey1 = (const SSmaKey *)arg1;
const SSmaKey *pKey2 = (const SSmaKey *)arg2;
ASSERT(len1 == len2 && len1 == sizeof(SSmaKey));
if (pKey1->skey < pKey2->skey) {
return -1;
} else if (pKey1->skey > pKey2->skey) {
return 1;
}
if (pKey1->groupId < pKey2->groupId) {
return -1;
} else if (pKey1->groupId > pKey2->groupId) {
return 1;
}
return 0;
}
static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) {
int ret;
tdb_cmpr_fn_t compFunc;
// Create a database
compFunc = tdSmaKeyCmpr;
ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
return 0;
}
static int32_t smaCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); }
int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) {
// TEnv is shared by a group of SDBFile
if (!pEnv || !pDBF) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
}
// Open DBF
if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
terrno = TSDB_CODE_TDB_INIT_FAILED;
smaCloseDBDb(pDBF->pDB);
return -1;
}
return 0;
}
int32_t smaCloseDBF(SDBFile *pDBF) {
int32_t ret = 0;
if (pDBF->pDB) {
ret = smaCloseDBDb(pDBF->pDB);
pDBF->pDB = NULL;
}
taosMemoryFreeClear(pDBF->path);
return ret;
}
int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
int32_t ret;
ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
if (ret < 0) {
smaError("failed to create insert sma data into db, ret = %d", ret);
return -1;
}
return 0;
}
void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) {
void *pVal = NULL;
int ret;
ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
if (ret < 0) {
smaError("failed to get sma data from db, ret = %d", ret);
return NULL;
}
ASSERT(*valLen >= 0);
// TODO: lock?
// TODO: Would the key/value be destoryed during return the data?
// TODO: How about the key is updated while value length is changed? The original value buffer would be freed
// automatically?
return pVal;
}

1103
source/dnode/vnode/src/sma/smaTimeRange.c Normal file
View File

File diff suppressed because it is too large Load Diff

90
source/dnode/vnode/src/tq/tq.c
View File

@ -14,6 +14,7 @@
*/ */
#include "tq.h" #include "tq.h"
#include "tqueue.h"
int32_t tqInit() { int32_t tqInit() {
// //
@ -234,7 +235,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
if (msgType != TDMT_VND_SUBMIT) return 0; if (msgType != TDMT_VND_SUBMIT) return 0;
// make sure msgType == TDMT_VND_SUBMIT // make sure msgType == TDMT_VND_SUBMIT
if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) { if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
return -1; return -1;
} }
@ -1031,3 +1032,90 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId)
} }
return 0; return 0;
} }
int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
void* pIter = NULL;
bool failed = false;
SStreamDataSubmit* pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
if (pSubmit == NULL) {
failed = true;
}
pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t));
if (pSubmit->dataRef == NULL) {
failed = true;
}
pSubmit->type = STREAM_DATA_TYPE_SUBMIT_BLOCK;
pSubmit->sourceVer = ver;
pSubmit->sourceVg = pTq->pVnode->config.vgId;
pSubmit->data = pReq;
*pSubmit->dataRef = 1;
while (1) {
pIter = taosHashIterate(pTq->pStreamTasks, pIter);
if (pIter == NULL) break;
SStreamTask* pTask = (SStreamTask*)pIter;
if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue;
int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
if (failed) {
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
continue;
}
streamDataSubmitRefInc(pSubmit);
taosWriteQitem(pTask->inputQ, pSubmit);
int8_t execStatus = atomic_load_8(&pTask->status);
if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
// TODO dispatch task launch msg to fetch queue
}
} else {
// blocked or stopped, do nothing
}
}
if (!failed) {
streamDataSubmitRefDec(pSubmit);
return 0;
} else {
return -1;
}
}
int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) {
SStreamTaskExecReq req = {0};
tDecodeSStreamTaskExecReq(msg, &req);
int32_t taskId = req.taskId;
SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
ASSERT(pTask);
ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK);
// enqueue
int32_t inputStatus = streamEnqueueDataBlk(pTask, (SStreamDataBlock*)req.data);
if (inputStatus == TASK_INPUT_STATUS__BLOCKED) {
// TODO rsp blocked
return 0;
}
// try exec
int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
if (execStatus == TASK_STATUS__IDLE) {
if (streamTaskRun(pTask) < 0) {
atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
goto FAIL;
}
} else if (execStatus == TASK_STATUS__EXECUTING) {
return 0;
}
// TODO rsp success
return 0;
FAIL:
return -1;
}

2
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -16,6 +16,8 @@
#include "tsdb.h" #include "tsdb.h"
int tsdbBegin(STsdb *pTsdb) { int tsdbBegin(STsdb *pTsdb) {
if (!pTsdb) return 0;
STsdbMemTable *pMem; STsdbMemTable *pMem;
if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) { if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) {

16
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -37,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
// static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbProcessExpiredFS(STsdb *pRepo);
// static int tsdbCreateMeta(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo);
static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) { static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) {
snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]); snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir);
} }
static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) { static void tsdbGetDataDir(int repoid, const char* dir, char dirName[]) {
snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]); snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir);
} }
// For backward compatibility // For backward compatibility
@ -591,7 +591,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]); pRepo->dir, tsdbTxnFname[ftype]);
} }
static int tsdbOpenFSFromCurrent(STsdb *pRepo) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
@ -721,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) {
STsdbFS *pfs = REPO_FS(pRepo); STsdbFS *pfs = REPO_FS(pRepo);
const STfsFile *pf; const STfsFile *pf;
tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir); tsdbGetRootDir(REPO_ID(pRepo), pRepo->dir, rootDir);
STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir);
if (tdir == NULL) { if (tdir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
@ -755,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) {
STsdbFS *pfs = REPO_FS(pRepo); STsdbFS *pfs = REPO_FS(pRepo);
const STfsFile *pf; const STfsFile *pf;
tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir);
STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir);
if (tdir == NULL) { if (tdir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno));
@ -803,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
regex_t regex; regex_t regex;
STsdbFS *pfs = REPO_FS(pRepo); STsdbFS *pfs = REPO_FS(pRepo);
tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir);
// Resource allocation and init // Resource allocation and init
regcomp(&regex, pattern, REG_EXTENDED); regcomp(&regex, pattern, REG_EXTENDED);

10
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -23,14 +23,6 @@ static const char *TSDB_FNAME_SUFFIX[] = {
"smal", // TSDB_FILE_SMAL "smal", // TSDB_FILE_SMAL
"", // TSDB_FILE_MAX "", // TSDB_FILE_MAX
"meta", // TSDB_FILE_META "meta", // TSDB_FILE_META
"tsma", // TSDB_FILE_TSMA
"rsma", // TSDB_FILE_RSMA
};
const char *TSDB_LEVEL_DNAME[] = {
"tsdb",
"rsma1",
"rsma2",
}; };
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname);
@ -51,7 +43,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t
pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
pDFile->info.fver = tsdbGetDFSVersion(ftype); pDFile->info.fver = tsdbGetDFSVersion(ftype);
tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname); tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, pRepo->dir, fname);
tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
} }

123
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -15,100 +15,17 @@
#include "tsdb.h" #include "tsdb.h"
#define TSDB_OPEN_RSMA_IMPL(v, l) \ static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg);
do { \
SRetention *r = VND_RETENTIONS(v)[0]; \
if (RETENTION_VALID(r)) { \
return tsdbOpenImpl((v), type, &VND_RSMA##l(v), VNODE_RSMA##l##_DIR, TSDB_RETENTION_L##l); \
} \
} while (0)
#define TSDB_SET_KEEP_CFG(l) \
do { \
SRetention *r = &pCfg->retentions[l]; \
pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
pKeepCfg->keep0 = pKeepCfg->keep2; \
pKeepCfg->keep1 = pKeepCfg->keep2; \
pKeepCfg->days = tsdbEvalDays(r, pCfg->precision); \
} while (0)
#define RETENTION_DAYS_SPLIT_RATIO 10 // implementation
#define RETENTION_DAYS_SPLIT_MIN 1
#define RETENTION_DAYS_SPLIT_MAX 30
static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type); static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg) {
static int32_t tsdbEvalDays(SRetention *r, int8_t precision);
static int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level);
int tsdbOpen(SVnode *pVnode, int8_t type) {
switch (type) {
case TSDB_TYPE_TSDB:
return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0);
case TSDB_TYPE_TSMA:
ASSERT(0);
break;
case TSDB_TYPE_RSMA_L0:
TSDB_OPEN_RSMA_IMPL(pVnode, 0);
break;
case TSDB_TYPE_RSMA_L1:
TSDB_OPEN_RSMA_IMPL(pVnode, 1);
break;
case TSDB_TYPE_RSMA_L2:
TSDB_OPEN_RSMA_IMPL(pVnode, 2);
break;
default:
ASSERT(0);
break;
}
return 0;
}
static int32_t tsdbEvalDays(SRetention *r, int8_t precision) {
int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY);
int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY);
int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO;
if (days <= RETENTION_DAYS_SPLIT_MIN) {
days = RETENTION_DAYS_SPLIT_MIN;
if (days < freqDays) {
days = freqDays + 1;
}
} else {
if (days > RETENTION_DAYS_SPLIT_MAX) {
days = RETENTION_DAYS_SPLIT_MAX;
}
if (days < freqDays) {
days = freqDays + 1;
}
}
return days * 1440;
}
static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type) {
pKeepCfg->precision = pCfg->precision; pKeepCfg->precision = pCfg->precision;
switch (type) { pKeepCfg->days = pCfg->days;
case TSDB_TYPE_TSDB: pKeepCfg->keep0 = pCfg->keep0;
pKeepCfg->days = pCfg->days; pKeepCfg->keep1 = pCfg->keep1;
pKeepCfg->keep0 = pCfg->keep0; pKeepCfg->keep2 = pCfg->keep2;
pKeepCfg->keep1 = pCfg->keep1;
pKeepCfg->keep2 = pCfg->keep2;
break;
case TSDB_TYPE_TSMA:
ASSERT(0);
break;
case TSDB_TYPE_RSMA_L0:
TSDB_SET_KEEP_CFG(0);
break;
case TSDB_TYPE_RSMA_L1:
TSDB_SET_KEEP_CFG(1);
break;
case TSDB_TYPE_RSMA_L2:
TSDB_SET_KEEP_CFG(2);
break;
default:
ASSERT(0);
break;
}
return 0; return 0;
} }
@ -116,18 +33,16 @@ static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t typ
* @brief * @brief
* *
* @param pVnode * @param pVnode
* @param type
* @param ppTsdb * @param ppTsdb
* @param dir * @param dir
* @param level retention level
* @return int * @return int
*/ */
int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) { int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg) {
STsdb *pTsdb = NULL; STsdb *pTsdb = NULL;
int slen = 0; int slen = 0;
*ppTsdb = NULL; *ppTsdb = NULL;
slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3; slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + TSDB_DATA_DIR_LEN + 3;
// create handle // create handle
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
@ -136,13 +51,18 @@ int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *di
return -1; return -1;
} }
ASSERT(strlen(dir) < TSDB_DATA_DIR_LEN);
memcpy(pTsdb->dir, dir, strlen(dir));
pTsdb->path = (char *)&pTsdb[1]; pTsdb->path = (char *)&pTsdb[1];
sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir);
pTsdb->pVnode = pVnode; pTsdb->pVnode = pVnode;
pTsdb->level = level;
pTsdb->repoLocked = false; pTsdb->repoLocked = false;
taosThreadMutexInit(&pTsdb->mutex, NULL); taosThreadMutexInit(&pTsdb->mutex, NULL);
tsdbSetKeepCfg(REPO_KEEP_CFG(pTsdb), REPO_CFG(pTsdb), type); if (!pKeepCfg) {
tsdbSetKeepCfg(&pTsdb->keepCfg, &pVnode->config.tsdbCfg);
} else {
memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg));
}
pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb)); pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb));
// create dir (TODO: use tfsMkdir) // create dir (TODO: use tfsMkdir)
@ -163,12 +83,13 @@ _err:
return -1; return -1;
} }
int tsdbClose(STsdb *pTsdb) { int tsdbClose(STsdb **pTsdb) {
if (pTsdb) { if (*pTsdb) {
// TODO: destroy mem/imem // TODO: destroy mem/imem
tsdbCloseFS(pTsdb); taosThreadMutexDestroy(&(*pTsdb)->mutex);
tsdbFreeFS(pTsdb->fs); tsdbCloseFS(*pTsdb);
taosMemoryFree(pTsdb); tsdbFreeFS((*pTsdb)->fs);
taosMemoryFreeClear(*pTsdb);
} }
return 0; return 0;
} }

12
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -2769,20 +2769,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
} while (moveToNextRowInMem(pCheckInfo)); } while (moveToNextRowInMem(pCheckInfo));
taosMemoryFreeClear(pSchema); // free the STSChema taosMemoryFreeClear(pSchema); // free the STSChema
assert(numOfRows <= maxRowsToRead); assert(numOfRows <= maxRowsToRead);
// if the buffer is not full in case of descending order query, move the data in the front of the buffer
if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && numOfRows < maxRowsToRead) {
int32_t emptySize = maxRowsToRead - numOfRows;
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes,
numOfRows * pColInfo->info.bytes);
}
}
int64_t elapsedTime = taosGetTimestampUs() - st; int64_t elapsedTime = taosGetTimestampUs() - st;
tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s", tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s",
pTsdbReadHandle, elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->idStr); pTsdbReadHandle, elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->idStr);

2
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -47,7 +47,7 @@ int vnodeBegin(SVnode *pVnode) {
} }
// begin tsdb // begin tsdb
if (vnodeIsRollup(pVnode)) { if (pVnode->pSma) {
if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { if (tsdbBegin(VND_RSMA0(pVnode)) < 0) {
vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1; return -1;

38
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -96,26 +96,15 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
} }
// open tsdb // open tsdb
if (vnodeIsRollup(pVnode)) { if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_TYPE_TSDB) < 0) {
if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) { vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err;
goto _err; }
}
if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) { // open sma
vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); if (smaOpen(pVnode)) {
goto _err; vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
} goto _err;
if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) {
vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
} else {
if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) {
vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
} }
// open wal // open wal
@ -161,10 +150,10 @@ _err:
if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pQuery) vnodeQueryClose(pVnode);
if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pTq) tqClose(pVnode->pTq);
if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pWal) walClose(pVnode->pWal);
if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
if (pVnode->pMeta) metaClose(pVnode->pMeta); if (pVnode->pMeta) metaClose(pVnode->pMeta);
tsdbClose(VND_RSMA1(pVnode)); if (pVnode->pSma) smaClose(pVnode->pSma);
tsdbClose(VND_RSMA2(pVnode));
tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&(pVnode->canCommit));
taosMemoryFree(pVnode); taosMemoryFree(pVnode);
return NULL; return NULL;
@ -177,9 +166,8 @@ void vnodeClose(SVnode *pVnode) {
vnodeQueryClose(pVnode); vnodeQueryClose(pVnode);
walClose(pVnode->pWal); walClose(pVnode->pWal);
tqClose(pVnode->pTq); tqClose(pVnode->pTq);
tsdbClose(VND_TSDB(pVnode)); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
tsdbClose(VND_RSMA1(pVnode)); smaClose(pVnode->pSma);
tsdbClose(VND_RSMA2(pVnode));
metaClose(pVnode->pMeta); metaClose(pVnode->pMeta);
vnodeCloseBufPool(pVnode); vnodeCloseBufPool(pVnode);
// destroy handle // destroy handle

49
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -22,6 +22,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
#if 0 #if 0
@ -86,10 +87,8 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
case TDMT_VND_DROP_TABLE: case TDMT_VND_DROP_TABLE:
if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break; break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA case TDMT_VND_CREATE_SMA: {
if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
// TODO
}
} break; } break;
/* TSDB */ /* TSDB */
case TDMT_VND_SUBMIT: case TDMT_VND_SUBMIT:
@ -195,7 +194,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
// TODO // TODO
// blockDebugShowData(data); // blockDebugShowData(data);
tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data);
} }
int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
@ -305,7 +304,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq,
goto _err; goto _err;
} }
tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req, &pVnode->msgCb); tdProcessRSmaCreate(pVnode->pSma, pVnode->pMeta, &req, &pVnode->msgCb);
tDecoderClear(&coder); tDecoderClear(&coder);
return 0; return 0;
@ -366,7 +365,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
} }
} else { } else {
cRsp.code = TSDB_CODE_SUCCESS; cRsp.code = TSDB_CODE_SUCCESS;
tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
} }
taosArrayPush(rsp.pArray, &cRsp); taosArrayPush(rsp.pArray, &cRsp);
@ -374,8 +373,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
tDecoderClear(&decoder); tDecoderClear(&decoder);
tsdbUpdateTbUidList(pVnode->pTsdb, pStore); tdUpdateTbUidList(pVnode->pSma, pStore);
tsdbUidStoreFree(pStore); tdUidStoreFree(pStore);
// prepare rsp // prepare rsp
SEncoder encoder = {0}; SEncoder encoder = {0};
@ -649,8 +648,38 @@ _exit:
// TODO: refactor // TODO: refactor
if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) && if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) &&
(pRsp->code == TSDB_CODE_SUCCESS)) { (pRsp->code == TSDB_CODE_SUCCESS)) {
tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
} }
return 0; return 0;
} }
static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
SVCreateTSmaReq req = {0};
SDecoder coder;
pRsp->msgType = TDMT_VND_CREATE_SMA_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
pRsp->pCont = NULL;
pRsp->contLen = 0;
// decode and process req
tDecoderInit(&coder, pReq, len);
if (tDecodeSVCreateTSmaReq(&coder, &req) < 0) {
pRsp->code = terrno;
goto _err;
}
if (metaCreateTSma(pVnode->pMeta, version, &req) < 0) {
pRsp->code = terrno;
goto _err;
}
tDecoderClear(&coder);
return 0;
_err:
tDecoderClear(&coder);
return -1;
}

2
source/libs/executor/inc/executorimpl.h
View File

@ -651,7 +651,7 @@ void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag);
void doSetOperatorCompleted(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo);
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset);
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols);
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);

8
source/libs/executor/src/executorMain.c
View File

@ -21,13 +21,14 @@
#include "tcache.h" #include "tcache.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
#include "tudf.h"
#include "thash.h"
#include "executorimpl.h"
#include "executor.h" #include "executor.h"
#include "executorimpl.h"
#include "query.h"
#include "thash.h"
#include "tlosertree.h" #include "tlosertree.h"
#include "ttypes.h" #include "ttypes.h"
#include "query.h"
typedef struct STaskMgmt { typedef struct STaskMgmt {
TdThreadMutex lock; TdThreadMutex lock;
@ -156,6 +157,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0; int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0;
pTaskInfo->totalRows += current; pTaskInfo->totalRows += current;
teardownUdfs();
qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0); GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0);

17
source/libs/executor/src/executorimpl.c
View File

@ -2115,7 +2115,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
} }
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep);
void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) {
if (pFilterNode == NULL) { if (pFilterNode == NULL) {
return; return;
} }
@ -2129,8 +2129,9 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
code = filterSetDataFromSlotId(filter, &param1); code = filterSetDataFromSlotId(filter, &param1);
int8_t* rowRes = NULL; int8_t* rowRes = NULL;
// todo the keep seems never to be True?? // todo the keep seems never to be True??
bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols);
filterFreeInfo(filter); filterFreeInfo(filter);
extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep);
@ -2152,11 +2153,6 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i);
// For the reserved column, the value is not filled yet, so the whole column data may be NULL.
if (pSrc->pData == NULL) {
continue;
}
int32_t numOfRows = 0; int32_t numOfRows = 0;
for (int32_t j = 0; j < totalRows; ++j) { for (int32_t j = 0; j < totalRows; ++j) {
if (rowRes[j] == 0) { if (rowRes[j] == 0) {
@ -5504,18 +5500,21 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId;
int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId;
int32_t rowIndex = -1;
SColumnInfoData* pSrc = NULL; SColumnInfoData* pSrc = NULL;
if (pJoinInfo->pLeft->info.blockId == blockId) { if (pJoinInfo->pLeft->info.blockId == blockId) {
pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId);
rowIndex = pJoinInfo->leftPos;
} else { } else {
pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId);
rowIndex = pJoinInfo->rightPos;
} }
if (colDataIsNull_s(pSrc, pJoinInfo->leftPos)) { if (colDataIsNull_s(pSrc, rowIndex)) {
colDataAppendNULL(pDst, nrows); colDataAppendNULL(pDst, nrows);
} else { } else {
char* p = colDataGetData(pSrc, pJoinInfo->leftPos); char* p = colDataGetData(pSrc, rowIndex);
colDataAppend(pDst, nrows, p, false); colDataAppend(pDst, nrows, p, false);
} }
} }

2
source/libs/executor/src/groupoperator.c
View File

@ -318,7 +318,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
while(1) { while(1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pInfo->pCondition, pRes); doFilter(pInfo->pCondition, pRes, NULL);
bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo);
if (!hasRemain) { if (!hasRemain) {

4
source/libs/executor/src/indexoperator.c
View File

@ -398,6 +398,10 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
output->status = SFLT_ACCURATE_INDEX; output->status = SFLT_ACCURATE_INDEX;
} }
if (ctx->noExec) {
SIF_RET(code);
}
return operFn(&params[0], nParam > 1 ? &params[1] : NULL, output); return operFn(&params[0], nParam > 1 ? &params[1] : NULL, output);
_return: _return:
taosMemoryFree(params); taosMemoryFree(params);

20
source/libs/executor/src/scanoperator.c
View File

@ -159,6 +159,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
return false; return false;
} }
static void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock);
static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) { uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -238,8 +240,15 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
} }
relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols);
// currently only the tbname pseudo column
if (pTableScanInfo->numOfPseudoExpr > 0) {
addTagPseudoColumnData(pTableScanInfo, pBlock);
}
// todo record the filter time cost // todo record the filter time cost
doFilter(pTableScanInfo->pFilterNode, pBlock); doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo);
if (pBlock->info.rows == 0) { if (pBlock->info.rows == 0) {
pCost->filterOutBlocks += 1; pCost->filterOutBlocks += 1;
qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
@ -260,7 +269,7 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction
pTableScanInfo->cond.order = TSDB_ORDER_DESC; pTableScanInfo->cond.order = TSDB_ORDER_DESC;
} }
static void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
// currently only the tbname pseudo column // currently only the tbname pseudo column
if (pTableScanInfo->numOfPseudoExpr == 0) { if (pTableScanInfo->numOfPseudoExpr == 0) {
return; return;
@ -330,11 +339,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
continue; continue;
} }
// currently only the tbname pseudo column
if (pTableScanInfo->numOfPseudoExpr > 0) {
addTagPseudoColumnData(pTableScanInfo, pBlock);
}
return pBlock; return pBlock;
} }
@ -750,7 +754,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
return NULL; return NULL;
} }
rows = pBlockInfo->rows; rows = pBlockInfo->rows;
doFilter(pInfo->pCondition, pInfo->pRes); doFilter(pInfo->pCondition, pInfo->pRes, NULL);
break; break;
} }

4
source/libs/executor/test/index_executor_tests.cpp
View File

@ -249,7 +249,7 @@ TEST(testCase, index_filter_varify) {
sifMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); sifMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight);
SIdxFltStatus st = idxGetFltStatus(opNode); SIdxFltStatus st = idxGetFltStatus(opNode);
EXPECT_EQ(st, SFLT_COARSE_INDEX); EXPECT_EQ(st, SFLT_ACCURATE_INDEX);
nodesDestroyNode(res); nodesDestroyNode(res);
} }
{ {
@ -269,7 +269,7 @@ TEST(testCase, index_filter_varify) {
sifMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); sifMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight);
SIdxFltStatus st = idxGetFltStatus(opNode); SIdxFltStatus st = idxGetFltStatus(opNode);
EXPECT_EQ(st, SFLT_COARSE_INDEX); EXPECT_EQ(st, SFLT_ACCURATE_INDEX);
nodesDestroyNode(res); nodesDestroyNode(res);
} }
} }

2
source/libs/function/inc/udfc.h
View File

@ -43,7 +43,7 @@ int32_t setupUdf(SUdfInfo* udf, UdfcFuncHandle* handle);
int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate,
int32_t *newStateSize, SSDataBlock *output); int32_t *newStateSize, SSDataBlock *output);
int32_t teardownUdf(UdfcFuncHandle handle); int32_t doTeardownUdf(UdfcFuncHandle handle);
typedef struct SUdfSetupRequest { typedef struct SUdfSetupRequest {
char udfName[16]; // char udfName[16]; //

209
source/libs/function/src/tudf.c
View File

@ -310,6 +310,12 @@ enum {
}; };
int64_t gUdfTaskSeqNum = 0; int64_t gUdfTaskSeqNum = 0;
typedef struct SUdfcFuncStub {
char udfName[TSDB_FUNC_NAME_LEN];
UdfcFuncHandle handle;
int32_t refCount;
} SUdfcFuncStub;
typedef struct SUdfcProxy { typedef struct SUdfcProxy {
char udfdPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2]; char udfdPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2];
uv_barrier_t initBarrier; uv_barrier_t initBarrier;
@ -325,12 +331,15 @@ typedef struct SUdfcProxy {
QUEUE taskQueue; QUEUE taskQueue;
QUEUE uvProcTaskQueue; QUEUE uvProcTaskQueue;
uv_mutex_t udfStubsMutex;
SArray* udfStubs; // SUdfcFuncStub
int8_t initialized; int8_t initialized;
} SUdfcProxy; } SUdfcProxy;
SUdfcProxy gUdfdProxy = {0}; SUdfcProxy gUdfdProxy = {0};
typedef struct SClientUdfUvSession { typedef struct SUdfcUvSession {
SUdfcProxy *udfc; SUdfcProxy *udfc;
int64_t severHandle; int64_t severHandle;
uv_pipe_t *udfUvPipe; uv_pipe_t *udfUvPipe;
@ -338,7 +347,9 @@ typedef struct SClientUdfUvSession {
int8_t outputType; int8_t outputType;
int32_t outputLen; int32_t outputLen;
int32_t bufSize; int32_t bufSize;
} SClientUdfUvSession;
char udfName[TSDB_FUNC_NAME_LEN];
} SUdfcUvSession;
typedef struct SClientUvTaskNode { typedef struct SClientUvTaskNode {
SUdfcProxy *udfc; SUdfcProxy *udfc;
@ -361,7 +372,7 @@ typedef struct SClientUvTaskNode {
typedef struct SClientUdfTask { typedef struct SClientUdfTask {
int8_t type; int8_t type;
SClientUdfUvSession *session; SUdfcUvSession *session;
int32_t errCode; int32_t errCode;
@ -393,7 +404,7 @@ typedef struct SClientUvConn {
uv_pipe_t *pipe; uv_pipe_t *pipe;
QUEUE taskQueue; QUEUE taskQueue;
SClientConnBuf readBuf; SClientConnBuf readBuf;
SClientUdfUvSession *session; SUdfcUvSession *session;
} SClientUvConn; } SClientUvConn;
enum { enum {
@ -1072,6 +1083,8 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) {
int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask);
int32_t code = 0;
switch (uvTask->type) { switch (uvTask->type) {
case UV_TASK_CONNECT: { case UV_TASK_CONNECT: {
uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
@ -1091,22 +1104,34 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t));
connReq->data = uvTask; connReq->data = uvTask;
uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfcPipeConnect); uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfcPipeConnect);
code = 0;
break; break;
} }
case UV_TASK_REQ_RSP: { case UV_TASK_REQ_RSP: {
uv_pipe_t *pipe = uvTask->pipe; uv_pipe_t *pipe = uvTask->pipe;
uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); if (pipe == NULL) {
write->data = uvTask; code = TSDB_CODE_UDF_PIPE_NO_PIPE;
int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite); } else {
if (err != 0) { uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t));
fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err)); write->data = uvTask;
int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite);
if (err != 0) {
fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err));
}
code = err;
} }
break; break;
} }
case UV_TASK_DISCONNECT: { case UV_TASK_DISCONNECT: {
SClientUvConn *conn = uvTask->pipe->data; uv_pipe_t *pipe = uvTask->pipe;
QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); if (pipe == NULL) {
uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); code = TSDB_CODE_UDF_PIPE_NO_PIPE;
} else {
SClientUvConn *conn = pipe->data;
QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue);
uv_close((uv_handle_t *)uvTask->pipe, onUdfcPipeClose);
code = 0;
}
break; break;
} }
default: { default: {
@ -1115,10 +1140,10 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
} }
} }
return 0; return code;
} }
void udfClientAsyncCb(uv_async_t *async) { void udfcAsyncTaskCb(uv_async_t *async) {
SUdfcProxy *udfc = async->data; SUdfcProxy *udfc = async->data;
QUEUE wq; QUEUE wq;
@ -1133,6 +1158,9 @@ void udfClientAsyncCb(uv_async_t *async) {
int32_t code = udfcStartUvTask(task); int32_t code = udfcStartUvTask(task);
if (code == 0) { if (code == 0) {
QUEUE_INSERT_TAIL(&udfc->uvProcTaskQueue, &task->procTaskQueue); QUEUE_INSERT_TAIL(&udfc->uvProcTaskQueue, &task->procTaskQueue);
} else {
task->errCode = code;
uv_sem_post(&task->taskSem);
} }
} }
@ -1179,7 +1207,7 @@ void constructUdfService(void *argsThread) {
SUdfcProxy *udfc = (SUdfcProxy *)argsThread; SUdfcProxy *udfc = (SUdfcProxy *)argsThread;
uv_loop_init(&udfc->uvLoop); uv_loop_init(&udfc->uvLoop);
uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfClientAsyncCb); uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfcAsyncTaskCb);
udfc->loopTaskAync.data = udfc; udfc->loopTaskAync.data = udfc;
uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb); uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb);
udfc->loopStopAsync.data = udfc; udfc->loopStopAsync.data = udfc;
@ -1205,6 +1233,8 @@ int32_t udfcOpen() {
atomic_store_8(&proxy->udfcState, UDFC_STATE_READY); atomic_store_8(&proxy->udfcState, UDFC_STATE_READY);
proxy->udfcState = UDFC_STATE_READY; proxy->udfcState = UDFC_STATE_READY;
uv_barrier_wait(&proxy->initBarrier); uv_barrier_wait(&proxy->initBarrier);
uv_mutex_init(&proxy->udfStubsMutex);
proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub));
fnInfo("udfc initialized") fnInfo("udfc initialized")
return 0; return 0;
} }
@ -1221,6 +1251,8 @@ int32_t udfcClose() {
uv_thread_join(&udfc->loopThread); uv_thread_join(&udfc->loopThread);
uv_mutex_destroy(&udfc->taskQueueMutex); uv_mutex_destroy(&udfc->taskQueueMutex);
uv_barrier_destroy(&udfc->initBarrier); uv_barrier_destroy(&udfc->initBarrier);
taosArrayDestroy(udfc->udfStubs);
uv_mutex_destroy(&udfc->udfStubsMutex);
udfc->udfcState = UDFC_STATE_INITAL; udfc->udfcState = UDFC_STATE_INITAL;
fnInfo("udfc cleaned up"); fnInfo("udfc cleaned up");
return 0; return 0;
@ -1242,14 +1274,14 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
return task->errCode; return task->errCode;
} }
int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
fnInfo("udfc setup udf. udfName: %s", udfName); fnInfo("udfc setup udf. udfName: %s", udfName);
if (gUdfdProxy.udfcState != UDFC_STATE_READY) { if (gUdfdProxy.udfcState != UDFC_STATE_READY) {
return TSDB_CODE_UDF_INVALID_STATE; return TSDB_CODE_UDF_INVALID_STATE;
} }
SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask)); SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask));
task->errCode = 0; task->errCode = 0;
task->session = taosMemoryCalloc(1, sizeof(SClientUdfUvSession)); task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession));
task->session->udfc = &gUdfdProxy; task->session->udfc = &gUdfdProxy;
task->type = UDF_TASK_SETUP; task->type = UDF_TASK_SETUP;
@ -1269,8 +1301,9 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
task->session->outputType = rsp->outputType; task->session->outputType = rsp->outputType;
task->session->outputLen = rsp->outputLen; task->session->outputLen = rsp->outputLen;
task->session->bufSize = rsp->bufSize; task->session->bufSize = rsp->bufSize;
strcpy(task->session->udfName, udfName);
if (task->errCode != 0) { if (task->errCode != 0) {
fnError("failed to setup udf. err: %d", task->errCode) fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode)
} else { } else {
fnInfo("sucessfully setup udf func handle. handle: %p", task->session); fnInfo("sucessfully setup udf func handle. handle: %p", task->session);
*funcHandle = task->session; *funcHandle = task->session;
@ -1283,14 +1316,14 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
SSDataBlock* output, SUdfInterBuf *newState) { SSDataBlock* output, SUdfInterBuf *newState) {
fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle); fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
SClientUdfUvSession *session = (SClientUdfUvSession *) handle; SUdfcUvSession *session = (SUdfcUvSession *) handle;
if (session->udfUvPipe == NULL) { if (session->udfUvPipe == NULL) {
fnError("No pipe to udfd"); fnError("No pipe to udfd");
return TSDB_CODE_UDF_PIPE_NO_PIPE; return TSDB_CODE_UDF_PIPE_NO_PIPE;
} }
SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
task->errCode = 0; task->errCode = 0;
task->session = (SClientUdfUvSession *) handle; task->session = (SUdfcUvSession *) handle;
task->type = UDF_TASK_CALL; task->type = UDF_TASK_CALL;
SUdfCallRequest *req = &task->_call.req; SUdfCallRequest *req = &task->_call.req;
@ -1356,7 +1389,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
return err; return err;
} }
int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
int8_t callType = TSDB_UDF_CALL_AGG_INIT; int8_t callType = TSDB_UDF_CALL_AGG_INIT;
int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf); int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf);
@ -1366,7 +1399,7 @@ int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
// input: block, state // input: block, state
// output: interbuf, // output: interbuf,
int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) {
int8_t callType = TSDB_UDF_CALL_AGG_PROC; int8_t callType = TSDB_UDF_CALL_AGG_PROC;
int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState); int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState);
return err; return err;
@ -1374,7 +1407,7 @@ int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBu
// input: interbuf1, interbuf2 // input: interbuf1, interbuf2
// output: resultBuf // output: resultBuf
int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) {
int8_t callType = TSDB_UDF_CALL_AGG_MERGE; int8_t callType = TSDB_UDF_CALL_AGG_MERGE;
int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf); int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf);
return err; return err;
@ -1382,13 +1415,13 @@ int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInte
// input: interBuf // input: interBuf
// output: resultData // output: resultData
int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) {
int8_t callType = TSDB_UDF_CALL_AGG_FIN; int8_t callType = TSDB_UDF_CALL_AGG_FIN;
int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData);
return err; return err;
} }
int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output) { int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output) {
int8_t callType = TSDB_UDF_CALL_SCALA_PROC; int8_t callType = TSDB_UDF_CALL_SCALA_PROC;
SSDataBlock inputBlock = {0}; SSDataBlock inputBlock = {0};
convertScalarParamToDataBlock(input, numOfCols, &inputBlock); convertScalarParamToDataBlock(input, numOfCols, &inputBlock);
@ -1400,10 +1433,68 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
return err; return err;
} }
int32_t teardownUdf(UdfcFuncHandle handle) { int compareUdfcFuncSub(const void* elem1, const void* elem2) {
SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1;
SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2;
return strcmp(stub1->udfName, stub2->udfName);
}
int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) {
int32_t code = 0;
uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
strcpy(key.udfName, udfName);
SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
if (foundStub != NULL) {
uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
*pHandle = foundStub->handle;
++foundStub->refCount;
return 0;
}
*pHandle = NULL;
code = doSetupUdf(udfName, pHandle);
if (code == TSDB_CODE_SUCCESS) {
SUdfcFuncStub stub = {0};
strcpy(stub.udfName, udfName);
stub.handle = *pHandle;
++stub.refCount;
taosArrayPush(gUdfdProxy.udfStubs, &stub);
taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub);
} else {
*pHandle = NULL;
}
uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
return code;
}
void releaseUdfFuncHandle(char* udfName) {
uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
strcpy(key.udfName, udfName);
SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
ASSERT(foundStub);
--foundStub->refCount;
ASSERT(foundStub->refCount>=0);
uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
}
int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) {
UdfcFuncHandle handle = NULL;
int32_t code = accquireUdfFuncHandle(udfName, &handle);
if (code != 0) {
return code;
}
code = doCallUdfScalarFunc(handle, input, numOfCols, output);
releaseUdfFuncHandle(udfName);
return code;
}
//TODO: when to teardown udf. teardown udf is not called
int32_t doTeardownUdf(UdfcFuncHandle handle) {
fnInfo("tear down udf. udf func handle: %p", handle); fnInfo("tear down udf. udf func handle: %p", handle);
SClientUdfUvSession *session = (SClientUdfUvSession *) handle; SUdfcUvSession *session = (SUdfcUvSession *) handle;
if (session->udfUvPipe == NULL) { if (session->udfUvPipe == NULL) {
fnError("pipe to udfd does not exist"); fnError("pipe to udfd does not exist");
return TSDB_CODE_UDF_PIPE_NO_PIPE; return TSDB_CODE_UDF_PIPE_NO_PIPE;
@ -1433,7 +1524,7 @@ int32_t teardownUdf(UdfcFuncHandle handle) {
//memory layout |---SUdfAggRes----|-----final result-----|---inter result----| //memory layout |---SUdfAggRes----|-----final result-----|---inter result----|
typedef struct SUdfAggRes { typedef struct SUdfAggRes {
SClientUdfUvSession *session; SUdfcUvSession *session;
int8_t finalResNum; int8_t finalResNum;
int8_t interResNum; int8_t interResNum;
char* finalResBuf; char* finalResBuf;
@ -1454,11 +1545,11 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
} }
UdfcFuncHandle handle; UdfcFuncHandle handle;
int32_t udfCode = 0; int32_t udfCode = 0;
if ((udfCode = setupUdf((char*)pCtx->udfName, &handle)) != 0) { if ((udfCode = accquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
fnError("udfAggInit error. step setupUdf. udf code: %d", udfCode); fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode);
return false; return false;
} }
SClientUdfUvSession *session = (SClientUdfUvSession *)handle; SUdfcUvSession *session = (SUdfcUvSession *)handle;
SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo);
int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize;
memset(udfRes, 0, envSize); memset(udfRes, 0, envSize);
@ -1466,10 +1557,10 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
udfRes->session = (SClientUdfUvSession *)handle; udfRes->session = (SUdfcUvSession *)handle;
SUdfInterBuf buf = {0}; SUdfInterBuf buf = {0};
if ((udfCode = callUdfAggInit(handle, &buf)) != 0) { if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) {
fnError("udfAggInit error. step callUdfAggInit. udf code: %d", udfCode); fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode);
return false; return false;
} }
udfRes->interResNum = buf.numOfResult; udfRes->interResNum = buf.numOfResult;
@ -1482,7 +1573,10 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
int32_t numOfCols = pInput->numOfInputCols; int32_t numOfCols = pInput->numOfInputCols;
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SClientUdfUvSession *session = udfRes->session; SUdfcUvSession *session = udfRes->session;
if (session == NULL) {
return TSDB_CODE_UDF_NO_FUNC_HANDLE;
}
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
@ -1513,7 +1607,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
.numOfResult = udfRes->interResNum}; .numOfResult = udfRes->interResNum};
SUdfInterBuf newState = {0}; SUdfInterBuf newState = {0};
int32_t udfCode = callUdfAggProcess(session, inputBlock, &state, &newState); int32_t udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState);
if (udfCode != 0) { if (udfCode != 0) {
fnError("udfAggProcess error. code: %d", udfCode); fnError("udfAggProcess error. code: %d", udfCode);
newState.numOfResult = 0; newState.numOfResult = 0;
@ -1534,7 +1628,10 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SClientUdfUvSession *session = udfRes->session; SUdfcUvSession *session = udfRes->session;
if (session == NULL) {
return TSDB_CODE_UDF_NO_FUNC_HANDLE;
}
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
@ -1544,9 +1641,9 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
.bufLen = session->bufSize, .bufLen = session->bufSize,
.numOfResult = udfRes->interResNum}; .numOfResult = udfRes->interResNum};
int32_t udfCallCode= 0; int32_t udfCallCode= 0;
udfCallCode= callUdfAggFinalize(session, &state, &resultBuf); udfCallCode= doCallUdfAggFinalize(session, &state, &resultBuf);
if (udfCallCode!= 0) { if (udfCallCode != 0) {
fnError("udfAggFinalize error. callUdfAggFinalize step. udf code:%d", udfCallCode); fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode);
GET_RES_INFO(pCtx)->numOfRes = 0; GET_RES_INFO(pCtx)->numOfRes = 0;
} else { } else {
memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen); memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen);
@ -1554,11 +1651,31 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
} }
int32_t code = teardownUdf(session); // int32_t code = doTeardownUdf(session);
if (code != 0) { // if (code != 0) {
fnError("udfAggFinalize error. teardownUdf step. udf code: %d", code); // fnError("udfAggFinalize error. doTeardownUdf step. udf code: %d", code);
// }
int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
releaseUdfFuncHandle(pCtx->udfName);
return udfCallCode == 0 ? numOfResults : udfCallCode;
}
int32_t teardownUdfs() {
uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
int32_t i = 0;
SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) {
SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i);
if (stub->refCount == 0) {
doTeardownUdf(stub->handle);
} else {
taosArrayPush(udfStubs, stub);
}
++i;
} }
taosArrayDestroy(gUdfdProxy.udfStubs);
return functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); gUdfdProxy.udfStubs = udfStubs;
uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
return 0;
} }

6
source/libs/function/test/runUdf.c
View File

@ -47,7 +47,7 @@ int main(int argc, char *argv[]) {
UdfcFuncHandle handle; UdfcFuncHandle handle;
setupUdf("udf1", &handle); doSetupUdf("udf1", &handle);
SSDataBlock block = {0}; SSDataBlock block = {0};
SSDataBlock *pBlock = &block; SSDataBlock *pBlock = &block;
@ -73,12 +73,12 @@ int main(int argc, char *argv[]) {
input.numOfRows = pBlock->info.rows; input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0); input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0}; SScalarParam output = {0};
callUdfScalarFunc(handle, &input, 1, &output); doCallUdfScalarFunc(handle, &input, 1, &output);
SColumnInfoData *col = output.columnData; SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) { for (int32_t i = 0; i < output.numOfRows; ++i) {
fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t))); fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t)));
} }
teardownUdf(handle); doTeardownUdf(handle);
udfcClose(); udfcClose();
} }

2
source/libs/index/inc/indexComm.h
View File

@ -37,6 +37,8 @@ TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b);
_cache_range_compare indexGetCompare(RangeType ty); _cache_range_compare indexGetCompare(RangeType ty);
int32_t indexConvertData(void* src, int8_t type, void** dst);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

9
source/libs/index/inc/indexInt.h
View File

@ -46,9 +46,7 @@ typedef struct SIndexStat {
} SIndexStat; } SIndexStat;
struct SIndex { struct SIndex {
#ifdef USE_LUCENE int64_t refId;
index_t* index;
#endif
void* cache; void* cache;
void* tindex; void* tindex;
SHashObj* colObj; // < field name, field id> SHashObj* colObj; // < field name, field id>
@ -124,6 +122,11 @@ typedef struct TFileCacheKey {
int indexFlushCacheToTFile(SIndex* sIdx, void*); int indexFlushCacheToTFile(SIndex* sIdx, void*);
int64_t indexAddRef(void* p);
int32_t indexRemoveRef(int64_t ref);
void indexAcquireRef(int64_t ref);
void indexReleaseRef(int64_t ref);
int32_t indexSerialCacheKey(ICacheKey* key, char* buf); int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf);
// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);

92
source/libs/index/src/index.c
View File

@ -19,7 +19,10 @@
#include "indexInt.h" #include "indexInt.h"
#include "indexTfile.h" #include "indexTfile.h"
#include "indexUtil.h" #include "indexUtil.h"
#include "tcoding.h"
#include "tdataformat.h"
#include "tdef.h" #include "tdef.h"
#include "tref.h"
#include "tsched.h" #include "tsched.h"
#ifdef USE_LUCENE #ifdef USE_LUCENE
@ -27,36 +30,40 @@
#endif #endif
#define INDEX_NUM_OF_THREADS 4 #define INDEX_NUM_OF_THREADS 4
#define INDEX_QUEUE_SIZE 200 #define INDEX_QUEUE_SIZE 200
void* indexQhandle = NULL; #define INDEX_DATA_BOOL_NULL 0x02
#define INDEX_DATA_TINYINT_NULL 0x80
#define INDEX_DATA_BOOL_NULL 0x02 #define INDEX_DATA_SMALLINT_NULL 0x8000
#define INDEX_DATA_TINYINT_NULL 0x80 #define INDEX_DATA_INT_NULL 0x80000000L
#define INDEX_DATA_SMALLINT_NULL 0x8000 #define INDEX_DATA_BIGINT_NULL 0x8000000000000000L
#define INDEX_DATA_INT_NULL 0x80000000L
#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L
#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN #define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN
#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF #define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF
#define INDEX_DATA_BINARY_NULL 0xFF #define INDEX_DATA_BINARY_NULL 0xFF
#define INDEX_DATA_JSON_NULL 0xFFFFFFFF #define INDEX_DATA_JSON_NULL 0xFFFFFFFF
#define INDEX_DATA_JSON_null 0xFFFFFFFE #define INDEX_DATA_JSON_null 0xFFFFFFFE
#define INDEX_DATA_JSON_NOT_NULL 0x01 #define INDEX_DATA_JSON_NOT_NULL 0x01
#define INDEX_DATA_UTINYINT_NULL 0xFF #define INDEX_DATA_UTINYINT_NULL 0xFF
#define INDEX_DATA_USMALLINT_NULL 0xFFFF #define INDEX_DATA_USMALLINT_NULL 0xFFFF
#define INDEX_DATA_UINT_NULL 0xFFFFFFFF #define INDEX_DATA_UINT_NULL 0xFFFFFFFF
#define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL #define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL
#define INDEX_DATA_NULL_STR "NULL" #define INDEX_DATA_NULL_STR "NULL"
#define INDEX_DATA_NULL_STR_L "null" #define INDEX_DATA_NULL_STR_L "null"
void* indexQhandle = NULL;
int32_t indexRefMgt;
static void indexDestroy(void* sIdx);
void indexInit() { void indexInit() {
// refactor later // refactor later
indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index");
indexRefMgt = taosOpenRef(10, indexDestroy);
} }
void indexCleanUp() { void indexCleanUp() {
// refacto later // refacto later
@ -100,7 +107,12 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
sIdx->cVersion = 1; sIdx->cVersion = 1;
sIdx->path = tstrdup(path); sIdx->path = tstrdup(path);
taosThreadMutexInit(&sIdx->mtx, NULL); taosThreadMutexInit(&sIdx->mtx, NULL);
sIdx->refId = indexAddRef(sIdx);
taosAcquireRef(indexRefMgt, sIdx->refId);
*index = sIdx; *index = sIdx;
return 0; return 0;
END: END:
@ -112,8 +124,9 @@ END:
return -1; return -1;
} }
void indexClose(SIndex* sIdx) { void indexDestroy(void* handle) {
void* iter = taosHashIterate(sIdx->colObj, NULL); SIndex* sIdx = handle;
void* iter = taosHashIterate(sIdx->colObj, NULL);
while (iter) { while (iter) {
IndexCache** pCache = iter; IndexCache** pCache = iter;
if (*pCache) { if (*pCache) {
@ -128,6 +141,27 @@ void indexClose(SIndex* sIdx) {
taosMemoryFree(sIdx); taosMemoryFree(sIdx);
return; return;
} }
void indexClose(SIndex* sIdx) {
indexReleaseRef(sIdx->refId);
indexRemoveRef(sIdx->refId);
}
int64_t indexAddRef(void* p) {
// impl
return taosAddRef(indexRefMgt, p);
}
int32_t indexRemoveRef(int64_t ref) {
// impl later
return taosRemoveRef(indexRefMgt, ref);
}
void indexAcquireRef(int64_t ref) {
// impl
taosAcquireRef(indexRefMgt, ref);
}
void indexReleaseRef(int64_t ref) {
// impl
taosReleaseRef(indexRefMgt, ref);
}
int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
// TODO(yihao): reduce the lock range // TODO(yihao): reduce the lock range
@ -222,6 +256,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
tm->operType = oper; tm->operType = oper;
tm->colType = colType; tm->colType = colType;
#if 0
tm->colName = (char*)taosMemoryCalloc(1, nColName + 1); tm->colName = (char*)taosMemoryCalloc(1, nColName + 1);
memcpy(tm->colName, colName, nColName); memcpy(tm->colName, colName, nColName);
tm->nColName = nColName; tm->nColName = nColName;
@ -229,6 +264,22 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1);
memcpy(tm->colVal, colVal, nColVal); memcpy(tm->colVal, colVal, nColVal);
tm->nColVal = nColVal; tm->nColVal = nColVal;
#endif
#if 1
tm->colName = (char*)taosMemoryCalloc(1, nColName + 1);
memcpy(tm->colName, colName, nColName);
tm->nColName = nColName;
char* buf = NULL;
int32_t len = indexConvertData((void*)colVal, INDEX_TYPE_GET_TYPE(colType), (void**)&buf);
assert(len != -1);
tm->colVal = buf;
tm->nColVal = len;
#endif
return tm; return tm;
} }
@ -457,6 +508,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
} else { } else {
indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000);
} }
indexReleaseRef(sIdx->refId);
return ret; return ret;
} }
void iterateValueDestroy(IterateValue* value, bool destroy) { void iterateValueDestroy(IterateValue* value, bool destroy) {

3
source/libs/index/src/indexCache.c
View File

@ -460,8 +460,11 @@ int indexCacheSchedToMerge(IndexCache* pCache) {
schedMsg.fp = doMergeWork; schedMsg.fp = doMergeWork;
schedMsg.ahandle = pCache; schedMsg.ahandle = pCache;
schedMsg.thandle = NULL; schedMsg.thandle = NULL;
// schedMsg.thandle = taosMemoryCalloc(1, sizeof(int64_t));
// memcpy((char*)(schedMsg.thandle), (char*)&(pCache->index->refId), sizeof(int64_t));
schedMsg.msg = NULL; schedMsg.msg = NULL;
indexAcquireRef(pCache->index->refId);
taosScheduleTask(indexQhandle, &schedMsg); taosScheduleTask(indexQhandle, &schedMsg);
return 0; return 0;

114
source/libs/index/src/indexComm.c
View File

@ -16,25 +16,33 @@
#include "indexComm.h" #include "indexComm.h"
#include "index.h" #include "index.h"
#include "indexInt.h" #include "indexInt.h"
#include "tcoding.h"
#include "tcompare.h" #include "tcompare.h"
#include "tdataformat.h"
char JSON_COLUMN[] = "JSON"; char JSON_COLUMN[] = "JSON";
char JSON_VALUE_DELIM = '&'; char JSON_VALUE_DELIM = '&';
static __compar_fn_t indexGetCompar(int8_t type) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
return (__compar_fn_t)strcmp;
}
return getComparFunc(type, 0);
}
static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0); __compar_fn_t func = indexGetCompar(type);
return tDoCommpare(func, QUERY_LESS_THAN, a, b); return tDoCommpare(func, QUERY_LESS_THAN, a, b);
} }
static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0); __compar_fn_t func = indexGetCompar(type);
return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
} }
static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0); __compar_fn_t func = indexGetCompar(type);
return tDoCommpare(func, QUERY_GREATER_THAN, a, b); return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
} }
static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
__compar_fn_t func = getComparFunc(type, 0); __compar_fn_t func = indexGetCompar(type);
return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
} }
@ -120,3 +128,101 @@ char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
return buf; return buf;
} }
int32_t indexConvertData(void* src, int8_t type, void** dst) {
int tlen = -1;
switch (type) {
case TSDB_DATA_TYPE_TIMESTAMP:
tlen = taosEncodeFixedI64(NULL, *(int64_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedI64(dst, *(int64_t*)src);
break;
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT:
tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
break;
case TSDB_DATA_TYPE_TINYINT:
tlen = taosEncodeFixedI8(NULL, *(uint8_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedI8(dst, *(uint8_t*)src);
break;
case TSDB_DATA_TYPE_SMALLINT:
tlen = taosEncodeFixedI16(NULL, *(int16_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedI16(dst, *(int16_t*)src);
break;
case TSDB_DATA_TYPE_USMALLINT:
tlen = taosEncodeFixedU16(NULL, *(uint16_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedU16(dst, *(uint16_t*)src);
break;
case TSDB_DATA_TYPE_INT:
tlen = taosEncodeFixedI32(NULL, *(int32_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedI32(dst, *(int32_t*)src);
break;
case TSDB_DATA_TYPE_FLOAT:
tlen = taosEncodeBinary(NULL, src, sizeof(float));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, sizeof(float));
break;
case TSDB_DATA_TYPE_UINT:
tlen = taosEncodeFixedU32(NULL, *(uint32_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedU32(dst, *(uint32_t*)src);
break;
case TSDB_DATA_TYPE_BIGINT:
tlen = taosEncodeFixedI64(NULL, *(uint32_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedI64(dst, *(uint32_t*)src);
break;
case TSDB_DATA_TYPE_DOUBLE:
tlen = taosEncodeBinary(NULL, src, sizeof(double));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, sizeof(double));
break;
case TSDB_DATA_TYPE_UBIGINT:
tlen = taosEncodeFixedU64(NULL, *(uint32_t*)src);
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeFixedU64(dst, *(uint32_t*)src);
break;
case TSDB_DATA_TYPE_NCHAR: {
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
break;
}
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
#if 1
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
#endif
}
case TSDB_DATA_TYPE_VARBINARY:
#if 1
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
#endif
default:
TASSERT(0);
break;
}
*dst = *dst - tlen;
if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_VARBINARY &&
type == TSDB_DATA_TYPE_VARCHAR) {
uint8_t* p = *dst;
for (int i = 0; i < tlen; i++) {
if (p[i] == 0) {
p[i] = (uint8_t)'0';
}
}
}
return tlen;
}

5
source/libs/index/src/indexFstUtil.c
View File

@ -82,7 +82,10 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) {
str->ref = 1; str->ref = 1;
str->len = len; str->len = len;
str->data = taosMemoryMalloc(len * sizeof(uint8_t)); str->data = taosMemoryMalloc(len * sizeof(uint8_t));
memcpy(str->data, data, len);
if (data != NULL) {
memcpy(str->data, data, len);
}
FstSlice s = {.str = str, .start = 0, .end = len - 1}; FstSlice s = {.str = str, .start = 0, .end = len - 1};
return s; return s;

13
source/libs/index/src/indexTfile.c
View File

@ -469,13 +469,19 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
FstSlice* s = &rt->data; FstSlice* s = &rt->data;
char* ch = (char*)fstSliceData(s, NULL); int32_t sz = 0;
if (0 != strncmp(ch, p, skip)) { char* ch = (char*)fstSliceData(s, &sz);
char* tmp = taosMemoryCalloc(1, sz + 1);
memcpy(tmp, ch, sz);
if (0 != strncmp(tmp, p, skip)) {
swsResultDestroy(rt); swsResultDestroy(rt);
taosMemoryFree(tmp);
break; break;
} }
TExeCond cond = cmpFn(ch + skip, tem->colVal, tem->colType); TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
if (MATCH == cond) { if (MATCH == cond) {
tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
} else if (CONTINUE == cond) { } else if (CONTINUE == cond) {
@ -483,6 +489,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
swsResultDestroy(rt); swsResultDestroy(rt);
break; break;
} }
taosMemoryFree(tmp);
swsResultDestroy(rt); swsResultDestroy(rt);
} }
streamWithStateDestroy(st); streamWithStateDestroy(st);

20
source/libs/index/test/jsonUT.cc
View File

@ -17,12 +17,32 @@
#include "tutil.h" #include "tutil.h"
static std::string dir = "/tmp/json"; static std::string dir = "/tmp/json";
static std::string logDir = "/tmp/log";
static void initLog() {
const char* defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
tsAsyncLog = 0;
sDebugFlag = 143;
strcpy(tsLogDir, logDir.c_str());
taosRemoveDir(tsLogDir);
taosMkDir(tsLogDir);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}
class JsonEnv : public ::testing::Test { class JsonEnv : public ::testing::Test {
protected: protected:
virtual void SetUp() { virtual void SetUp() {
taosRemoveDir(logDir.c_str());
taosMkDir(logDir.c_str());
taosRemoveDir(dir.c_str()); taosRemoveDir(dir.c_str());
taosMkDir(dir.c_str()); taosMkDir(dir.c_str());
printf("set up\n"); printf("set up\n");
initLog();
opts = indexOptsCreate(); opts = indexOptsCreate();
int ret = tIndexJsonOpen(opts, dir.c_str(), &index); int ret = tIndexJsonOpen(opts, dir.c_str(), &index);
assert(ret == 0); assert(ret == 0);

204
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -318,15 +318,19 @@ static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) {
STableComInfo* pNode = (STableComInfo*)pObj; STableComInfo* pNode = (STableComInfo*)pObj;
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code);; tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code);
;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code);; tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code);; tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code);; tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code);
;
} }
return code; return code;
@ -358,12 +362,15 @@ static int32_t jsonToSchema(const SJson* pJson, void* pObj) {
SSchema* pNode = (SSchema*)pObj; SSchema* pNode = (SSchema*)pObj;
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code);; tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code);
;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code);; tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code);; tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name);
@ -415,21 +422,27 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) {
STableMeta* pNode = (STableMeta*)pObj; STableMeta* pNode = (STableMeta*)pObj;
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code);; tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code);
;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code);; tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code);; tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code);; tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code);; tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code);; tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo);
@ -605,7 +618,8 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToLogicPlanNode(pJson, pObj); int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code);; tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs);
@ -881,7 +895,8 @@ static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) {
code = jsonToNodeObject(pJson, jkLogicSubplanRootNode, (SNode**)&pNode->pNode); code = jsonToNodeObject(pJson, jkLogicSubplanRootNode, (SNode**)&pNode->pNode);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType, code);; tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType, code);
;
} }
int32_t objSize = 0; int32_t objSize = 0;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1121,33 +1136,43 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanRatio, &pNode->ratio); code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanRatio, &pNode->ratio);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired, code);; tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs); code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval, code);; tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset, code);; tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding, code);; tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit, code);; tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code);; tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code);
;
} }
return code; return code;
} }
static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { return physiTableScanNodeToJson(pObj, pJson); } static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) {
return physiTableScanNodeToJson(pObj, pJson);
}
static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiTableScanNode(pJson, pObj); } static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) {
return jsonToPhysiTableScanNode(pJson, pObj);
}
static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet";
static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite";
@ -1181,7 +1206,8 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) {
code = tjsonGetBoolValue(pJson, jkSysTableScanPhysiPlanShowRewrite, &pNode->showRewrite); code = tjsonGetBoolValue(pJson, jkSysTableScanPhysiPlanShowRewrite, &pNode->showRewrite);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId, code);; tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId, code);
;
} }
return code; return code;
@ -1265,7 +1291,8 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToPhysicPlanNode(pJson, pObj); int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);; tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions); code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions);
@ -1427,10 +1454,12 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) {
code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsPk, (SNode**)&pNode->pTspk); code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsPk, (SNode**)&pNode->pTspk);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType, code);; tjsonGetNumberValue(pJson, jkWindowPhysiPlanTriggerType, pNode->triggerType, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkWindowPhysiPlanWatermark, pNode->watermark, code);; tjsonGetNumberValue(pJson, jkWindowPhysiPlanWatermark, pNode->watermark, code);
;
} }
return code; return code;
@ -1526,7 +1555,8 @@ static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToPhysicPlanNode(pJson, pObj); int32_t code = jsonToPhysicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode, code);; tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillPhysiPlanWStartTs, &pNode->pWStartTs); code = jsonToNodeObject(pJson, jkFillPhysiPlanWStartTs, &pNode->pWStartTs);
@ -1565,7 +1595,8 @@ static int32_t jsonToPhysiSessionWindowNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToPhysiWindowNode(pJson, pObj); int32_t code = jsonToPhysiWindowNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code);; tjsonGetNumberValue(pJson, jkSessionWindowPhysiPlanGap, pNode->gap, code);
;
} }
return code; return code;
@ -1727,7 +1758,8 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
int32_t code = tjsonToObject(pJson, jkSubplanId, jsonToSubplanId, &pNode->id); int32_t code = tjsonToObject(pJson, jkSubplanId, jsonToSubplanId, &pNode->id);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType, code);; tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkSubplanMsgType, &pNode->msgType); code = tjsonGetIntValue(pJson, jkSubplanMsgType, &pNode->msgType);
@ -1917,7 +1949,8 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code);; tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName);
@ -2171,7 +2204,8 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj); int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code);; tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft);
@ -2205,7 +2239,8 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj); int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code);; tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList);
@ -2350,6 +2385,30 @@ static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkTempTableSubquery = "Subquery";
static int32_t tempTableNodeToJson(const void* pObj, SJson* pJson) {
const STempTableNode* pNode = (const STempTableNode*)pObj;
int32_t code = tableNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkTempTableSubquery, nodeToJson, pNode->pSubquery);
}
return code;
}
static int32_t jsonToTempTableNode(const SJson* pJson, void* pObj) {
STempTableNode* pNode = (STempTableNode*)pObj;
int32_t code = jsonToTableNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkTempTableSubquery, &pNode->pSubquery);
}
return code;
}
static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetType = "GroupingSetType";
static const char* jkGroupingSetParameter = "Parameters"; static const char* jkGroupingSetParameter = "Parameters";
@ -2387,10 +2446,12 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr); int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code);; tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code);
;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code);; tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code);
;
} }
return code; return code;
@ -2497,7 +2558,8 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) {
SFillNode* pNode = (SFillNode*)pObj; SFillNode* pNode = (SFillNode*)pObj;
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code);; tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code);
;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues);
} }
@ -2663,6 +2725,60 @@ static int32_t jsonToDataBlockDescNode(const SJson* pJson, void* pObj) {
return code; return code;
} }
static const char* jkSetOperatorOpType = "OpType";
static const char* jkSetOperatorProjections = "Projections";
static const char* jkSetOperatorLeft = "Left";
static const char* jkSetOperatorRight = "Right";
static const char* jkSetOperatorOrderByList = "OrderByList";
static const char* jkSetOperatorLimit = "Limit";
static int32_t setOperatorToJson(const void* pObj, SJson* pJson) {
const SSetOperator* pNode = (const SSetOperator*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSetOperatorOpType, pNode->opType);
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSetOperatorProjections, pNode->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSetOperatorLeft, nodeToJson, pNode->pLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSetOperatorRight, nodeToJson, pNode->pRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkSetOperatorOrderByList, pNode->pOrderByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSetOperatorLimit, nodeToJson, pNode->pLimit);
}
return code;
}
static int32_t jsonToSetOperator(const SJson* pJson, void* pObj) {
SSetOperator* pNode = (SSetOperator*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
tjsonGetNumberValue(pJson, jkSetOperatorOpType, pNode->opType, code);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSetOperatorProjections, &pNode->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSetOperatorLeft, &pNode->pLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSetOperatorRight, &pNode->pRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkSetOperatorOrderByList, &pNode->pOrderByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkSetOperatorLimit, &pNode->pLimit);
}
return code;
}
static const char* jkSelectStmtDistinct = "Distinct"; static const char* jkSelectStmtDistinct = "Distinct";
static const char* jkSelectStmtProjections = "Projections"; static const char* jkSelectStmtProjections = "Projections";
static const char* jkSelectStmtFrom = "From"; static const char* jkSelectStmtFrom = "From";
@ -2677,7 +2793,7 @@ static const char* jkSelectStmtSlimit = "Slimit";
static const char* jkSelectStmtStmtName = "StmtName"; static const char* jkSelectStmtStmtName = "StmtName";
static const char* jkSelectStmtHasAggFuncs = "HasAggFuncs"; static const char* jkSelectStmtHasAggFuncs = "HasAggFuncs";
static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { static int32_t selectStmtToJson(const void* pObj, SJson* pJson) {
const SSelectStmt* pNode = (const SSelectStmt*)pObj; const SSelectStmt* pNode = (const SSelectStmt*)pObj;
int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); int32_t code = tjsonAddBoolToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct);
@ -2819,6 +2935,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return realTableNodeToJson(pObj, pJson); return realTableNodeToJson(pObj, pJson);
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
return tempTableNodeToJson(pObj, pJson);
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
break; break;
case QUERY_NODE_GROUPING_SET: case QUERY_NODE_GROUPING_SET:
@ -2848,9 +2965,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_DOWNSTREAM_SOURCE: case QUERY_NODE_DOWNSTREAM_SOURCE:
return downstreamSourceNodeToJson(pObj, pJson); return downstreamSourceNodeToJson(pObj, pJson);
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
break; return setOperatorToJson(pObj, pJson);
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return selectStmtTojson(pObj, pJson); return selectStmtToJson(pObj, pJson);
case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_VNODE_MODIF_STMT:
case QUERY_NODE_CREATE_DATABASE_STMT: case QUERY_NODE_CREATE_DATABASE_STMT:
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
@ -2918,7 +3035,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN: case QUERY_NODE_PHYSICAL_PLAN:
return planToJson(pObj, pJson); return planToJson(pObj, pJson);
default: default:
// assert(0);
break; break;
} }
nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj))); nodesWarn("specificNodeToJson unknown node = %s", nodesNodeName(nodeType(pObj)));
@ -2939,6 +3055,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToFunctionNode(pJson, pObj); return jsonToFunctionNode(pJson, pObj);
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return jsonToRealTableNode(pJson, pObj); return jsonToRealTableNode(pJson, pObj);
case QUERY_NODE_TEMP_TABLE:
return jsonToTempTableNode(pJson, pObj);
case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_ORDER_BY_EXPR:
return jsonToOrderByExprNode(pJson, pObj); return jsonToOrderByExprNode(pJson, pObj);
case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_INTERVAL_WINDOW:
@ -2955,6 +3073,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToSlotDescNode(pJson, pObj); return jsonToSlotDescNode(pJson, pObj);
case QUERY_NODE_DOWNSTREAM_SOURCE: case QUERY_NODE_DOWNSTREAM_SOURCE:
return jsonToDownstreamSourceNode(pJson, pObj); return jsonToDownstreamSourceNode(pJson, pObj);
case QUERY_NODE_SET_OPERATOR:
return jsonToSetOperator(pJson, pObj);
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return jsonToSelectStmt(pJson, pObj); return jsonToSelectStmt(pJson, pObj);
case QUERY_NODE_CREATE_TOPIC_STMT: case QUERY_NODE_CREATE_TOPIC_STMT:
@ -3007,7 +3127,6 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN: case QUERY_NODE_PHYSICAL_PLAN:
return jsonToPlan(pJson, pObj); return jsonToPlan(pJson, pObj);
default: default:
assert(0);
break; break;
} }
nodesWarn("jsonToSpecificNode unknown node = %s", nodesNodeName(nodeType(pObj))); nodesWarn("jsonToSpecificNode unknown node = %s", nodesNodeName(nodeType(pObj)));
@ -3038,7 +3157,8 @@ static int32_t jsonToNode(const SJson* pJson, void* pObj) {
SNode* pNode = (SNode*)pObj; SNode* pNode = (SNode*)pObj;
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, jkNodeType, pNode->type, code);; tjsonGetNumberValue(pJson, jkNodeType, pNode->type, code);
;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode); code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {

17
source/libs/parser/src/parAstCreater.c
View File

@ -14,6 +14,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <regex.h>
#include "parAst.h" #include "parAst.h"
#include "parUtil.h" #include "parUtil.h"
#include "ttime.h" #include "ttime.h"
@ -76,6 +78,19 @@ static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) {
return TSDB_CODE_SUCCESS == pCxt->errCode; return TSDB_CODE_SUCCESS == pCxt->errCode;
} }
static bool invalidPassword(const char* pPassword) {
regex_t regex;
if (regcomp(&regex, "[ '\"`\\]", REG_EXTENDED | REG_ICASE) != 0) {
return false;
}
/* Execute regular expression */
int32_t res = regexec(&regex, pPassword, 0, NULL, 0);
regfree(&regex);
return 0 == res;
}
static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) { static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) {
if (NULL == pPasswordToken) { if (NULL == pPasswordToken) {
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
@ -86,6 +101,8 @@ static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken,
strdequote(pPassword); strdequote(pPassword);
if (strtrim(pPassword) <= 0) { if (strtrim(pPassword) <= 0) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY);
} else if (invalidPassword(pPassword)) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PASSWD);
} }
} }
return TSDB_CODE_SUCCESS == pCxt->errCode; return TSDB_CODE_SUCCESS == pCxt->errCode;

13
source/libs/parser/src/parAuthenticator.c
View File

@ -65,13 +65,19 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) {
return code; return code;
} }
static int32_t authDropUser(SAuthCxt* pCxt, SDropUserReq* pStmt) {
if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->user, TSDB_DEFAULT_USER)) {
return TSDB_CODE_PAR_PERMISSION_DENIED;
}
return TSDB_CODE_SUCCESS;
}
static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
return authSetOperator(pCxt, (SSetOperator*)pStmt); return authSetOperator(pCxt, (SSetOperator*)pStmt);
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return authSelect(pCxt, (SSelectStmt*)pStmt); return authSelect(pCxt, (SSelectStmt*)pStmt);
case QUERY_NODE_VNODE_MODIF_STMT:
case QUERY_NODE_CREATE_DATABASE_STMT: case QUERY_NODE_CREATE_DATABASE_STMT:
case QUERY_NODE_DROP_DATABASE_STMT: case QUERY_NODE_DROP_DATABASE_STMT:
case QUERY_NODE_ALTER_DATABASE_STMT: case QUERY_NODE_ALTER_DATABASE_STMT:
@ -84,7 +90,10 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_ALTER_TABLE_STMT: case QUERY_NODE_ALTER_TABLE_STMT:
case QUERY_NODE_CREATE_USER_STMT: case QUERY_NODE_CREATE_USER_STMT:
case QUERY_NODE_ALTER_USER_STMT: case QUERY_NODE_ALTER_USER_STMT:
case QUERY_NODE_DROP_USER_STMT: break;
case QUERY_NODE_DROP_USER_STMT: {
return authDropUser(pCxt, (SDropUserReq*)pStmt);
}
case QUERY_NODE_USE_DATABASE_STMT: case QUERY_NODE_USE_DATABASE_STMT:
case QUERY_NODE_CREATE_DNODE_STMT: case QUERY_NODE_CREATE_DNODE_STMT:
case QUERY_NODE_DROP_DNODE_STMT: case QUERY_NODE_DROP_DNODE_STMT:

4
source/libs/parser/src/parCalcConst.c
View File

@ -262,9 +262,9 @@ static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subque
break; break;
case QUERY_NODE_SET_OPERATOR: { case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pSetOp = (SSetOperator*)pStmt; SSetOperator* pSetOp = (SSetOperator*)pStmt;
code = calcConstQuery(pCxt, pSetOp->pLeft, subquery); code = calcConstQuery(pCxt, pSetOp->pLeft, false);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = calcConstQuery(pCxt, pSetOp->pRight, subquery); code = calcConstQuery(pCxt, pSetOp->pRight, false);
} }
break; break;
} }

3
source/libs/parser/src/parInsert.c
View File

@ -1826,6 +1826,9 @@ void smlDestroyHandle(void* pHandle) {
if (!pHandle) return; if (!pHandle) return;
SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; SSmlExecHandle* handle = (SSmlExecHandle*)pHandle;
destroyBlockHashmap(handle->pBlockHash); destroyBlockHashmap(handle->pBlockHash);
tdDestroyKVRowBuilder(&handle->tagsBuilder);
destroyBoundColumnInfo(&handle->tags);
destroyCreateSubTbReq(&handle->createTblReq);
taosMemoryFree(handle); taosMemoryFree(handle);
} }

6
source/libs/parser/src/parInsertData.c
View File

@ -237,9 +237,9 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) {
taosMemoryFreeClear(pDataBlock->pData); taosMemoryFreeClear(pDataBlock->pData);
if (!pDataBlock->cloned) { if (!pDataBlock->cloned) {
// free the refcount for metermeta // free the refcount for metermeta
// if (pDataBlock->pTableMeta != NULL) { if (pDataBlock->pTableMeta != NULL) {
// taosMemoryFreeClear(pDataBlock->pTableMeta); taosMemoryFreeClear(pDataBlock->pTableMeta);
// } }
destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
} }

53
source/libs/parser/src/parTranslater.c
View File

@ -480,6 +480,31 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) {
return res; return res;
} }
static int32_t parseTimeFromValueNode(SValueNode* pVal) {
if (IS_SIGNED_NUMERIC_TYPE(pVal->node.resType.type)) {
return TSDB_CODE_SUCCESS;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) {
pVal->datum.i = pVal->datum.u;
return TSDB_CODE_SUCCESS;
} else if (IS_FLOAT_TYPE(pVal->node.resType.type)) {
pVal->datum.i = pVal->datum.d;
return TSDB_CODE_SUCCESS;
} else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) {
pVal->datum.i = pVal->datum.b;
return TSDB_CODE_SUCCESS;
} else if (IS_VAR_DATA_TYPE(pVal->node.resType.type) || TSDB_DATA_TYPE_TIMESTAMP == pVal->node.resType.type) {
if (TSDB_CODE_SUCCESS == taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes,
pVal->node.resType.precision, tsDaylight)) {
return TSDB_CODE_SUCCESS;
}
char* pEnd = NULL;
pVal->datum.i = strtoll(pVal->literal, &pEnd, 10);
return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED;
} else {
return TSDB_CODE_FAILED;
}
}
static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) { static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) {
uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : targetDt.precision); uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : targetDt.precision);
pVal->node.resType.precision = precision; pVal->node.resType.precision = precision;
@ -571,7 +596,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
break; break;
} }
case TSDB_DATA_TYPE_TIMESTAMP: { case TSDB_DATA_TYPE_TIMESTAMP: {
if (taosParseTime(pVal->literal, &pVal->datum.i, targetDt.bytes, precision, tsDaylight) != TSDB_CODE_SUCCESS) { if (TSDB_CODE_SUCCESS != parseTimeFromValueNode(pVal)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
} }
*(int64_t*)&pVal->typeData = pVal->datum.i; *(int64_t*)&pVal->typeData = pVal->datum.i;
@ -1660,10 +1685,10 @@ static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* p
if (NULL == pCol) { if (NULL == pCol) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); strcpy(pCol->colName, PK_TS_COL_INTERNAL_NAME);
} else { if (!findAndSetColumn(pCol, pTable)) {
// todo return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_FUNC);
} }
*pPrimaryKey = (SNode*)pCol; *pPrimaryKey = (SNode*)pCol;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2924,6 +2949,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p
} }
static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) {
SEncoder encoder = {0};
int32_t contLen = 0;
SVDropTSmaReq dropSmaReq = {0}; SVDropTSmaReq dropSmaReq = {0};
strcpy(dropSmaReq.indexName, pStmt->indexName); strcpy(dropSmaReq.indexName, pStmt->indexName);
@ -2931,16 +2958,26 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt
if (NULL == pCxt->pCmdMsg) { if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t ret = 0;
tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret);
if (ret < 0) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA;
pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq); pCxt->pCmdMsg->msgLen = contLen;
pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) { if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
void* pBuf = pCxt->pCmdMsg->pMsg; void* pBuf = pCxt->pCmdMsg->pMsg;
tSerializeSVDropTSmaReq(&pBuf, &dropSmaReq); if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) {
tEncoderClear(&encoder);
return TSDB_CODE_OUT_OF_MEMORY;
}
tEncoderClear(&encoder);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

4
source/libs/parser/src/parUtil.c
View File

@ -148,6 +148,10 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Invalid number of tag columns"; return "Invalid number of tag columns";
case TSDB_CODE_PAR_INVALID_INTERNAL_PK: case TSDB_CODE_PAR_INVALID_INTERNAL_PK:
return "Invalid _c0 or _rowts expression"; return "Invalid _c0 or _rowts expression";
case TSDB_CODE_PAR_INVALID_TIMELINE_FUNC:
return "Invalid timeline function";
case TSDB_CODE_PAR_INVALID_PASSWD:
return "Invalid password";
case TSDB_CODE_OUT_OF_MEMORY: case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory"; return "Out of memory";
default: default:

8
source/libs/parser/test/parSelectTest.cpp
View File

@ -187,7 +187,7 @@ TEST_F(ParserSelectTest, semanticError) {
run("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); run("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE);
// TSDB_CODE_PAR_WRONG_VALUE_TYPE // TSDB_CODE_PAR_WRONG_VALUE_TYPE
run("SELECT timestamp '2010' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); run("SELECT timestamp '2010a' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE);
// TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION
run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION,
@ -235,9 +235,11 @@ TEST_F(ParserSelectTest, semanticError) {
TEST_F(ParserSelectTest, setOperator) { TEST_F(ParserSelectTest, setOperator) {
useDb("root", "test"); useDb("root", "test");
run("SELECT * FROM t1 UNION ALL SELECT * FROM t1"); // run("SELECT * FROM t1 UNION ALL SELECT * FROM t1");
run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); // run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)");
run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
} }
} // namespace ParserTest } // namespace ParserTest

10
source/libs/planner/src/planOptimizer.c
View File

@ -582,7 +582,7 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
return false; return false;
} }
SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions; SOperatorNode* pOper = (SOperatorNode*)pCond;
if (OP_TYPE_EQUAL != pOper->opType) { if (OP_TYPE_EQUAL != pOper->opType) {
return false; return false;
} }
@ -608,12 +608,16 @@ static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin,
if (LOGIC_COND_TYPE_AND != pOnCond->condType) { if (LOGIC_COND_TYPE_AND != pOnCond->condType) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
} }
bool hasPrimaryKeyEqualCond = false;
SNode* pCond = NULL; SNode* pCond = NULL;
FOREACH(pCond, pOnCond->pParameterList) { FOREACH(pCond, pOnCond->pParameterList) {
if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); hasPrimaryKeyEqualCond = true;
} }
} }
if (!hasPrimaryKeyEqualCond) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

18
source/libs/planner/src/planPhysiCreater.c
View File

@ -261,6 +261,22 @@ typedef struct SSetSlotIdCxt {
SHashObj* pRightHash; SHashObj* pRightHash;
} SSetSlotIdCxt; } SSetSlotIdCxt;
static void dumpSlots(const char* pName, SHashObj* pHash) {
if (NULL == pHash) {
return;
}
planDebug("%s", pName);
void* pIt = taosHashIterate(pHash, NULL);
while (NULL != pIt) {
size_t len = 0;
char* pKey = taosHashGetKey(pIt, &len);
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0};
strncpy(name, pKey, len);
planDebug("\tslot name = %s", name);
pIt = taosHashIterate(pHash, pIt);
}
}
static EDealRes doSetSlotId(SNode* pNode, void* pContext) { static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) {
SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext;
@ -273,6 +289,8 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
// pIndex is definitely not NULL, otherwise it is a bug // pIndex is definitely not NULL, otherwise it is a bug
if (NULL == pIndex) { if (NULL == pIndex) {
planError("doSetSlotId failed, invalid slot name %s", name); planError("doSetSlotId failed, invalid slot name %s", name);
dumpSlots("left datablock desc", pCxt->pLeftHash);
dumpSlots("right datablock desc", pCxt->pRightHash);
pCxt->errCode = TSDB_CODE_PLAN_INTERNAL_ERROR; pCxt->errCode = TSDB_CODE_PLAN_INTERNAL_ERROR;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }

14
source/libs/planner/test/planJoinTest.cpp
View File

@ -23,10 +23,16 @@ class PlanJoinTest : public PlannerTestBase {};
TEST_F(PlanJoinTest, basic) { TEST_F(PlanJoinTest, basic) {
useDb("root", "test"); useDb("root", "test");
run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); run("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts");
run("select t1.*, t2.* from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); run("SELECT t1.*, t2.* FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts");
// run("select t1.c1, t2.c1 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts where t1.c1 > t2.c1 and t1.c2 = 'abc' and " run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts");
// "t2.c2 = 'qwe'"); }
TEST_F(PlanJoinTest, withWhere) {
useDb("root", "test");
run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts "
"WHERE t1.c1 > t2.c1 AND t1.c2 = 'abc' AND t2.c2 = 'qwe'");
} }

8
source/libs/planner/test/planSubqueryTest.cpp
View File

@ -23,9 +23,13 @@ class PlanSubqeuryTest : public PlannerTestBase {};
TEST_F(PlanSubqeuryTest, basic) { TEST_F(PlanSubqeuryTest, basic) {
useDb("root", "test"); useDb("root", "test");
run("SELECT * FROM (SELECT * FROM t1)"); if (0 == g_skipSql) {
run("SELECT * FROM (SELECT * FROM t1)");
// run("SELECT LAST(c1) FROM ( SELECT * FROM t1)"); run("SELECT LAST(c1) FROM (SELECT * FROM t1)");
}
run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
} }
TEST_F(PlanSubqeuryTest, doubleGroupBy) { TEST_F(PlanSubqeuryTest, doubleGroupBy) {

32
source/libs/planner/test/planTestMain.cpp
View File

@ -25,23 +25,53 @@ class PlannerEnv : public testing::Environment {
virtual void SetUp() { virtual void SetUp() {
initMetaDataEnv(); initMetaDataEnv();
generateMetaData(); generateMetaData();
initLog("/tmp/td");
} }
virtual void TearDown() { destroyMetaDataEnv(); } virtual void TearDown() { destroyMetaDataEnv(); }
PlannerEnv() {} PlannerEnv() {}
virtual ~PlannerEnv() {} virtual ~PlannerEnv() {}
private:
void initLog(const char* path) {
dDebugFlag = 143;
vDebugFlag = 0;
mDebugFlag = 143;
cDebugFlag = 0;
jniDebugFlag = 0;
tmrDebugFlag = 135;
uDebugFlag = 135;
rpcDebugFlag = 143;
qDebugFlag = 143;
wDebugFlag = 0;
sDebugFlag = 0;
tsdbDebugFlag = 0;
tsLogEmbedded = 1;
tsAsyncLog = 0;
taosRemoveDir(path);
taosMkDir(path);
tstrncpy(tsLogDir, path, PATH_MAX);
if (taosInitLog("taoslog", 1) != 0) {
std::cout << "failed to init log file" << std::endl;
}
}
}; };
static void parseArg(int argc, char* argv[]) { static void parseArg(int argc, char* argv[]) {
int opt = 0; int opt = 0;
const char* optstring = ""; const char* optstring = "";
static struct option long_options[] = {{"dump", optional_argument, NULL, 'd'}, {0, 0, 0, 0}}; static struct option long_options[] = {
{"dump", optional_argument, NULL, 'd'}, {"skipSql", optional_argument, NULL, 's'}, {0, 0, 0, 0}};
while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) {
switch (opt) { switch (opt) {
case 'd': case 'd':
setDumpModule(optarg); setDumpModule(optarg);
break; break;
case 's':
g_skipSql = 1;
break;
default: default:
break; break;
} }

1
source/libs/planner/test/planTestUtil.cpp
View File

@ -47,6 +47,7 @@ enum DumpModule {
}; };
DumpModule g_dumpModule = DUMP_MODULE_NOTHING; DumpModule g_dumpModule = DUMP_MODULE_NOTHING;
int32_t g_skipSql = 0;
void setDumpModule(const char* pModule) { void setDumpModule(const char* pModule) {
if (NULL == pModule) { if (NULL == pModule) {

2
source/libs/planner/test/planTestUtil.h
View File

@ -32,6 +32,8 @@ class PlannerTestBase : public testing::Test {
std::unique_ptr<PlannerTestBaseImpl> impl_; std::unique_ptr<PlannerTestBaseImpl> impl_;
}; };
extern int32_t g_skipSql;
extern void setDumpModule(const char* pModule); extern void setDumpModule(const char* pModule);
#endif // PLAN_TEST_UTIL_H #endif // PLAN_TEST_UTIL_H

2
source/libs/scalar/inc/sclInt.h
View File

@ -27,13 +27,11 @@ typedef struct SScalarCtx {
SArray *pBlockList; /* element is SSDataBlock* */ SArray *pBlockList; /* element is SSDataBlock* */
SHashObj *pRes; /* element is SScalarParam */ SHashObj *pRes; /* element is SScalarParam */
void *param; // additional parameter (meta actually) for acquire value such as tbname/tags values void *param; // additional parameter (meta actually) for acquire value such as tbname/tags values
SHashObj *udf2Handle;
} SScalarCtx; } SScalarCtx;
#define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DATA_TYPE_DUMMY_HASH 9000
#define SCL_DEFAULT_OP_NUM 10 #define SCL_DEFAULT_OP_NUM 10
#define SCL_DEFAULT_UDF_NUM 8
#define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) #define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type))
#define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList)

50
source/libs/scalar/src/scalar.c
View File

@ -154,18 +154,6 @@ void sclFreeRes(SHashObj *res) {
taosHashCleanup(res); taosHashCleanup(res);
} }
void sclFreeUdfHandles(SHashObj *udf2handle) {
void *pIter = taosHashIterate(udf2handle, NULL);
while (pIter) {
UdfcFuncHandle *handle = (UdfcFuncHandle *)pIter;
if (handle) {
teardownUdf(*handle);
}
pIter = taosHashIterate(udf2handle, pIter);
}
taosHashCleanup(udf2handle);
}
void sclFreeParam(SScalarParam *param) { void sclFreeParam(SScalarParam *param) {
if (param->columnData != NULL) { if (param->columnData != NULL) {
colDataDestroy(param->columnData); colDataDestroy(param->columnData);
@ -374,25 +362,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum)); SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
if (fmIsUserDefinedFunc(node->funcId)) { if (fmIsUserDefinedFunc(node->funcId)) {
UdfcFuncHandle udfHandle = NULL; code = callUdfScalarFunc(node->functionName, params, paramNum, output);
char* udfName = node->functionName;
if (ctx->udf2Handle) {
UdfcFuncHandle *pHandle = taosHashGet(ctx->udf2Handle, udfName, strlen(udfName));
if (pHandle) {
udfHandle = *pHandle;
}
}
if (udfHandle == NULL) {
code = setupUdf(udfName, &udfHandle);
if (code != 0) {
sclError("fmExecFunction error. setupUdf. function name: %s, code:%d", udfName, code);
goto _return;
}
if (ctx->udf2Handle) {
taosHashPut(ctx->udf2Handle, udfName, strlen(udfName), &udfHandle, sizeof(UdfcFuncHandle));
}
}
code = callUdfScalarFunc(udfHandle, params, paramNum, output);
if (code != 0) { if (code != 0) {
sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code); sclError("fmExecFunction error. callUdfScalarFunc. function name: %s, udf code:%d", node->functionName, code);
goto _return; goto _return;
@ -910,20 +880,15 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) {
SScalarCtx ctx = {0}; SScalarCtx ctx = {0};
ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (NULL == ctx.pRes) { if (NULL == ctx.pRes) {
sclError("taosHashInit result map failed, num:%d", SCL_DEFAULT_OP_NUM); sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM);
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
ctx.udf2Handle = taosHashInit(SCL_DEFAULT_UDF_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (NULL == ctx.udf2Handle) {
sclError("taosHashInit udf to handle map failed, num:%d", SCL_DEFAULT_OP_NUM);
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx);
SCL_ERR_JRET(ctx.code); SCL_ERR_JRET(ctx.code);
*pRes = pNode; *pRes = pNode;
_return: _return:
sclFreeUdfHandles(ctx.udf2Handle);
sclFreeRes(ctx.pRes); sclFreeRes(ctx.pRes);
return code; return code;
} }
@ -939,14 +904,10 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
// TODO: OPT performance // TODO: OPT performance
ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (NULL == ctx.pRes) { if (NULL == ctx.pRes) {
sclError("taosHashInit result map failed, num:%d", SCL_DEFAULT_OP_NUM); sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM);
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
ctx.udf2Handle = taosHashInit(SCL_DEFAULT_UDF_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (NULL == ctx.udf2Handle) {
sclError("taosHashInit udf to handle map failed, num:%d", SCL_DEFAULT_OP_NUM);
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx);
SCL_ERR_JRET(ctx.code); SCL_ERR_JRET(ctx.code);
@ -964,7 +925,6 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
_return: _return:
//nodesDestroyNode(pNode); //nodesDestroyNode(pNode);
sclFreeUdfHandles(ctx.udf2Handle);
sclFreeRes(ctx.pRes); sclFreeRes(ctx.pRes);
return code; return code;
} }

405
source/libs/stream/src/tstream.c
View File

@ -16,6 +16,25 @@
#include "tstream.h" #include "tstream.h"
#include "executor.h" #include "executor.h"
int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pOutput->type);
tlen += taosEncodeFixedI32(buf, pOutput->sourceVg);
tlen += taosEncodeFixedI64(buf, pOutput->sourceVer);
ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK);
tlen += tEncodeDataBlocks(buf, pOutput->blocks);
return tlen;
}
void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
buf = taosDecodeFixedI8(buf, &pInput->type);
buf = taosDecodeFixedI32(buf, &pInput->sourceVg);
buf = taosDecodeFixedI64(buf, &pInput->sourceVer);
ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
buf = tDecodeDataBlocks(buf, &pInput->blocks);
return (void*)buf;
}
static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
SStreamTaskExecReq req = { SStreamTaskExecReq req = {
.streamId = pTask->streamId, .streamId = pTask->streamId,
@ -97,6 +116,363 @@ static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashOb
return 0; return 0;
} }
int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input) {
ASSERT(pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK);
int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
streamDataSubmitRefInc(input);
taosWriteQitem(pTask->inputQ, input);
}
return inputStatus;
}
int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK);
taosWriteQitem(pTask->inputQ, input);
int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
return inputStatus;
}
int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
void* exec = pTask->exec.runners[0].executor;
// set input
if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT);
qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
} else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
SArray* blocks = pBlock->blocks;
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
}
// exec
while (1) {
SSDataBlock* output;
uint64_t ts = 0;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
if (output == NULL) break;
taosArrayPush(pRes, &output);
}
// destroy
if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
streamDataSubmitRefDec((SStreamDataSubmit*)data);
} else {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
}
return 0;
}
// TODO: handle version
int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
if (pRes == NULL) return -1;
while (1) {
int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
void* exec = pTask->exec.runners[0].executor;
if (execStatus == TASK_STATUS__IDLE) {
// first run, from qall, handle failure from last exec
while (1) {
void* data = NULL;
taosGetQitem(pTask->inputQAll, &data);
if (data == NULL) break;
streamTaskExecImpl(pTask, data, pRes);
taosFreeQitem(data);
if (taosArrayGetSize(pRes) != 0) {
SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
resQ->type = STREAM_INPUT__DATA_BLOCK;
resQ->blocks = pRes;
taosWriteQitem(pTask->outputQ, resQ);
pRes = taosArrayInit(0, sizeof(SSDataBlock));
if (pRes == NULL) goto FAIL;
}
}
// second run, from inputQ
taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
while (1) {
void* data = NULL;
taosGetQitem(pTask->inputQAll, &data);
if (data == NULL) break;
streamTaskExecImpl(pTask, data, pRes);
taosFreeQitem(data);
if (taosArrayGetSize(pRes) != 0) {
SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
resQ->type = STREAM_INPUT__DATA_BLOCK;
resQ->blocks = pRes;
taosWriteQitem(pTask->outputQ, resQ);
pRes = taosArrayInit(0, sizeof(SSDataBlock));
if (pRes == NULL) goto FAIL;
}
}
// set status closing
atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
// third run, make sure all inputQ is cleared
taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
while (1) {
void* data = NULL;
taosGetQitem(pTask->inputQAll, &data);
if (data == NULL) break;
streamTaskExecImpl(pTask, data, pRes);
taosFreeQitem(data);
if (taosArrayGetSize(pRes) != 0) {
SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
resQ->type = STREAM_INPUT__DATA_BLOCK;
resQ->blocks = pRes;
taosWriteQitem(pTask->outputQ, resQ);
pRes = taosArrayInit(0, sizeof(SSDataBlock));
if (pRes == NULL) goto FAIL;
}
}
// set status closing
atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
// third run, make sure all inputQ is cleared
taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
while (1) {
void* data = NULL;
taosGetQitem(pTask->inputQAll, &data);
if (data == NULL) break;
}
atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
break;
} else if (execStatus == TASK_STATUS__CLOSING) {
continue;
} else if (execStatus == TASK_STATUS__EXECUTING) {
break;
} else {
ASSERT(0);
}
}
return 0;
FAIL:
atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
return -1;
}
int32_t streamTaskDispatchDown(SStreamTask* pTask, SMsgCb* pMsgCb) {
//
return 0;
}
int32_t streamTaskSink(SStreamTask* pTask) {
//
return 0;
}
int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDataBlock* pBlock, SRpcMsg* pRsp) {
// 1. handle input
// 1.1 enqueue
taosWriteQitem(pTask->inputQ, pBlock);
// 1.2 calc back pressure
// 1.3 rsp by input status
int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
pCont->status = inputStatus;
pRsp->pCont = pCont;
pRsp->contLen = sizeof(SStreamDispatchRsp);
tmsgSendRsp(pRsp);
// 2. try exec
// 2.1. idle: exec
// 2.2. executing: return
// 2.3. closing: keep trying
while (1) {
int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
if (execStatus == TASK_STATUS__IDLE) {
void* exec = pTask->exec.runners[0].executor;
SArray* pRes = taosArrayInit(0, sizeof(void*));
const SArray* blocks = pBlock->blocks;
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
while (1) {
SSDataBlock* output;
uint64_t ts = 0;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
if (output == NULL) break;
taosArrayPush(pRes, &output);
}
// TODO: wrap destroy block
taosArrayDestroyP(pBlock->blocks, (FDelete)blockDataDestroy);
if (taosArrayGetSize(pRes) != 0) {
SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
*resQ = pRes;
taosWriteQitem(pTask->outputQ, resQ);
}
} else if (execStatus == TASK_STATUS__CLOSING) {
continue;
} else if (execStatus == TASK_STATUS__EXECUTING)
break;
else {
ASSERT(0);
}
}
// 3. handle output
// 3.1 check and set status
// 3.2 dispatch / sink
STaosQall* qall = taosAllocateQall();
taosReadAllQitems(pTask->outputQ, qall);
SArray** ppRes = NULL;
while (1) {
taosGetQitem(qall, (void**)&ppRes);
if (ppRes == NULL) break;
SArray* pRes = *ppRes;
if (pTask->sinkType == TASK_SINK__TABLE) {
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, pBlock->sourceVer, pRes);
} else if (pTask->sinkType == TASK_SINK__SMA) {
pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
} else {
}
// dispatch
if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
SRpcMsg dispatchMsg = {0};
if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
ASSERT(0);
return -1;
}
int32_t qType;
if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) {
qType = FETCH_QUEUE;
} else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||
pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {
qType = MERGE_QUEUE;
} else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {
qType = WRITE_QUEUE;
} else {
ASSERT(0);
}
tmsgPutToQueue(pMsgCb, qType, &dispatchMsg);
} else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
SRpcMsg dispatchMsg = {0};
SEpSet* pEpSet = NULL;
if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
ASSERT(0);
return -1;
}
tmsgSendReq(pMsgCb, pEpSet, &dispatchMsg);
} else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (pShuffleRes == NULL) {
return -1;
}
int32_t sz = taosArrayGetSize(pRes);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGet(pRes, i);
SArray* pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t));
if (pArray == NULL) {
pArray = taosArrayInit(0, sizeof(SSDataBlock));
if (pArray == NULL) {
return -1;
}
taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*));
}
taosArrayPush(pArray, pDataBlock);
}
if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) {
return -1;
}
} else {
ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
}
}
//
return 0;
}
int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, char* msg, int32_t msgLen) {
//
return 0;
}
int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) {
//
return 0;
}
int32_t streamTaskRun(SStreamTask* pTask) {
SArray* pRes = NULL;
if (pTask->execType == TASK_EXEC__PIPE || pTask->execType == TASK_EXEC__MERGE) {
// TODO remove multi runner
void* exec = pTask->exec.runners[0].executor;
int8_t status = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
if (status == TASK_STATUS__IDLE) {
pRes = taosArrayInit(0, sizeof(void*));
if (pRes == NULL) {
return -1;
}
void* input = NULL;
taosWriteQitem(pTask->inputQ, &input);
if (input == NULL) return 0;
// TODO: fix type
if (pTask->sourceType == TASK_SOURCE__SCAN) {
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input;
qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
while (1) {
SSDataBlock* output;
uint64_t ts = 0;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
if (output == NULL) break;
taosArrayPush(pRes, &output);
}
streamDataSubmitRefDec(pSubmit);
} else {
SStreamDataBlock* pStreamBlock = (SStreamDataBlock*)input;
const SArray* blocks = pStreamBlock->blocks;
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
while (1) {
SSDataBlock* output;
uint64_t ts = 0;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
if (output == NULL) break;
taosArrayPush(pRes, &output);
}
// TODO: wrap destroy block
taosArrayDestroyP(pStreamBlock->blocks, (FDelete)blockDataDestroy);
}
if (taosArrayGetSize(pRes) != 0) {
SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
*resQ = pRes;
taosWriteQitem(pTask->outputQ, resQ);
}
}
}
return 0;
}
int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) { int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) {
SArray* pRes = NULL; SArray* pRes = NULL;
// source // source
@ -251,15 +627,29 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
} }
pTask->taskId = tGenIdPI32(); pTask->taskId = tGenIdPI32();
pTask->streamId = streamId; pTask->streamId = streamId;
pTask->status = STREAM_TASK_STATUS__RUNNING; pTask->status = TASK_STATUS__IDLE;
/*pTask->qmsg = NULL;*/
pTask->inputQ = taosOpenQueue();
pTask->outputQ = taosOpenQueue();
pTask->inputQAll = taosAllocateQall();
pTask->outputQAll = taosAllocateQall();
if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
goto FAIL;
return pTask; return pTask;
FAIL:
if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
if (pTask) taosMemoryFree(pTask);
return NULL;
} }
int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
/*if (tStartEncode(pEncoder) < 0) return -1;*/ /*if (tStartEncode(pEncoder) < 0) return -1;*/
if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
@ -305,6 +695,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
/*if (tStartDecode(pDecoder) < 0) return -1;*/ /*if (tStartDecode(pDecoder) < 0) return -1;*/
if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
@ -349,10 +740,16 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
} }
void tFreeSStreamTask(SStreamTask* pTask) { void tFreeSStreamTask(SStreamTask* pTask) {
taosCloseQueue(pTask->inputQ);
taosCloseQueue(pTask->outputQ);
// TODO // TODO
/*taosMemoryFree(pTask->qmsg);*/ if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
for (int32_t i = 0; i < pTask->exec.numOfRunners; i++) {
qDestroyTask(pTask->exec.runners[i].executor);
}
taosMemoryFree(pTask->exec.runners);
/*taosMemoryFree(pTask->executor);*/ /*taosMemoryFree(pTask->executor);*/
/*taosMemoryFree(pTask);*/ taosMemoryFree(pTask);
} }
#if 0 #if 0

2
source/libs/tdb/CMakeLists.txt
View File

@ -1,5 +1,5 @@
# tdb # tdb
add_library(tdb SHARED "") add_library(tdb STATIC "")
target_sources(tdb target_sources(tdb
PRIVATE PRIVATE
"src/db/tdbPCache.c" "src/db/tdbPCache.c"

3
source/util/src/terror.c
View File

@ -354,6 +354,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "Tsma already exists")
// query // query
@ -463,7 +464,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe")
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure")
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state")
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input")
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle")
//schemaless //schemaless
TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type")
TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type")

2
source/util/src/tlog.c
View File

@ -94,6 +94,7 @@ int32_t tqDebugFlag = 135;
int32_t fsDebugFlag = 135; int32_t fsDebugFlag = 135;
int32_t metaDebugFlag = 135; int32_t metaDebugFlag = 135;
int32_t fnDebugFlag = 135; int32_t fnDebugFlag = 135;
int32_t smaDebugFlag = 135;
int64_t dbgEmptyW = 0; int64_t dbgEmptyW = 0;
int64_t dbgWN = 0; int64_t dbgWN = 0;
@ -755,6 +756,7 @@ void taosSetAllDebugFlag(int32_t flag) {
tqDebugFlag = flag; tqDebugFlag = flag;
fsDebugFlag = flag; fsDebugFlag = flag;
fnDebugFlag = flag; fnDebugFlag = flag;
smaDebugFlag = flag;
uInfo("all debug flag are set to %d", flag); uInfo("all debug flag are set to %d", flag);
} }

2
tests/script/tsim/tstream/basic1.sim
View File

@ -136,7 +136,7 @@ if $data35 != 3 then
endi endi
sql insert into t1 values(1648791223001,12,14,13,11.1); sql insert into t1 values(1648791223001,12,14,13,11.1);
sleep 100 sleep 500
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt;
if $rows != 4 then if $rows != 4 then

544
tests/system-test/0-others/udfTest.py Normal file
View File

@ -0,0 +1,544 @@
import taos
import sys
import time
import os
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
import subprocess
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def prepare_udf_so(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
print(projPath)
libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
os.system("mkdir /tmp/udf/")
os.system("sudo cp %s /tmp/udf/ "%libudf1.replace("\n" ,""))
os.system("sudo cp %s /tmp/udf/ "%libudf2.replace("\n" ,""))
def prepare_data(self):
tdSql.execute("use db")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t1 int)
'''
)
tdSql.execute(
'''
create table t1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
'''
)
for i in range(4):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
for i in range(9):
tdSql.execute(
f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute(
f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
)
tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute(
f'''insert into t1 values
( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
'''
)
tdSql.execute("create table tb (ts timestamp , num1 int , num2 int, num3 double , num4 binary(30))")
tdSql.execute(
f'''insert into tb values
( '2020-04-21 01:01:01.000', NULL, 1, 1, "binary1" )
( '2020-10-21 01:01:01.000', 1, 1, 1.11, "binary1" )
( '2020-12-31 01:01:01.000', 2, 22222, 22, "binary1" )
( '2021-01-01 01:01:06.000', 3, 33333, 33, "binary1" )
( '2021-05-07 01:01:10.000', 4, 44444, 44, "binary1" )
( '2021-07-21 01:01:01.000', NULL, NULL, NULL, "binary1" )
( '2021-09-30 01:01:16.000', 5, 55555, 55, "binary1" )
( '2022-02-01 01:01:20.000', 6, 66666, 66, "binary1" )
( '2022-10-28 01:01:26.000', 0, 00000, 00, "binary1" )
( '2022-12-01 01:01:30.000', 8, -88888, -88, "binary1" )
( '2022-12-31 01:01:36.000', 9, -9999999, -99, "binary1" )
( '2023-02-21 01:01:01.000', NULL, NULL, NULL, "binary1" )
'''
)
def create_udf_function(self):
for i in range(10):
# create scalar functions
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
# create aggregate functions
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
functions = tdSql.getResult("show functions")
function_nums = len(functions)
if function_nums == 2:
tdLog.info("create two udf functions success ")
# drop functions
tdSql.execute("drop function udf1")
tdSql.execute("drop function udf2")
functions = tdSql.getResult("show functions")
for function in functions:
if "udf1" in function[0] or "udf2" in function[0]:
tdLog.info("drop udf functions failed ")
tdLog.exit("drop udf functions failed")
tdLog.info("drop two udf functions success ")
# create scalar functions
tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
# create aggregate functions
tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
functions = tdSql.getResult("show functions")
function_nums = len(functions)
if function_nums == 2:
tdLog.info("create two udf functions success ")
def basic_udf_query(self):
# scalar functions
tdSql.execute("use db ")
tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb")
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(0,2,1)
tdSql.checkData(0,3,88)
tdSql.checkData(0,4,1.000000000)
tdSql.checkData(0,5,88)
tdSql.checkData(0,6,"binary1")
tdSql.checkData(0,7,88)
tdSql.checkData(3,0,3)
tdSql.checkData(3,1,88)
tdSql.checkData(3,2,33333)
tdSql.checkData(3,3,88)
tdSql.checkData(3,4,33.000000000)
tdSql.checkData(3,5,88)
tdSql.checkData(3,6,"binary1")
tdSql.checkData(3,7,88)
tdSql.checkData(11,0,None)
tdSql.checkData(11,1,None)
tdSql.checkData(11,2,None)
tdSql.checkData(11,3,None)
tdSql.checkData(11,4,None)
tdSql.checkData(11,5,None)
tdSql.checkData(11,6,"binary1")
tdSql.checkData(11,7,88)
tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1")
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(0,2,None)
tdSql.checkData(0,3,None)
tdSql.checkData(0,4,None)
tdSql.checkData(0,5,None)
tdSql.checkData(0,6,None)
tdSql.checkData(0,7,None)
tdSql.checkData(20,0,8)
tdSql.checkData(20,1,88)
tdSql.checkData(20,2,88888)
tdSql.checkData(20,3,88)
tdSql.checkData(20,4,888)
tdSql.checkData(20,5,88)
tdSql.checkData(20,6,88)
tdSql.checkData(20,7,88)
# aggregate functions
tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb")
tdSql.checkData(0,0,15.362291496)
tdSql.checkData(0,1,10000949.553189287)
tdSql.checkData(0,2,168.633425216)
# Arithmetic compute
tdSql.query("select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb")
tdSql.checkData(0,0,115.362291496)
tdSql.checkData(0,1,10000849.553189287)
tdSql.checkData(0,2,16863.342521576)
tdSql.checkData(0,3,1.686334252)
tdSql.query("select udf2(c1) ,udf2(c6) from stb1 ")
tdSql.checkData(0,0,25.514701644)
tdSql.checkData(0,1,265.247614504)
tdSql.query("select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 ")
tdSql.checkData(0,0,125.514701644)
tdSql.checkData(0,1,165.247614504)
tdSql.checkData(0,2,2551.470164435)
tdSql.checkData(0,3,2.652476145)
# # bug for crash when query sub table
tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1")
tdSql.checkData(0,0,378.215547010)
tdSql.checkData(0,1,353.808067460)
tdSql.checkData(0,2,2114.237451187)
tdSql.checkData(0,3,2.125468151)
tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ")
tdSql.checkData(0,0,490.358032462)
tdSql.checkData(0,1,400.460106627)
tdSql.checkData(0,2,2551.470164435)
tdSql.checkData(0,3,2.652476145)
# regular table with aggregate functions
tdSql.error("select udf1(num1) , count(num1) from tb;")
tdSql.error("select udf1(num1) , avg(num1) from tb;")
tdSql.error("select udf1(num1) , twa(num1) from tb;")
tdSql.error("select udf1(num1) , irate(num1) from tb;")
tdSql.error("select udf1(num1) , sum(num1) from tb;")
tdSql.error("select udf1(num1) , stddev(num1) from tb;")
tdSql.error("select udf1(num1) , mode(num1) from tb;")
tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;")
# stable
tdSql.error("select udf1(c1) , count(c1) from stb1;")
tdSql.error("select udf1(c1) , avg(c1) from stb1;")
tdSql.error("select udf1(c1) , twa(c1) from stb1;")
tdSql.error("select udf1(c1) , irate(c1) from stb1;")
tdSql.error("select udf1(c1) , sum(c1) from stb1;")
tdSql.error("select udf1(c1) , stddev(c1) from stb1;")
tdSql.error("select udf1(c1) , mode(c1) from stb1;")
tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;")
# regular table with select functions
tdSql.query("select udf1(num1) , max(num1) from tb;")
tdSql.checkRows(1)
tdSql.query("select floor(num1) , max(num1) from tb;")
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , min(num1) from tb;")
tdSql.checkRows(1)
tdSql.query("select ceil(num1) , min(num1) from tb;")
tdSql.checkRows(1)
tdSql.error("select udf1(num1) , first(num1) from tb;")
tdSql.error("select abs(num1) , first(num1) from tb;")
tdSql.error("select udf1(num1) , last(num1) from tb;")
tdSql.error("select round(num1) , last(num1) from tb;")
tdSql.query("select udf1(num1) , top(num1,1) from tb;")
tdSql.checkRows(1)
tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
tdSql.checkRows(1)
tdSql.error("select udf1(num1) , last_row(num1) from tb;")
tdSql.error("select round(num1) , last_row(num1) from tb;")
# stable
tdSql.query("select udf1(c1) , max(c1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select abs(c1) , max(c1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select udf1(c1) , min(c1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select floor(c1) , min(c1) from stb1;")
tdSql.checkRows(1)
tdSql.error("select udf1(c1) , first(c1) from stb1;")
tdSql.error("select udf1(c1) , last(c1) from stb1;")
tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select abs(c1) , top(c1 ,1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select udf1(c1) , bottom(c1,1) from stb1;")
tdSql.checkRows(1)
tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
tdSql.checkRows(1)
tdSql.error("select udf1(c1) , last_row(c1) from stb1;")
tdSql.error("select ceil(c1) , last_row(c1) from stb1;")
# regular table with compute functions
tdSql.query("select udf1(num1) , abs(num1) from tb;")
tdSql.checkRows(12)
tdSql.query("select floor(num1) , abs(num1) from tb;")
tdSql.checkRows(12)
# # bug need fix
tdSql.query("select udf1(num1) , csum(num1) from tb;")
tdSql.checkRows(9)
tdSql.query("select ceil(num1) , csum(num1) from tb;")
tdSql.checkRows(9)
tdSql.query("select udf1(c1) , csum(c1) from stb1;")
tdSql.checkRows(22)
tdSql.query("select floor(c1) , csum(c1) from stb1;")
tdSql.checkRows(22)
# stable with compute functions
tdSql.query("select udf1(c1) , abs(c1) from stb1;")
tdSql.checkRows(25)
tdSql.query("select abs(c1) , ceil(c1) from stb1;")
tdSql.checkRows(25)
# nest query
tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;")
tdSql.checkRows(25)
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.checkData(1,0,88)
tdSql.checkData(1,1,8)
tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;")
tdSql.checkRows(13)
tdSql.checkData(0,0,88)
tdSql.checkData(0,1,8)
tdSql.checkData(1,0,88)
tdSql.checkData(1,1,7)
# bug fix for crash
# order by udf function result
for _ in range(50):
tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)")
print(tdSql.queryResult)
# udf functions with filter
tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 where c1 is null order by ts;")
tdSql.checkRows(3)
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,None)
tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts")
tdSql.checkRows(3)
tdSql.checkData(0,0,9)
tdSql.checkData(0,1,88)
tdSql.checkData(0,2,-99.990000000)
tdSql.checkData(0,3,88)
# udf functions with join
ts_start = 1652517451000
tdSql.execute("create stable st (ts timestamp , c1 int , c2 int ,c3 double ,c4 double ) tags(ind int)")
tdSql.execute("create table sub1 using st tags(1)")
tdSql.execute("create table sub2 using st tags(2)")
for i in range(10):
ts = ts_start + i *1000
tdSql.execute(" insert into sub1 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0))
tdSql.execute(" insert into sub2 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0))
tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
tdSql.checkData(0,1,0)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,10)
tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,88)
tdSql.checkData(0,1,88)
tdSql.checkData(1,0,88)
tdSql.checkData(1,1,88)
tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,0)
tdSql.checkData(0,1,88)
tdSql.checkData(0,2,0)
tdSql.checkData(0,3,88)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,88)
tdSql.checkData(1,2,10)
tdSql.checkData(1,3,88)
tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,16.881943016)
tdSql.checkData(0,1,168.819430161)
tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
# udf functions with group by
tdSql.query("select udf1(c1) from ct1 group by c1")
tdSql.checkRows(10)
tdSql.query("select udf1(c1) from stb1 group by c1")
tdSql.checkRows(11)
tdSql.query("select c1,c2, udf1(c1,c2) from ct1 group by c1,c2")
tdSql.checkRows(10)
tdSql.query("select c1,c2, udf1(c1,c2) from stb1 group by c1,c2")
tdSql.checkRows(11)
tdSql.query("select udf2(c1) from ct1 group by c1")
tdSql.checkRows(10)
tdSql.query("select udf2(c1) from stb1 group by c1")
tdSql.checkRows(11)
tdSql.query("select c1,c2, udf2(c1,c6) from ct1 group by c1,c2")
tdSql.checkRows(10)
tdSql.query("select c1,c2, udf2(c1,c6) from stb1 group by c1,c2")
tdSql.checkRows(11)
tdSql.query("select udf2(c1) from stb1 group by udf1(c1)")
tdSql.checkRows(2)
tdSql.query("select udf2(c1) from stb1 group by floor(c1)")
tdSql.checkRows(11)
# udf mix with order by
tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)")
tdSql.checkRows(11)
def multi_cols_udf(self):
tdSql.query("select num1,num2,num3,udf1(num1,num2,num3) from tb")
tdSql.checkData(0,0,None)
tdSql.checkData(0,1,1)
tdSql.checkData(0,2,1.000000000)
tdSql.checkData(0,3,None)
tdSql.checkData(1,0,1)
tdSql.checkData(1,1,1)
tdSql.checkData(1,2,1.110000000)
tdSql.checkData(1,3,88)
tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts")
tdSql.checkData(1,0,8)
tdSql.checkData(1,1,88.880000000)
tdSql.checkData(1,2,88)
tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;")
tdSql.checkRows(22)
tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,169.661427555)
tdSql.checkData(0,1,169.661427555)
def unexpected_create(self):
tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
def loop_kill_udfd(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
cfgPath = buildPath + "/../sim/dnode1/cfg"
udfdPath = buildPath +'/build/bin/udfd'
for i in range(5):
tdLog.info(" loop restart udfd %d_th" % i)
tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,169.661427555)
tdSql.checkData(0,1,169.661427555)
# stop udfd cmds
get_processID = "ps -ef | grep -w udfd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}'"
processID = subprocess.check_output(get_processID, shell=True).decode("utf-8")
stop_udfd = " kill -9 %s" % processID
os.system(stop_udfd)
time.sleep(2)
tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,169.661427555)
tdSql.checkData(0,1,169.661427555)
# # start udfd cmds
# start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &"
# tdLog.info("start udfd : %s " % start_udfd)
def restart_taosd_query_udf(self):
for i in range(5):
time.sleep(5)
tdLog.info(" this is %d_th restart taosd " %i)
tdSql.execute("use db ")
tdSql.query("select count(*) from stb1")
tdSql.checkRows(1)
tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
tdSql.checkData(0,0,169.661427555)
tdSql.checkData(0,1,169.661427555)
tdDnodes.stop(1)
time.sleep(2)
tdDnodes.start(1)
time.sleep(5)
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
self.prepare_udf_so()
self.prepare_data()
self.create_udf_function()
self.basic_udf_query()
self.loop_kill_udfd()
# self.restart_taosd_query_udf()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

1
tests/system-test/fulltest.sh
View File

@ -7,6 +7,7 @@ python3 ./test.py -f 0-others/taosShellError.py
python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/taosShellNetChk.py
python3 ./test.py -f 0-others/telemetry.py python3 ./test.py -f 0-others/telemetry.py
python3 ./test.py -f 0-others/taosdMonitor.py python3 ./test.py -f 0-others/taosdMonitor.py
python3 ./test.py -f 0-others/udfTest.py
python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/user_control.py

1
tools/shell/src/shellEngine.c
View File

@ -604,6 +604,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
return TMAX(25, width); return TMAX(25, width);
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
if (field->bytes > shell.args.displayWidth) { if (field->bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width); return TMAX(shell.args.displayWidth, width);