diff --git a/include/common/taosdef.h b/include/common/taosdef.h
index e1f8832edf..72d2c142d2 100644
--- a/include/common/taosdef.h
+++ b/include/common/taosdef.h
@@ -37,7 +37,8 @@ typedef enum {
TSDB_STREAM_TABLE = 4, // table created from stream computing
TSDB_TEMP_TABLE = 5, // temp table created by nest query
TSDB_SYSTEM_TABLE = 6,
- TSDB_TABLE_MAX = 7
+ TSDB_TSMA_TABLE = 7, // time-range-wise sma
+ TSDB_TABLE_MAX = 8
} ETableType;
typedef enum {
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index ae21986c56..b54ea58c4e 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2160,26 +2160,23 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
typedef struct {
- int8_t version; // for compatibility(default 0)
- int8_t intervalUnit; // MACRO: TIME_UNIT_XXX
- int8_t slidingUnit; // MACRO: TIME_UNIT_XXX
- int8_t timezoneInt; // sma data expired if timezone changes.
- char indexName[TSDB_INDEX_NAME_LEN];
- int32_t exprLen;
- int32_t tagsFilterLen;
- int64_t indexUid;
- tb_uid_t tableUid; // super/child/common table uid
- int64_t interval;
- int64_t offset; // use unit by precision of DB
- int64_t sliding;
- char* expr; // sma expression
- char* tagsFilter;
+ int8_t version; // for compatibility(default 0)
+ int8_t intervalUnit; // MACRO: TIME_UNIT_XXX
+ int8_t slidingUnit; // MACRO: TIME_UNIT_XXX
+ int8_t timezoneInt; // sma data expired if timezone changes.
+ char indexName[TSDB_INDEX_NAME_LEN];
+ int32_t exprLen;
+ int32_t tagsFilterLen;
+ int64_t indexUid;
+ tb_uid_t tableUid; // super/child/common table uid
+ int64_t interval;
+ int64_t offset; // use unit by precision of DB
+ int64_t sliding;
+ const char* expr; // sma expression
+ const char* tagsFilter;
} STSma; // Time-range-wise SMA
-typedef struct {
- int64_t ver; // use a general definition
- STSma tSma;
-} SVCreateTSmaReq;
+typedef STSma SVCreateTSmaReq;
typedef struct {
int8_t type; // 0 status report, 1 update data
@@ -2188,7 +2185,6 @@ typedef struct {
} STSmaMsg;
typedef struct {
- int64_t ver; // use a general definition
int64_t indexUid;
char indexName[TSDB_INDEX_NAME_LEN];
} SVDropTSmaReq;
@@ -2197,28 +2193,21 @@ typedef struct {
int tmp; // TODO: to avoid compile error
} SVCreateTSmaRsp, SVDropTSmaRsp;
+#if 0
int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq);
void* tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq);
int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
+#endif
-// RSma: Rollup SMA
-typedef struct {
- int64_t interval;
- int32_t retention; // unit: day
- uint16_t days; // unit: day
- int8_t intervalUnit;
-} SSmaParams;
+int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
+int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
+int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq);
+int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
typedef struct {
- STSma tsma;
- float xFilesFactor;
- SArray* smaParams; // SSmaParams
-} SRSma;
-
-typedef struct {
- uint32_t number;
- STSma* tSma;
+ int32_t number;
+ STSma* tSma;
} STSmaWrapper;
static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
@@ -2245,96 +2234,26 @@ static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
return NULL;
}
-static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
- int32_t tlen = 0;
+int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
+int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
- tlen += taosEncodeFixedI8(buf, pSma->version);
- tlen += taosEncodeFixedI8(buf, pSma->intervalUnit);
- tlen += taosEncodeFixedI8(buf, pSma->slidingUnit);
- tlen += taosEncodeFixedI8(buf, pSma->timezoneInt);
- tlen += taosEncodeString(buf, pSma->indexName);
- tlen += taosEncodeFixedI32(buf, pSma->exprLen);
- tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen);
- tlen += taosEncodeFixedI64(buf, pSma->indexUid);
- tlen += taosEncodeFixedI64(buf, pSma->tableUid);
- tlen += taosEncodeFixedI64(buf, pSma->interval);
- tlen += taosEncodeFixedI64(buf, pSma->offset);
- tlen += taosEncodeFixedI64(buf, pSma->sliding);
+int32_t tEncodeTSma(SEncoder* pCoder, const STSma* pSma);
+int32_t tDecodeTSma(SDecoder* pCoder, STSma* pSma);
- if (pSma->exprLen > 0) {
- tlen += taosEncodeString(buf, pSma->expr);
+static int32_t tEncodeTSmaWrapper(SEncoder* pEncoder, const STSmaWrapper* pReq) {
+ if (tEncodeI32(pEncoder, pReq->number) < 0) return -1;
+ for (int32_t i = 0; i < pReq->number; ++i) {
+ tEncodeTSma(pEncoder, pReq->tSma + i);
}
-
- if (pSma->tagsFilterLen > 0) {
- tlen += taosEncodeString(buf, pSma->tagsFilter);
- }
-
- return tlen;
+ return 0;
}
-static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) {
- int32_t tlen = 0;
-
- tlen += taosEncodeFixedU32(buf, pSW->number);
- for (uint32_t i = 0; i < pSW->number; ++i) {
- tlen += tEncodeTSma(buf, pSW->tSma + i);
+static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) {
+ if (tDecodeI32(pDecoder, &pReq->number) < 0) return -1;
+ for (int32_t i = 0; i < pReq->number; ++i) {
+ tDecodeTSma(pDecoder, pReq->tSma + i);
}
- return tlen;
-}
-
-static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
- buf = taosDecodeFixedI8(buf, &pSma->version);
- buf = taosDecodeFixedI8(buf, &pSma->intervalUnit);
- buf = taosDecodeFixedI8(buf, &pSma->slidingUnit);
- buf = taosDecodeFixedI8(buf, &pSma->timezoneInt);
- buf = taosDecodeStringTo(buf, pSma->indexName);
- buf = taosDecodeFixedI32(buf, &pSma->exprLen);
- buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen);
- buf = taosDecodeFixedI64(buf, &pSma->indexUid);
- buf = taosDecodeFixedI64(buf, &pSma->tableUid);
- buf = taosDecodeFixedI64(buf, &pSma->interval);
- buf = taosDecodeFixedI64(buf, &pSma->offset);
- buf = taosDecodeFixedI64(buf, &pSma->sliding);
-
- if (pSma->exprLen > 0) {
- if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
- tdDestroyTSma(pSma);
- return NULL;
- }
- } else {
- pSma->expr = NULL;
- }
-
- if (pSma->tagsFilterLen > 0) {
- if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
- tdDestroyTSma(pSma);
- return NULL;
- }
- } else {
- pSma->tagsFilter = NULL;
- }
-
- return buf;
-}
-
-static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
- buf = taosDecodeFixedU32(buf, &pSW->number);
-
- pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma));
- if (pSW->tSma == NULL) {
- return NULL;
- }
-
- for (uint32_t i = 0; i < pSW->number; ++i) {
- if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) {
- for (uint32_t j = i; j >= 0; --i) {
- tdDestroyTSma(pSW->tSma + j);
- }
- taosMemoryFree(pSW->tSma);
- return NULL;
- }
- }
- return buf;
+ return 0;
}
typedef struct {
@@ -2574,6 +2493,14 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
}
+typedef struct {
+ void* data;
+} SStreamDispatchReq;
+
+typedef struct {
+ int8_t status;
+} SStreamDispatchRsp;
+
#define TD_AUTO_CREATE_TABLE 0x1
typedef struct {
int64_t suid;
diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h
index b37dcd2b61..6a98138c6c 100644
--- a/include/libs/function/tudf.h
+++ b/include/libs/function/tudf.h
@@ -121,6 +121,8 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
+
+int32_t cleanUpUdfs();
// end API to taosd and qworker
//=============================================================================================================================
// begin API to UDF writer.
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 56e6a39ce8..4460327b88 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#include "os.h"
#include "tdatablock.h"
#include "tmsg.h"
#include "tmsgcb.h"
@@ -29,8 +30,23 @@ extern "C" {
typedef struct SStreamTask SStreamTask;
enum {
- STREAM_TASK_STATUS__RUNNING = 1,
- STREAM_TASK_STATUS__STOP,
+ TASK_STATUS__IDLE = 1,
+ TASK_STATUS__EXECUTING,
+ TASK_STATUS__CLOSING,
+};
+
+enum {
+ TASK_INPUT_STATUS__NORMAL = 1,
+ TASK_INPUT_STATUS__BLOCKED,
+ TASK_INPUT_STATUS__RECOVER,
+ TASK_INPUT_STATUS__STOP,
+ TASK_INPUT_STATUS__FAILED,
+};
+
+enum {
+ TASK_OUTPUT_STATUS__NORMAL = 1,
+ TASK_OUTPUT_STATUS__WAIT,
+ TASK_OUTPUT_STATUS__BLOCKED,
};
enum {
@@ -38,10 +54,64 @@ enum {
STREAM_CREATED_BY__SMA,
};
+enum {
+ STREAM_INPUT__DATA_SUBMIT = 1,
+ STREAM_INPUT__DATA_BLOCK,
+ STREAM_INPUT__CHECKPOINT,
+};
+
typedef struct {
- int32_t nodeId; // 0 for snode
- SEpSet epSet;
-} SStreamTaskEp;
+ int8_t type;
+
+ int32_t sourceVg;
+ int64_t sourceVer;
+
+ int32_t* dataRef;
+ SSubmitReq* data;
+} SStreamDataSubmit;
+
+typedef struct {
+ int8_t type;
+
+ int32_t sourceVg;
+ int64_t sourceVer;
+
+ SArray* blocks; // SArray
+} SStreamDataBlock;
+
+typedef struct {
+ int8_t type;
+} SStreamCheckpoint;
+
+static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) {
+ SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosMemoryCalloc(1, sizeof(SStreamDataSubmit));
+ if (pDataSubmit == NULL) return NULL;
+ pDataSubmit->data = pReq;
+ pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t));
+ if (pDataSubmit->data == NULL) goto FAIL;
+ *pDataSubmit->dataRef = 1;
+ return pDataSubmit;
+FAIL:
+ taosMemoryFree(pDataSubmit);
+ return NULL;
+}
+
+static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) {
+ //
+ atomic_add_fetch_32(pDataSubmit->dataRef, 1);
+}
+
+static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) {
+ int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1);
+ ASSERT(ref >= 0);
+ if (ref == 0) {
+ taosMemoryFree(pDataSubmit->data);
+ taosMemoryFree(pDataSubmit->dataRef);
+ }
+}
+
+int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
+void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
typedef struct {
void* inputHandle;
@@ -122,9 +192,15 @@ enum {
TASK_SINK__FETCH,
};
+enum {
+ TASK_INPUT_TYPE__SUMBIT_BLOCK = 1,
+ TASK_INPUT_TYPE__DATA_BLOCK,
+};
+
struct SStreamTask {
int64_t streamId;
int32_t taskId;
+ int8_t inputType;
int8_t status;
int8_t sourceType;
@@ -155,9 +231,13 @@ struct SStreamTask {
STaskDispatcherShuffle shuffleDispatcher;
};
- // msg buffer
- int32_t memUsed;
+ int8_t inputStatus;
+ int8_t outputStatus;
+
STaosQueue* inputQ;
+ STaosQall* inputQAll;
+ STaosQueue* outputQ;
+ STaosQall* outputQAll;
// application storage
void* ahandle;
@@ -199,10 +279,16 @@ typedef struct {
SArray* res; // SArray
} SStreamSinkReq;
-int32_t streamEnqueueData(SStreamTask* pTask, const void* input, int32_t inputType);
+int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
+int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
+int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
+int32_t streamTaskRun(SStreamTask* pTask);
+
+int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
+
#ifdef __cplusplus
}
#endif
diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h
index e2a122a0fe..8600992d68 100644
--- a/include/os/osAtomic.h
+++ b/include/os/osAtomic.h
@@ -23,92 +23,92 @@ extern "C" {
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC
- #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID
- #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID
- #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID
- #define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID
- #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID
- #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID
- #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID
- #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID
- #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID
- #define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID
- #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID
- #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID
- #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID
- #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID
+#define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID
+#define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID
+#define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID
+#define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID
+#define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID
+#define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID
+#define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID
+#define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID
+#define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID
#endif
-int8_t atomic_load_8(int8_t volatile *ptr);
+int8_t atomic_load_8(int8_t volatile *ptr);
int16_t atomic_load_16(int16_t volatile *ptr);
int32_t atomic_load_32(int32_t volatile *ptr);
int64_t atomic_load_64(int64_t volatile *ptr);
-void* atomic_load_ptr(void *ptr);
-void atomic_store_8(int8_t volatile *ptr, int8_t val);
-void atomic_store_16(int16_t volatile *ptr, int16_t val);
-void atomic_store_32(int32_t volatile *ptr, int32_t val);
-void atomic_store_64(int64_t volatile *ptr, int64_t val);
-void atomic_store_ptr(void *ptr, void *val);
-int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val);
+void *atomic_load_ptr(void *ptr);
+void atomic_store_8(int8_t volatile *ptr, int8_t val);
+void atomic_store_16(int16_t volatile *ptr, int16_t val);
+void atomic_store_32(int32_t volatile *ptr, int32_t val);
+void atomic_store_64(int64_t volatile *ptr, int64_t val);
+void atomic_store_ptr(void *ptr, void *val);
+int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
-void* atomic_exchange_ptr(void *ptr, void *val);
-int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
+void *atomic_exchange_ptr(void *ptr, void *val);
+int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval);
int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval);
-void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval);
-int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
+void *atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval);
+int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_add_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
+void *atomic_add_fetch_ptr(void *ptr, void *val);
+int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_add_ptr(void *ptr, void *val);
-int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
+void *atomic_fetch_add_ptr(void *ptr, void *val);
+int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_sub_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
+void *atomic_sub_fetch_ptr(void *ptr, void *val);
+int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_sub_ptr(void *ptr, void *val);
-int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
+void *atomic_fetch_sub_ptr(void *ptr, void *val);
+int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_and_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val);
+void *atomic_and_fetch_ptr(void *ptr, void *val);
+int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_and_ptr(void *ptr, void *val);
-int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val);
+void *atomic_fetch_and_ptr(void *ptr, void *val);
+int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_or_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val);
+void *atomic_or_fetch_ptr(void *ptr, void *val);
+int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_or_ptr(void *ptr, void *val);
-int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val);
+void *atomic_fetch_or_ptr(void *ptr, void *val);
+int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_xor_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val);
+void *atomic_xor_fetch_ptr(void *ptr, void *val);
+int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_xor_ptr(void *ptr, void *val);
+void *atomic_fetch_xor_ptr(void *ptr, void *val);
#ifdef __cplusplus
}
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 03f445b498..96fb210e26 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -354,7 +354,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C)
-#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x062D)
+#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x061D)
+#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x061E)
// query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)
diff --git a/include/util/tlog.h b/include/util/tlog.h
index dead25a4a8..be31aa8115 100644
--- a/include/util/tlog.h
+++ b/include/util/tlog.h
@@ -61,6 +61,7 @@ extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag;
extern int32_t metaDebugFlag;
extern int32_t fnDebugFlag;
+extern int32_t smaDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles);
void taosCloseLog();
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 025ad0ca0d..aaad606feb 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -299,6 +299,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1;
+ if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, 0) != 0) return -1;
return 0;
}
@@ -480,6 +481,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32;
+ smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32;
}
static int32_t taosSetClientCfg(SConfig *pCfg) {
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 021ee8455e..b954edcbfa 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -3562,39 +3562,92 @@ int tDecodeSVCreateTbBatchRsp(SDecoder *pCoder, SVCreateTbBatchRsp *pRsp) {
return 0;
}
-int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) {
- int32_t tlen = 0;
-
- tlen += taosEncodeFixedI64(buf, pReq->ver);
- tlen += tEncodeTSma(buf, &pReq->tSma);
-
- return tlen;
-}
-
-void *tDeserializeSVCreateTSmaReq(void *buf, SVCreateTSmaReq *pReq) {
- buf = taosDecodeFixedI64(buf, &(pReq->ver));
-
- if ((buf = tDecodeTSma(buf, &pReq->tSma)) == NULL) {
- tdDestroyTSma(&pReq->tSma);
+int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) {
+ if (tEncodeI8(pCoder, pSma->version) < 0) return -1;
+ if (tEncodeI8(pCoder, pSma->intervalUnit) < 0) return -1;
+ if (tEncodeI8(pCoder, pSma->slidingUnit) < 0) return -1;
+ if (tEncodeI8(pCoder, pSma->timezoneInt) < 0) return -1;
+ if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1;
+ if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1;
+ if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1;
+ if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1;
+ if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1;
+ if (tEncodeI64(pCoder, pSma->interval) < 0) return -1;
+ if (tEncodeI64(pCoder, pSma->offset) < 0) return -1;
+ if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1;
+ if (pSma->exprLen > 0) {
+ if (tEncodeCStr(pCoder, pSma->expr) < 0) return -1;
}
- return buf;
+ if (pSma->tagsFilterLen > 0) {
+ if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1;
+ }
+
+ return 0;
}
-int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) {
- int32_t tlen = 0;
+int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) {
+ if (tDecodeI8(pCoder, &pSma->version) < 0) return -1;
+ if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1;
+ if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1;
+ if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1;
+ if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1;
+ if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1;
+ if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1;
+ if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1;
+ if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1;
+ if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1;
+ if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1;
+ if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1;
+ if (pSma->exprLen > 0) {
+ if (tDecodeCStr(pCoder, &pSma->expr) < 0) return -1;
+ } else {
+ pSma->expr = NULL;
+ }
+ if (pSma->tagsFilterLen > 0) {
+ if (tDecodeCStr(pCoder, &pSma->tagsFilter) < 0) return -1;
+ } else {
+ pSma->tagsFilter = NULL;
+ }
- tlen += taosEncodeFixedI64(buf, pReq->ver);
- tlen += taosEncodeFixedI64(buf, pReq->indexUid);
- tlen += taosEncodeString(buf, pReq->indexName);
-
- return tlen;
+ return 0;
}
-void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
- buf = taosDecodeFixedI64(buf, &(pReq->ver));
- buf = taosDecodeFixedI64(buf, &(pReq->indexUid));
- buf = taosDecodeStringTo(buf, pReq->indexName);
- return buf;
+int32_t tEncodeSVCreateTSmaReq(SEncoder *pCoder, const SVCreateTSmaReq *pReq) {
+ if (tStartEncode(pCoder) < 0) return -1;
+
+ tEncodeTSma(pCoder, pReq);
+
+ tEndEncode(pCoder);
+ return 0;
+}
+
+int32_t tDecodeSVCreateTSmaReq(SDecoder *pCoder, SVCreateTSmaReq *pReq) {
+ if (tStartDecode(pCoder) < 0) return -1;
+
+ tDecodeTSma(pCoder, pReq);
+
+ tEndDecode(pCoder);
+ return 0;
+}
+
+int32_t tEncodeSVDropTSmaReq(SEncoder *pCoder, const SVDropTSmaReq *pReq) {
+ if (tStartEncode(pCoder) < 0) return -1;
+
+ if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1;
+ if (tEncodeCStr(pCoder, pReq->indexName) < 0) return -1;
+
+ tEndEncode(pCoder);
+ return 0;
+}
+
+int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
+ if (tStartDecode(pCoder) < 0) return -1;
+
+ if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
+ if (tDecodeCStrTo(pCoder, pReq->indexName) < 0) return -1;
+
+ tEndDecode(pCoder);
+ return 0;
}
int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) {
diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index 8225eca659..35ba25acd5 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -17,6 +17,147 @@
#include "mndDef.h"
#include "mndConsumer.h"
+int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
+ int32_t sz = 0;
+ /*int32_t outputNameSz = 0;*/
+ if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
+ if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
+ if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1;
+ if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1;
+ if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1;
+ if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
+ if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
+ if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
+ if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
+ if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
+ if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
+ if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
+ if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
+ if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1;
+ if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1;
+ if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
+ if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
+ if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
+ /*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
+ if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
+ // TODO encode tasks
+ if (pObj->tasks) {
+ sz = taosArrayGetSize(pObj->tasks);
+ }
+ if (tEncodeI32(pEncoder, sz) < 0) return -1;
+
+ for (int32_t i = 0; i < sz; i++) {
+ SArray *pArray = taosArrayGetP(pObj->tasks, i);
+ int32_t innerSz = taosArrayGetSize(pArray);
+ if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
+ for (int32_t j = 0; j < innerSz; j++) {
+ SStreamTask *pTask = taosArrayGetP(pArray, j);
+ if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1;
+ }
+ }
+
+ if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
+
+#if 0
+ if (pObj->ColAlias != NULL) {
+ outputNameSz = taosArrayGetSize(pObj->ColAlias);
+ }
+ if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1;
+ for (int32_t i = 0; i < outputNameSz; i++) {
+ char *name = taosArrayGetP(pObj->ColAlias, i);
+ if (tEncodeCStr(pEncoder, name) < 0) return -1;
+ }
+#endif
+ return pEncoder->pos;
+}
+
+int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
+ if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
+ if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
+ if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1;
+ if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1;
+ if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1;
+ if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
+ if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
+ if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
+ if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
+ if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
+ if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
+ if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
+ if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
+ if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1;
+ if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1;
+ if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
+ if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
+ if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
+ /*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
+ if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
+ pObj->tasks = NULL;
+ int32_t sz;
+ if (tDecodeI32(pDecoder, &sz) < 0) return -1;
+ if (sz != 0) {
+ pObj->tasks = taosArrayInit(sz, sizeof(void *));
+ for (int32_t i = 0; i < sz; i++) {
+ int32_t innerSz;
+ if (tDecodeI32(pDecoder, &innerSz) < 0) return -1;
+ SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
+ for (int32_t j = 0; j < innerSz; j++) {
+ SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
+ if (pTask == NULL) return -1;
+ if (tDecodeSStreamTask(pDecoder, pTask) < 0) return -1;
+ taosArrayPush(pArray, &pTask);
+ }
+ taosArrayPush(pObj->tasks, &pArray);
+ }
+ }
+
+ if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
+#if 0
+ int32_t outputNameSz;
+ if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1;
+ if (outputNameSz != 0) {
+ pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *));
+ if (pObj->ColAlias == NULL) {
+ return -1;
+ }
+ }
+ for (int32_t i = 0; i < outputNameSz; i++) {
+ char *name;
+ if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1;
+ taosArrayPush(pObj->ColAlias, &name);
+ }
+#endif
+ return 0;
+}
+
+SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
+ SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
+ if (pVgEpNew == NULL) return NULL;
+ pVgEpNew->vgId = pVgEp->vgId;
+ pVgEpNew->qmsg = strdup(pVgEp->qmsg);
+ pVgEpNew->epSet = pVgEp->epSet;
+ return pVgEpNew;
+}
+
+void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
+ if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg);
+}
+
+int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
+ int32_t tlen = 0;
+ tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
+ tlen += taosEncodeString(buf, pVgEp->qmsg);
+ tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
+ return tlen;
+}
+
+void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
+ buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
+ buf = taosDecodeString(buf, &pVgEp->qmsg);
+ buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
+ return (void *)buf;
+}
+
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) {
SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
if (pConsumer == NULL) {
@@ -187,34 +328,6 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
return (void *)buf;
}
-SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
- SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
- if (pVgEpNew == NULL) return NULL;
- pVgEpNew->vgId = pVgEp->vgId;
- pVgEpNew->qmsg = strdup(pVgEp->qmsg);
- pVgEpNew->epSet = pVgEp->epSet;
- return pVgEpNew;
-}
-
-void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
- if (pVgEp->qmsg) taosMemoryFree(pVgEp->qmsg);
-}
-
-int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
- int32_t tlen = 0;
- tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
- tlen += taosEncodeString(buf, pVgEp->qmsg);
- tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
- return tlen;
-}
-
-void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
- buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
- buf = taosDecodeString(buf, &pVgEp->qmsg);
- buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
- return (void *)buf;
-}
-
SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
if (pConsumerEpNew == NULL) return NULL;
@@ -413,119 +526,6 @@ void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
return (void *)buf;
}
-int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
- int32_t sz = 0;
- /*int32_t outputNameSz = 0;*/
- if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
- if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
- if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1;
- if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1;
- if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1;
- if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
- if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
- if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
- if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
- if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
- if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
- if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
- if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
- if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1;
- if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1;
- if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
- if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
- if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
- /*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
- if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
- // TODO encode tasks
- if (pObj->tasks) {
- sz = taosArrayGetSize(pObj->tasks);
- }
- if (tEncodeI32(pEncoder, sz) < 0) return -1;
-
- for (int32_t i = 0; i < sz; i++) {
- SArray *pArray = taosArrayGetP(pObj->tasks, i);
- int32_t innerSz = taosArrayGetSize(pArray);
- if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
- for (int32_t j = 0; j < innerSz; j++) {
- SStreamTask *pTask = taosArrayGetP(pArray, j);
- if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1;
- }
- }
-
- if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
-
-#if 0
- if (pObj->ColAlias != NULL) {
- outputNameSz = taosArrayGetSize(pObj->ColAlias);
- }
- if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1;
- for (int32_t i = 0; i < outputNameSz; i++) {
- char *name = taosArrayGetP(pObj->ColAlias, i);
- if (tEncodeCStr(pEncoder, name) < 0) return -1;
- }
-#endif
- return pEncoder->pos;
-}
-
-int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
- if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
- if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
- if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1;
- if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1;
- if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1;
- if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
- if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
- if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
- if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
- if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
- if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
- if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
- if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
- if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1;
- if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1;
- if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
- if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
- if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
- /*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
- if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
- pObj->tasks = NULL;
- int32_t sz;
- if (tDecodeI32(pDecoder, &sz) < 0) return -1;
- if (sz != 0) {
- pObj->tasks = taosArrayInit(sz, sizeof(void *));
- for (int32_t i = 0; i < sz; i++) {
- int32_t innerSz;
- if (tDecodeI32(pDecoder, &innerSz) < 0) return -1;
- SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
- for (int32_t j = 0; j < innerSz; j++) {
- SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
- if (pTask == NULL) return -1;
- if (tDecodeSStreamTask(pDecoder, pTask) < 0) return -1;
- taosArrayPush(pArray, &pTask);
- }
- taosArrayPush(pObj->tasks, &pArray);
- }
- }
-
- if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
-#if 0
- int32_t outputNameSz;
- if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1;
- if (outputNameSz != 0) {
- pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *));
- if (pObj->ColAlias == NULL) {
- return -1;
- }
- }
- for (int32_t i = 0; i < outputNameSz; i++) {
- char *name;
- if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1;
- taosArrayPush(pObj->ColAlias, &name);
- }
-#endif
- return 0;
-}
-
int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pOffset->key);
diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index 824f031004..22a5f37334 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -194,6 +194,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p
// source
pTask->sourceType = TASK_SOURCE__MERGE;
+ pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// exec
pTask->execType = TASK_EXEC__NONE;
@@ -235,6 +236,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr
pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
// source
pTask->sourceType = TASK_SOURCE__MERGE;
+ pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// exec
pTask->execType = TASK_EXEC__NONE;
@@ -309,6 +311,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
// source part
pTask->sourceType = TASK_SOURCE__SCAN;
+ pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
// sink part
if (level == 0) {
@@ -372,6 +375,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
// source part, currently only support multi source
pTask->sourceType = TASK_SOURCE__PIPE;
+ pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// sink part
pTask->sinkType = TASK_SINK__NONE;
@@ -459,6 +463,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
// source part
pTask->sourceType = TASK_SOURCE__MERGE;
+ pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
// sink part
pTask->sinkType = TASK_SINK__NONE;
diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c
index 02a8eaa832..869bf0c0a9 100644
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@@ -242,26 +242,35 @@ SDbObj *mndAcquireDbBySma(SMnode *pMnode, const char *smaName) {
}
static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) {
- SName name = {0};
+ SEncoder encoder = {0};
+ int32_t contLen = 0;
+ SName name = {0};
tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
SVCreateTSmaReq req = {0};
- req.tSma.version = 0;
- req.tSma.intervalUnit = pSma->intervalUnit;
- req.tSma.slidingUnit = pSma->slidingUnit;
- req.tSma.timezoneInt = pSma->timezone;
- tstrncpy(req.tSma.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
- req.tSma.exprLen = pSma->exprLen;
- req.tSma.tagsFilterLen = pSma->tagsFilterLen;
- req.tSma.indexUid = pSma->uid;
- req.tSma.tableUid = pSma->stbUid;
- req.tSma.interval = pSma->interval;
- req.tSma.offset = pSma->offset;
- req.tSma.sliding = pSma->sliding;
- req.tSma.expr = pSma->expr;
- req.tSma.tagsFilter = pSma->tagsFilter;
+ req.version = 0;
+ req.intervalUnit = pSma->intervalUnit;
+ req.slidingUnit = pSma->slidingUnit;
+ req.timezoneInt = pSma->timezone;
+ tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
+ req.exprLen = pSma->exprLen;
+ req.tagsFilterLen = pSma->tagsFilterLen;
+ req.indexUid = pSma->uid;
+ req.tableUid = pSma->stbUid;
+ req.interval = pSma->interval;
+ req.offset = pSma->offset;
+ req.sliding = pSma->sliding;
+ req.expr = pSma->expr;
+ req.tagsFilter = pSma->tagsFilter;
+
+ // get length
+ int32_t ret = 0;
+ tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret);
+ if (ret < 0) {
+ return NULL;
+ }
+ contLen += sizeof(SMsgHead);
- int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead);
SMsgHead *pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -272,22 +281,38 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm
pHead->vgId = htonl(pVgroup->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
- tSerializeSVCreateTSmaReq(&pBuf, &req);
+ tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
+ if (tEncodeSVCreateTSmaReq(&encoder, &req) < 0) {
+ taosMemoryFreeClear(pHead);
+ tEncoderClear(&encoder);
+ return NULL;
+ }
+
+ tEncoderClear(&encoder);
*pContLen = contLen;
return pHead;
}
static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, int32_t *pContLen) {
+ SEncoder encoder = {0};
+ int32_t contLen;
SName name = {0};
tNameFromString(&name, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
SVDropTSmaReq req = {0};
- req.ver = 0;
req.indexUid = pSma->uid;
tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
- int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead);
+ // get length
+ int32_t ret = 0;
+ tEncodeSize(tEncodeSVDropTSmaReq, &req, contLen, ret);
+ if (ret < 0) {
+ return NULL;
+ }
+
+ contLen += sizeof(SMsgHead);
+
SMsgHead *pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -298,7 +323,14 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma,
pHead->vgId = htonl(pVgroup->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
- tDeserializeSVDropTSmaReq(&pBuf, &req);
+ tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
+
+ if (tEncodeSVDropTSmaReq(&encoder, &req) < 0) {
+ taosMemoryFreeClear(pHead);
+ tEncoderClear(&encoder);
+ return NULL;
+ }
+ tEncoderClear(&encoder);
*pContLen = contLen;
return pHead;
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 12e89277f4..e02d629c12 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -425,6 +425,10 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
if (tEncodeSVCreateStbReq(&encoder, &req) < 0) {
+ taosMemoryFreeClear(pHead);
+ taosMemoryFreeClear(req.pRSmaParam.qmsg1);
+ taosMemoryFreeClear(req.pRSmaParam.qmsg2);
+ tEncoderClear(&encoder);
return NULL;
}
tEncoderClear(&encoder);
diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp
index 974c86b423..061ede345d 100644
--- a/source/dnode/mnode/impl/test/trans/trans2.cpp
+++ b/source/dnode/mnode/impl/test/trans/trans2.cpp
@@ -510,4 +510,4 @@ TEST_F(MndTestTrans2, 04_Conflict) {
ASSERT_EQ(pUser, nullptr);
mndReleaseUser(pMnode, pUser);
}
-}
\ No newline at end of file
+}
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 58e00ee34a..a8e3860ed1 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -18,12 +18,21 @@ target_sources(
"src/meta/metaOpen.c"
"src/meta/metaIdx.c"
"src/meta/metaTable.c"
+ "src/meta/metaSma.c"
"src/meta/metaQuery.c"
"src/meta/metaCommit.c"
"src/meta/metaEntry.c"
+ # sma
+ "src/sma/sma.c"
+ "src/sma/smaTDBImpl.c"
+ "src/sma/smaEnv.c"
+ "src/sma/smaOpen.c"
+ "src/sma/smaRollup.c"
+ "src/sma/smaTimeRange.c"
+
# tsdb
- "src/tsdb/tsdbTDBImpl.c"
+ # "src/tsdb/tsdbTDBImpl.c"
"src/tsdb/tsdbCommit.c"
"src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbFile.c"
@@ -33,7 +42,7 @@ target_sources(
"src/tsdb/tsdbMemTable2.c"
"src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c"
- "src/tsdb/tsdbSma.c"
+ # "src/tsdb/tsdbSma.c"
"src/tsdb/tsdbWrite.c"
# tq
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 130cebf0b1..a2be393eb2 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -191,6 +191,9 @@ struct SMetaEntry {
int32_t ttlDays;
SSchemaWrapper schema;
} ntbEntry;
+ struct {
+ STSma *tsma;
+ } smaEntry;
};
};
diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index 60b3a889ef..c5ca806829 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -73,6 +73,7 @@ struct SMeta {
TDB* pCtbIdx;
TDB* pTagIdx;
TDB* pTtlIdx;
+ TDB* pSmaIdx;
SMetaIdx* pIdx;
};
@@ -108,6 +109,11 @@ typedef struct {
tb_uid_t uid;
} STtlIdxKey;
+typedef struct {
+ tb_uid_t uid;
+ int64_t smaUid;
+} SSmaIdxKey;
+
#if 1
SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
@@ -118,7 +124,7 @@ int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur);
// SMetaDB
int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta);
-// int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
+int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h
new file mode 100644
index 0000000000..76a30f58dd
--- /dev/null
+++ b/source/dnode/vnode/src/inc/sma.h
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef _TD_VNODE_SMA_H_
+#define _TD_VNODE_SMA_H_
+
+#include "vnodeInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// smaDebug ================
+// clang-format off
+#define smaFatal(...) do { if (smaDebugFlag & DEBUG_FATAL) { taosPrintLog("SMA FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
+#define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
+#define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
+#define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
+#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+// clang-format on
+
+typedef struct SSmaEnv SSmaEnv;
+typedef struct SSmaStat SSmaStat;
+typedef struct SSmaStatItem SSmaStatItem;
+typedef struct SSmaKey SSmaKey;
+typedef struct SRSmaInfo SRSmaInfo;
+
+#define SMA_IVLD_FID INT_MIN
+
+struct SSmaEnv {
+ TdThreadRwlock lock;
+ int8_t type;
+ TXN txn;
+ void *pPool; // SPoolMem
+ SDiskID did;
+ TENV *dbEnv; // TODO: If it's better to put it in smaIndex level?
+ char *path; // relative path
+ SSmaStat *pStat;
+};
+
+#define SMA_ENV_LOCK(env) ((env)->lock)
+#define SMA_ENV_TYPE(env) ((env)->type)
+#define SMA_ENV_DID(env) ((env)->did)
+#define SMA_ENV_ENV(env) ((env)->dbEnv)
+#define SMA_ENV_PATH(env) ((env)->path)
+#define SMA_ENV_STAT(env) ((env)->pStat)
+#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
+
+struct SSmaStatItem {
+ /**
+ * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service.
+ * - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from
+ * Streaming Module or TSDB local persistence.
+ * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
+ * without information about its previous state.
+ * - TSDB_SMA_STAT_DROPPED: 1)sma dropped
+ * N.B. only applicable to tsma
+ */
+ int8_t state; // ETsdbSmaStat
+ SHashObj *expiredWindows; // key: skey of time window, value: N/A
+ STSma *pTSma; // cache schema
+};
+
+struct SSmaStat {
+ union {
+ SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem for tsma
+ SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo;
+ };
+ T_REF_DECLARE()
+};
+#define SMA_STAT_ITEMS(s) ((s)->smaStatItems)
+#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash)
+
+struct SSmaKey {
+ TSKEY skey;
+ int64_t groupId;
+};
+
+typedef struct SDBFile SDBFile;
+
+struct SDBFile {
+ int32_t fid;
+ TDB *pDB;
+ char *path;
+};
+
+int32_t tdSmaBeginCommit(SSmaEnv *pEnv);
+int32_t tdSmaEndCommit(SSmaEnv *pEnv);
+
+int32_t smaOpenDBEnv(TENV **ppEnv, const char *path);
+int32_t smaCloseDBEnv(TENV *pEnv);
+int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF);
+int32_t smaCloseDBF(SDBFile *pDBF);
+int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
+void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
+
+void tdDestroySmaEnv(SSmaEnv *pSmaEnv);
+void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
+#if 0
+int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result);
+int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin);
+#endif
+
+static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
+ int32_t len = 0;
+ len += taosEncodeFixedI64(pData, tsKey);
+ len += taosEncodeFixedI64(pData, groupId);
+ return len;
+}
+
+int32_t tdInitSma(SSma *pSma);
+int32_t tdDropTSma(SSma *pSma, char *pMsg);
+int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid);
+int32_t tdInsertRSmaData(SSma *pSma, char *msg);
+
+int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
+int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
+int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType);
+
+int32_t tdLockSma(SSma *pSma);
+int32_t tdUnLockSma(SSma *pSma);
+
+int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
+
+static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); }
+static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); }
+
+static FORCE_INLINE int32_t tdRLockSmaEnv(SSmaEnv *pEnv) {
+ int code = taosThreadRwlockRdlock(&(pEnv->lock));
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+ return 0;
+}
+
+static FORCE_INLINE int32_t tdWLockSmaEnv(SSmaEnv *pEnv) {
+ int code = taosThreadRwlockWrlock(&(pEnv->lock));
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+ return 0;
+}
+
+static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) {
+ int code = taosThreadRwlockUnlock(&(pEnv->lock));
+ if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+ return 0;
+}
+
+static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) {
+ if (pStatItem) {
+ return atomic_load_8(&pStatItem->state);
+ }
+ return TSDB_SMA_STAT_UNKNOWN;
+}
+
+static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) {
+ if (!pStatItem) {
+ return false;
+ }
+
+ if (state) {
+ *state = atomic_load_8(&pStatItem->state);
+ return *state == TSDB_SMA_STAT_OK;
+ }
+ return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK;
+}
+
+static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) {
+ return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true;
+}
+
+static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) {
+ return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true;
+}
+
+static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) {
+ if (pStatItem) {
+ atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK);
+ }
+}
+
+static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) {
+ if (pStatItem) {
+ atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED);
+ }
+}
+
+static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) {
+ if (pStatItem) {
+ atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED);
+ }
+}
+
+static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType);
+void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem);
+static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
+static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did);
+static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv);
+
+void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_SMA_H_*/
\ No newline at end of file
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 102c40337d..93a25da0a8 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -60,31 +60,22 @@ typedef struct {
TSKEY minKey;
} SRtn;
-struct SSmaEnvs {
- int16_t nTSma;
- int16_t nRSma;
- SSmaEnv *pTSmaEnv;
- SSmaEnv *pRSmaEnv;
-};
-
+#define TSDB_DATA_DIR_LEN 6
struct STsdb {
char *path;
SVnode *pVnode;
TdThreadMutex mutex;
+ char dir[TSDB_DATA_DIR_LEN];
bool repoLocked;
- int8_t level; // retention level
STsdbKeepCfg keepCfg;
STsdbMemTable *mem;
STsdbMemTable *imem;
SRtn rtn;
STsdbFS *fs;
- SSmaEnvs smaEnvs;
};
#if 1 // ======================================
-typedef struct SSmaStat SSmaStat;
-
struct STable {
uint64_t tid;
uint64_t uid;
@@ -95,10 +86,6 @@ struct STable {
#define TABLE_UID(t) (t)->uid
int tsdbPrepareCommit(STsdb *pTsdb);
-int32_t tsdbInitSma(STsdb *pTsdb);
-int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
-int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
-int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
typedef enum {
TSDB_FILE_HEAD = 0, // .head
TSDB_FILE_DATA, // .data
@@ -107,8 +94,6 @@ typedef enum {
TSDB_FILE_SMAL, // .smal(Block-wise SMA)
TSDB_FILE_MAX, //
TSDB_FILE_META, // meta
- TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA
- TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
} E_TSDB_FILE_T;
typedef struct {
@@ -186,15 +171,10 @@ struct STsdbFS {
#define REPO_ID(r) TD_VID((r)->pVnode)
#define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg)
#define REPO_KEEP_CFG(r) (&(r)->keepCfg)
-#define REPO_LEVEL(r) ((r)->level)
#define REPO_FS(r) ((r)->fs)
#define REPO_META(r) ((r)->pVnode->pMeta)
#define REPO_TFS(r) ((r)->pVnode->pTfs)
#define IS_REPO_LOCKED(r) ((r)->repoLocked)
-#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma)
-#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma)
-#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv)
-#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv)
int tsdbLockRepo(STsdb *pTsdb);
int tsdbUnlockRepo(STsdb *pTsdb);
@@ -794,25 +774,6 @@ typedef struct {
} SFSHeader;
// ================== TSDB File System Meta
-
-/**
- * @brief Directory structure of .tsma data files.
- *
- * /vnode2/tsdb $ tree tsma/
- * tsma/
- * ├── v2f100.index_name_1
- * ├── v2f101.index_name_1
- * ├── v2f102.index_name_1
- * ├── v2f1900.index_name_3
- * ├── v2f1901.index_name_3
- * ├── v2f1902.index_name_3
- * ├── v2f200.index_name_2
- * ├── v2f201.index_name_2
- * └── v2f202.index_name_2
- *
- * 0 directories, 9 files
- */
-
#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
#define FS_IN_TXN(pfs) (pfs)->intxn
@@ -874,43 +835,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
return 0;
}
-typedef struct SSmaKey SSmaKey;
-
-struct SSmaKey {
- TSKEY skey;
- int64_t groupId;
-};
-
-typedef struct SDBFile SDBFile;
-
-struct SDBFile {
- int32_t fid;
- TDB *pDB;
- char *path;
-};
-
-int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path);
-int32_t tsdbCloseDBEnv(TENV *pEnv);
-int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF);
-int32_t tsdbCloseDBF(SDBFile *pDBF);
-int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
-void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
-
-void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
-void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
-#if 0
-int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
-int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
-#endif
-
-// internal func
-static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
- int32_t len = 0;
- len += taosEncodeFixedI64(pData, tsKey);
- len += taosEncodeFixedI64(pData, groupId);
- return len;
-}
-
#endif
#ifdef __cplusplus
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 9a36fc6eae..c9d1a0e06e 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -47,13 +47,15 @@
extern "C" {
#endif
-typedef struct SVnodeInfo SVnodeInfo;
-typedef struct SMeta SMeta;
-typedef struct STsdb STsdb;
-typedef struct STQ STQ;
-typedef struct SVState SVState;
-typedef struct SVBufPool SVBufPool;
-typedef struct SQWorker SQHandle;
+typedef struct SVnodeInfo SVnodeInfo;
+typedef struct SMeta SMeta;
+typedef struct SSma SSma;
+typedef struct STsdb STsdb;
+typedef struct STQ STQ;
+typedef struct SVState SVState;
+typedef struct SVBufPool SVBufPool;
+typedef struct SQWorker SQHandle;
+typedef struct STsdbKeepCfg STsdbKeepCfg;
#define VNODE_META_DIR "meta"
#define VNODE_TSDB_DIR "tsdb"
@@ -90,17 +92,14 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup);
void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
-int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
+int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
// tsdb
-int tsdbOpen(SVnode* pVnode, int8_t type);
-int tsdbClose(STsdb* pTsdb);
+int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg *pKeepCfg);
+int tsdbClose(STsdb** pTsdb);
int tsdbBegin(STsdb* pTsdb);
int tsdbCommit(STsdb* pTsdb);
-int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
-int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
-int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);
tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
@@ -121,13 +120,31 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
// sma
+int32_t smaOpen(SVnode* pVnode);
+int32_t smaClose(SSma* pSma);
+int32_t tdUpdateExpireWindow(SSma* pSma, SSubmitReq* pMsg, int64_t version);
+int32_t tdProcessTSmaCreate(SSma* pSma, char* pMsg);
+int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
+
+int32_t tdProcessRSmaCreate(SSma* pSma, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
+int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);
+int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
+int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
+void tdUidStoreDestory(STbUidStore* pStore);
+void* tdUidStoreFree(STbUidStore* pStore);
+
+#if 0
+int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
+int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
+int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
void tsdbUidStoreDestory(STbUidStore* pStore);
void* tsdbUidStoreFree(STbUidStore* pStore);
int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType);
+#endif
typedef struct {
int8_t streamType; // sma or other
@@ -164,13 +181,13 @@ typedef enum {
TSDB_TYPE_RSMA_L2 = 4, // RSMA Level 2
} ETsdbType;
-typedef struct {
+struct STsdbKeepCfg{
int8_t precision; // precision always be used with below keep cfgs
int32_t days;
int32_t keep0;
int32_t keep1;
int32_t keep2;
-} STsdbKeepCfg;
+};
struct SVnode {
char* path;
@@ -183,9 +200,8 @@ struct SVnode {
SVBufPool* onCommit;
SVBufPool* onRecycle;
SMeta* pMeta;
+ SSma* pSma;
STsdb* pTsdb;
- STsdb* pRSma1;
- STsdb* pRSma2;
SWal* pWal;
STQ* pTq;
SSink* pSink;
@@ -194,10 +210,12 @@ struct SVnode {
SQHandle* pQuery;
};
+#define TD_VID(PVNODE) (PVNODE)->config.vgId
+
#define VND_TSDB(vnd) ((vnd)->pTsdb)
#define VND_RSMA0(vnd) ((vnd)->pTsdb)
-#define VND_RSMA1(vnd) ((vnd)->pRSma1)
-#define VND_RSMA2(vnd) ((vnd)->pRSma2)
+#define VND_RSMA1(vnd) ((vnd)->pSma->pRSmaTsdb1)
+#define VND_RSMA2(vnd) ((vnd)->pSma->pRSmaTsdb2)
#define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions)
struct STbUidStore {
@@ -207,7 +225,29 @@ struct STbUidStore {
SHashObj* uidHash;
};
-#define TD_VID(PVNODE) (PVNODE)->config.vgId
+struct SSma {
+ int16_t nTSma;
+ bool locked;
+ TdThreadMutex mutex;
+ SVnode* pVnode;
+ STsdb* pRSmaTsdb1;
+ STsdb* pRSmaTsdb2;
+ void* pTSmaEnv;
+ void* pRSmaEnv;
+};
+
+#define SMA_CFG(s) (&(s)->pVnode->config)
+#define SMA_TSDB_CFG(s) (&(s)->pVnode->config.tsdbCfg)
+#define SMA_LOCKED(s) ((s)->locked)
+#define SMA_META(s) ((s)->pVnode->pMeta)
+#define SMA_VID(s) TD_VID((s)->pVnode)
+#define SMA_TFS(s) ((s)->pVnode->pTfs)
+#define SMA_TSMA_NUM(s) ((s)->nTSma)
+#define SMA_TSMA_ENV(s) ((s)->pTSmaEnv)
+#define SMA_RSMA_ENV(s) ((s)->pRSmaEnv)
+#define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb)
+#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1)
+#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2)
static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) {
SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]);
diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c
index 581b876e84..dd36906b19 100644
--- a/source/dnode/vnode/src/meta/metaEntry.c
+++ b/source/dnode/vnode/src/meta/metaEntry.c
@@ -35,6 +35,8 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
+ } else if (pME->type == TSDB_TSMA_TABLE) {
+ if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
} else {
ASSERT(0);
}
@@ -64,7 +66,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
- } else {
+ } else if (pME->type == TSDB_TSMA_TABLE) {
+ if (tDecodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
+ } else {
ASSERT(0);
}
diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c
index 853b2ecefb..3f52071315 100644
--- a/source/dnode/vnode/src/meta/metaIdx.c
+++ b/source/dnode/vnode/src/meta/metaIdx.c
@@ -112,35 +112,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
#endif
// TODO
return 0;
-}
-
-int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
- // TODO: Validate the cfg
- // The table uid should exists and be super table or common table.
- // Check other cfg value
-
- // TODO: add atomicity
-
-#ifdef META_REFACT
-#else
- if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) {
- // TODO: handle error
- return -1;
- }
-#endif
- return TSDB_CODE_SUCCESS;
-}
-
-int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
- // TODO: Validate the cfg
- // TODO: add atomicity
-
-#ifdef META_REFACT
-#else
- if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
- // TODO: handle error
- return -1;
- }
-#endif
- return TSDB_CODE_SUCCESS;
}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c
index 07422e3193..3ce146904c 100644
--- a/source/dnode/vnode/src/meta/metaOpen.c
+++ b/source/dnode/vnode/src/meta/metaOpen.c
@@ -21,6 +21,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); }
static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); }
@@ -104,6 +105,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
goto _err;
}
+ // open pSmaIdx
+ ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx);
+ if (ret < 0) {
+ metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
+ }
+
// open index
if (metaOpenIdx(pMeta) < 0) {
metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
@@ -117,11 +125,12 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
_err:
if (pMeta->pIdx) metaCloseIdx(pMeta);
+ if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
- if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
+ if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
@@ -133,11 +142,12 @@ _err:
int metaClose(SMeta *pMeta) {
if (pMeta) {
if (pMeta->pIdx) metaCloseIdx(pMeta);
+ if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
- if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
+ if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
@@ -295,3 +305,22 @@ static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
return 0;
}
+
+static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+ SSmaIdxKey *pSmaIdxKey1 = (SSmaIdxKey *)pKey1;
+ SSmaIdxKey *pSmaIdxKey2 = (SSmaIdxKey *)pKey2;
+
+ if (pSmaIdxKey1->uid > pSmaIdxKey2->uid) {
+ return 1;
+ } else if (pSmaIdxKey1->uid < pSmaIdxKey2->uid) {
+ return -1;
+ }
+
+ if (pSmaIdxKey1->smaUid > pSmaIdxKey2->smaUid) {
+ return 1;
+ } else if (pSmaIdxKey1->smaUid < pSmaIdxKey2->smaUid) {
+ return -1;
+ }
+
+ return 0;
+}
diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c
new file mode 100644
index 0000000000..8ce7ea5895
--- /dev/null
+++ b/source/dnode/vnode/src/meta/metaSma.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "meta.h"
+
+static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME);
+static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME);
+
+int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
+ // TODO: Validate the cfg
+ // The table uid should exists and be super table or normal table.
+ // Check other cfg value
+
+ SMetaEntry me = {0};
+ int kLen = 0;
+ int vLen = 0;
+ const void *pKey = NULL;
+ const void *pVal = NULL;
+ void *pBuf = NULL;
+ int32_t szBuf = 0;
+ void *p = NULL;
+ SMetaReader mr = {0};
+
+ // validate req
+ metaReaderInit(&mr, pMeta, 0);
+ if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) {
+// TODO: just for pass case
+#if 1
+ terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST;
+ metaReaderClear(&mr);
+ return -1;
+#else
+ metaReaderClear(&mr);
+ return 0;
+#endif
+ }
+ metaReaderClear(&mr);
+
+ // set structs
+ me.version = version;
+ me.type = TSDB_TSMA_TABLE;
+ me.uid = pCfg->indexUid;
+ me.name = pCfg->indexName;
+ me.smaEntry.tsma = pCfg;
+
+ if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err;
+
+ metaDebug("vgId:%d tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid);
+
+ return 0;
+
+_err:
+ metaError("vgId:%d failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName,
+ pCfg->indexUid, tstrerror(terrno));
+ return -1;
+}
+
+int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
+ // TODO: Validate the cfg
+ // TODO: add atomicity
+
+#ifdef META_REFACT
+#else
+ if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
+ // TODO: handle error
+ return -1;
+ }
+#endif
+ return TSDB_CODE_SUCCESS;
+}
+
+// static int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
+// int32_t ret = 0;
+// void *pBuf = NULL, *qBuf = NULL;
+// void *key = {0}, *val = {0};
+
+// // save sma info
+// int32_t len = tEncodeTSma(NULL, pSmaCfg);
+// pBuf = taosMemoryCalloc(1, len);
+// if (pBuf == NULL) {
+// terrno = TSDB_CODE_OUT_OF_MEMORY;
+// return -1;
+// }
+
+// key = (void *)&pSmaCfg->indexUid;
+// qBuf = pBuf;
+// tEncodeTSma(&qBuf, pSmaCfg);
+// val = pBuf;
+
+// int32_t kLen = sizeof(pSmaCfg->indexUid);
+// int32_t vLen = POINTER_DISTANCE(qBuf, pBuf);
+
+// ret = tdbDbInsert(pMeta->pTbDb, key, kLen, val, vLen, &pMeta->txn);
+// if (ret < 0) {
+// taosMemoryFreeClear(pBuf);
+// return -1;
+// }
+
+// // add sma idx
+// SSmaIdxKey smaIdxKey;
+// smaIdxKey.uid = pSmaCfg->tableUid;
+// smaIdxKey.smaUid = pSmaCfg->indexUid;
+// key = &smaIdxKey;
+// kLen = sizeof(smaIdxKey);
+// val = NULL;
+// vLen = 0;
+
+// ret = tdbDbInsert(pMeta->pSmaIdx, key, kLen, val, vLen, &pMeta->txn);
+// if (ret < 0) {
+// taosMemoryFreeClear(pBuf);
+// return -1;
+// }
+
+// // release
+// taosMemoryFreeClear(pBuf);
+
+// return 0;
+// }
+
+
+static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
+ STbDbKey tbDbKey;
+ void *pKey = NULL;
+ void *pVal = NULL;
+ int kLen = 0;
+ int vLen = 0;
+ SEncoder coder = {0};
+
+ // set key and value
+ tbDbKey.version = pME->version;
+ tbDbKey.uid = pME->uid;
+
+ pKey = &tbDbKey;
+ kLen = sizeof(tbDbKey);
+
+ int32_t ret = 0;
+ tEncodeSize(metaEncodeEntry, pME, vLen, ret);
+ if (ret < 0) {
+ goto _err;
+ }
+
+ pVal = taosMemoryMalloc(vLen);
+ if (pVal == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+
+ tEncoderInit(&coder, pVal, vLen);
+
+ if (metaEncodeEntry(&coder, pME) < 0) {
+ goto _err;
+ }
+
+ tEncoderClear(&coder);
+
+ // write to table.db
+ if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
+ goto _err;
+ }
+
+ taosMemoryFree(pVal);
+ return 0;
+
+_err:
+ taosMemoryFree(pVal);
+ return -1;
+}
+
+static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
+ return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
+}
+
+static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) {
+ SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid};
+
+ return tdbDbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn);
+}
+
+static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) {
+ metaWLock(pMeta);
+
+ // save to table.db
+ if (metaSaveSmaToDB(pMeta, pME) < 0) goto _err;
+
+ // // update uid.idx
+ if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err;
+
+ if (metaUpdateSmaIdx(pMeta, pME) < 0) goto _err;
+
+ metaULock(pMeta);
+ return 0;
+
+_err:
+ metaULock(pMeta);
+ return -1;
+}
diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/sma/sma.c
similarity index 50%
rename from source/dnode/vnode/src/inc/tsdbSma.h
rename to source/dnode/vnode/src/sma/sma.c
index 5215812ac5..2c54e10087 100644
--- a/source/dnode/vnode/src/inc/tsdbSma.h
+++ b/source/dnode/vnode/src/sma/sma.c
@@ -13,35 +13,18 @@
* along with this program. If not, see .
*/
-#ifndef _TD_VNODE_TSDB_SMA_H_
-#define _TD_VNODE_TSDB_SMA_H_
+#include "sma.h"
-#include "tsdb.h"
-#ifdef __cplusplus
-extern "C" {
-#endif
+// TODO: Who is responsible for resource allocate and release?
+int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) {
+ int32_t code = TSDB_CODE_SUCCESS;
-// typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2);
-
-// struct STbDdlH {
-// void *ahandle;
-// void *result;
-// __tb_ddl_fn_t fp;
-// };
-
-static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
- ASSERT(*pStore == NULL);
- *pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
- if (*pStore == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- return TSDB_CODE_FAILED;
+ if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) {
+ smaWarn("vgId:%d insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
}
- return TSDB_CODE_SUCCESS;
+ // TODO: destroy SSDataBlocks(msg)
+ return code;
}
-#ifdef __cplusplus
-}
-#endif
-#endif /*_TD_VNODE_TSDB_SMA_H_*/
\ No newline at end of file
diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c
new file mode 100644
index 0000000000..c02276f5fe
--- /dev/null
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@@ -0,0 +1,463 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "sma.h"
+
+typedef struct SSmaStat SSmaStat;
+
+static const char *TSDB_SMA_DNAME[] = {
+ "", // TSDB_SMA_TYPE_BLOCK
+ "tsma", // TSDB_SMA_TYPE_TIME_RANGE
+ "rsma", // TSDB_SMA_TYPE_ROLLUP
+};
+
+#define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test
+#define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test
+#define SMA_STATE_HASH_SLOT 4
+
+#define RSMA_TASK_INFO_HASH_SLOT 8
+
+typedef struct SPoolMem {
+ int64_t size;
+ struct SPoolMem *prev;
+ struct SPoolMem *next;
+} SPoolMem;
+
+// declaration of static functions
+
+// insert data
+
+static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]);
+
+// Pool Memory
+static SPoolMem *openPool();
+static void clearPool(SPoolMem *pPool);
+static void closePool(SPoolMem *pPool);
+static void *poolMalloc(void *arg, size_t size);
+static void poolFree(void *arg, void *ptr);
+
+// implementation
+
+static SPoolMem *openPool() {
+ SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool));
+
+ pPool->prev = pPool->next = pPool;
+ pPool->size = 0;
+
+ return pPool;
+}
+
+static void clearPool(SPoolMem *pPool) {
+ if (!pPool) return;
+
+ SPoolMem *pMem;
+
+ do {
+ pMem = pPool->next;
+
+ if (pMem == pPool) break;
+
+ pMem->next->prev = pMem->prev;
+ pMem->prev->next = pMem->next;
+ pPool->size -= pMem->size;
+
+ taosMemoryFree(pMem);
+ } while (1);
+
+ assert(pPool->size == 0);
+}
+
+static void closePool(SPoolMem *pPool) {
+ if (pPool) {
+ clearPool(pPool);
+ taosMemoryFree(pPool);
+ }
+}
+
+static void *poolMalloc(void *arg, size_t size) {
+ void *ptr = NULL;
+ SPoolMem *pPool = (SPoolMem *)arg;
+ SPoolMem *pMem;
+
+ pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size);
+ if (!pMem) {
+ assert(0);
+ }
+
+ pMem->size = sizeof(*pMem) + size;
+ pMem->next = pPool->next;
+ pMem->prev = pPool;
+
+ pPool->next->prev = pMem;
+ pPool->next = pMem;
+ pPool->size += pMem->size;
+
+ ptr = (void *)(&pMem[1]);
+ return ptr;
+}
+
+static void poolFree(void *arg, void *ptr) {
+ SPoolMem *pPool = (SPoolMem *)arg;
+ SPoolMem *pMem;
+
+ pMem = &(((SPoolMem *)ptr)[-1]);
+
+ pMem->next->prev = pMem->prev;
+ pMem->prev->next = pMem->next;
+ pPool->size -= pMem->size;
+
+ taosMemoryFree(pMem);
+}
+
+int32_t tdInitSma(SSma *pSma) {
+ // tSma
+ int32_t numOfTSma = taosArrayGetSize(metaGetSmaTbUids(SMA_META(pSma), false));
+ if (numOfTSma > 0) {
+ atomic_store_16(&SMA_TSMA_NUM(pSma), (int16_t)numOfTSma);
+ }
+ // TODO: rSma
+ return TSDB_CODE_SUCCESS;
+}
+
+static void tdGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
+ snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]);
+}
+
+static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path, SDiskID did) {
+ SSmaEnv *pEnv = NULL;
+
+ pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv));
+ if (!pEnv) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+
+ SMA_ENV_TYPE(pEnv) = smaType;
+
+ int code = taosThreadRwlockInit(&(pEnv->lock), NULL);
+ if (code) {
+ terrno = TAOS_SYSTEM_ERROR(code);
+ taosMemoryFree(pEnv);
+ return NULL;
+ }
+
+ ASSERT(path && (strlen(path) > 0));
+ SMA_ENV_PATH(pEnv) = strdup(path);
+ if (!SMA_ENV_PATH(pEnv)) {
+ tdFreeSmaEnv(pEnv);
+ return NULL;
+ }
+
+ SMA_ENV_DID(pEnv) = did;
+
+ if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) {
+ tdFreeSmaEnv(pEnv);
+ return NULL;
+ }
+
+ char aname[TSDB_FILENAME_LEN] = {0};
+ tfsAbsoluteName(SMA_TFS(pSma), did, path, aname);
+ if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
+ tdFreeSmaEnv(pEnv);
+ return NULL;
+ }
+
+ if (!(pEnv->pPool = openPool())) {
+ tdFreeSmaEnv(pEnv);
+ return NULL;
+ }
+
+ return pEnv;
+}
+
+static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) {
+ if (!pEnv) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ return TSDB_CODE_FAILED;
+ }
+
+ if (!(*pEnv)) {
+ if (!(*pEnv = tdNewSmaEnv(pSma, smaType, path, did))) {
+ return TSDB_CODE_FAILED;
+ }
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Release resources allocated for its member fields, not including itself.
+ *
+ * @param pSmaEnv
+ * @return int32_t
+ */
+void tdDestroySmaEnv(SSmaEnv *pSmaEnv) {
+ if (pSmaEnv) {
+ tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv));
+ taosMemoryFreeClear(pSmaEnv->pStat);
+ taosMemoryFreeClear(pSmaEnv->path);
+ taosThreadRwlockDestroy(&(pSmaEnv->lock));
+ smaCloseDBEnv(pSmaEnv->dbEnv);
+ closePool(pSmaEnv->pPool);
+ }
+}
+
+void *tdFreeSmaEnv(SSmaEnv *pSmaEnv) {
+ tdDestroySmaEnv(pSmaEnv);
+ taosMemoryFreeClear(pSmaEnv);
+ return NULL;
+}
+
+int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) {
+ if (!pStat) return 0;
+
+ int ref = T_REF_INC(pStat);
+ smaDebug("vgId:%d ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
+ return 0;
+}
+
+int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
+ if (!pStat) return 0;
+
+ int ref = T_REF_DEC(pStat);
+ smaDebug("vgId:%d unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
+ return 0;
+}
+
+static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
+ ASSERT(pSmaStat != NULL);
+
+ if (*pSmaStat) { // no lock
+ return TSDB_CODE_SUCCESS;
+ }
+
+ /**
+ * 1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew).
+ * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if
+ * tdInitSmaStat invoked in other multithread environment later.
+ */
+ if (!(*pSmaStat)) {
+ *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat));
+ if (!(*pSmaStat)) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+
+ if (smaType == TSDB_SMA_TYPE_ROLLUP) {
+ SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit(
+ RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
+
+ if (!SMA_STAT_INFO_HASH(*pSmaStat)) {
+ taosMemoryFreeClear(*pSmaStat);
+ return TSDB_CODE_FAILED;
+ }
+ } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
+ SMA_STAT_ITEMS(*pSmaStat) =
+ taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+
+ if (!SMA_STAT_ITEMS(*pSmaStat)) {
+ taosMemoryFreeClear(*pSmaStat);
+ return TSDB_CODE_FAILED;
+ }
+ } else {
+ ASSERT(0);
+ }
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) {
+ if (pSmaStatItem) {
+ tdDestroyTSma(pSmaStatItem->pTSma);
+ taosMemoryFreeClear(pSmaStatItem->pTSma);
+ taosHashCleanup(pSmaStatItem->expiredWindows);
+ taosMemoryFreeClear(pSmaStatItem);
+ }
+ return NULL;
+}
+
+/**
+ * @brief Release resources allocated for its member fields, not including itself.
+ *
+ * @param pSmaStat
+ * @return int32_t
+ */
+int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
+ if (pSmaStat) {
+ // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
+ if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
+ void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL);
+ while (item) {
+ SSmaStatItem *pItem = *(SSmaStatItem **)item;
+ tdFreeSmaStatItem(pItem);
+ item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item);
+ }
+ taosHashCleanup(SMA_STAT_ITEMS(pSmaStat));
+ } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
+ void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL);
+ while (infoHash) {
+ SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash;
+ tdFreeRSmaInfo(pInfoHash);
+ infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash);
+ }
+ taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat));
+ } else {
+ ASSERT(0);
+ }
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdLockSma(SSma *pSma) {
+ int code = taosThreadMutexLock(&pSma->mutex);
+ if (code != 0) {
+ smaError("vgId:%d failed to lock td since %s", SMA_VID(pSma), strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+ pSma->locked = true;
+ return 0;
+}
+
+int32_t tdUnLockSma(SSma *pSma) {
+ ASSERT(SMA_LOCKED(pSma));
+ pSma->locked = false;
+ int code = taosThreadMutexUnlock(&pSma->mutex);
+ if (code != 0) {
+ smaError("vgId:%d failed to unlock td since %s", SMA_VID(pSma), strerror(errno));
+ terrno = TAOS_SYSTEM_ERROR(code);
+ return -1;
+ }
+ return 0;
+}
+
+int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
+ SSmaEnv *pEnv = NULL;
+
+ // return if already init
+ switch (smaType) {
+ case TSDB_SMA_TYPE_TIME_RANGE:
+ if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_TSMA_ENV(pSma)))) {
+ return TSDB_CODE_SUCCESS;
+ }
+ break;
+ case TSDB_SMA_TYPE_ROLLUP:
+ if ((pEnv = (SSmaEnv *)atomic_load_ptr(&SMA_RSMA_ENV(pSma)))) {
+ return TSDB_CODE_SUCCESS;
+ }
+ break;
+ default:
+ TASSERT(0);
+ return TSDB_CODE_FAILED;
+ }
+
+ // init sma env
+ tdLockSma(pSma);
+ pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&SMA_TSMA_ENV(pSma))
+ : atomic_load_ptr(&SMA_RSMA_ENV(pSma));
+ if (!pEnv) {
+ char rname[TSDB_FILENAME_LEN] = {0};
+
+ SDiskID did = {0};
+ if (tfsAllocDisk(SMA_TFS(pSma), TFS_PRIMARY_LEVEL, &did) < 0) {
+ tdUnLockSma(pSma);
+ return TSDB_CODE_FAILED;
+ }
+
+ if (did.level < 0 || did.id < 0) {
+ tdUnLockSma(pSma);
+ smaError("vgId:%d init sma env failed since invalid did(%d,%d)", SMA_VID(pSma), did.level, did.id);
+ return TSDB_CODE_FAILED;
+ }
+
+ tdGetSmaDir(SMA_VID(pSma), smaType, rname);
+
+ if (tfsMkdirRecurAt(SMA_TFS(pSma), rname, did) < 0) {
+ tdUnLockSma(pSma);
+ return TSDB_CODE_FAILED;
+ }
+
+ if (tdInitSmaEnv(pSma, smaType, rname, did, &pEnv) < 0) {
+ tdUnLockSma(pSma);
+ return TSDB_CODE_FAILED;
+ }
+
+ (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), pEnv)
+ : atomic_store_ptr(&SMA_RSMA_ENV(pSma), pEnv);
+ }
+ tdUnLockSma(pSma);
+
+ return TSDB_CODE_SUCCESS;
+};
+
+int32_t tdSmaBeginCommit(SSmaEnv *pEnv) {
+ TXN *pTxn = &pEnv->txn;
+ // start a new txn
+ tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+ if (tdbBegin(pEnv->dbEnv, pTxn) != 0) {
+ smaWarn("tdSma tdb begin commit fail");
+ return -1;
+ }
+ return 0;
+}
+
+int32_t tdSmaEndCommit(SSmaEnv *pEnv) {
+ TXN *pTxn = &pEnv->txn;
+
+ // Commit current txn
+ if (tdbCommit(pEnv->dbEnv, pTxn) != 0) {
+ smaWarn("tdSma tdb end commit fail");
+ return -1;
+ }
+ tdbTxnClose(pTxn);
+ clearPool(pEnv->pPool);
+ return 0;
+}
+
+#if 0
+/**
+ * @brief Get the start TS key of the last data block of one interval/sliding.
+ *
+ * @param pSma
+ * @param param
+ * @param result
+ * @return int32_t
+ * 1) Return 0 and fill the result if the check procedure is normal;
+ * 2) Return -1 if error occurs during the check procedure.
+ */
+int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) {
+ const char *procedure = "";
+ if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) {
+ return -1;
+ }
+ // fill the result
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Remove the tSma data files related to param between pWin.
+ *
+ * @param pSma
+ * @param param
+ * @param pWin
+ * @return int32_t
+ */
+int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) {
+ // for ("tSmaFiles of param-interval-sliding between pWin") {
+ // // remove the tSmaFile
+ // }
+ return TSDB_CODE_SUCCESS;
+}
+#endif
diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c
new file mode 100644
index 0000000000..1c7db28e18
--- /dev/null
+++ b/source/dnode/vnode/src/sma/smaOpen.c
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "sma.h"
+#include "tsdb.h"
+
+static int32_t smaEvalDays(SRetention *r, int8_t precision);
+static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
+
+#define SMA_SET_KEEP_CFG(l) \
+ do { \
+ SRetention *r = &pCfg->retentions[l]; \
+ pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
+ pKeepCfg->keep0 = pKeepCfg->keep2; \
+ pKeepCfg->keep1 = pKeepCfg->keep2; \
+ pKeepCfg->days = smaEvalDays(r, pCfg->precision); \
+ } while (0)
+
+#define SMA_OPEN_RSMA_IMPL(v, l) \
+ do { \
+ SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \
+ if (!RETENTION_VALID(r)) { \
+ if (l == 0) { \
+ goto _err; \
+ } \
+ break; \
+ } \
+ smaSetKeepCfg(&keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \
+ if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg) < 0) { \
+ goto _err; \
+ } \
+ } while (0)
+
+#define RETENTION_DAYS_SPLIT_RATIO 10
+#define RETENTION_DAYS_SPLIT_MIN 1
+#define RETENTION_DAYS_SPLIT_MAX 30
+
+static int32_t smaEvalDays(SRetention *r, int8_t precision) {
+ int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY);
+ int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY);
+
+ int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO;
+ if (days <= RETENTION_DAYS_SPLIT_MIN) {
+ days = RETENTION_DAYS_SPLIT_MIN;
+ if (days < freqDays) {
+ days = freqDays + 1;
+ }
+ } else {
+ if (days > RETENTION_DAYS_SPLIT_MAX) {
+ days = RETENTION_DAYS_SPLIT_MAX;
+ }
+ if (days < freqDays) {
+ days = freqDays + 1;
+ }
+ }
+ return days * 1440;
+}
+
+int smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) {
+ pKeepCfg->precision = pCfg->precision;
+ switch (type) {
+ case TSDB_TYPE_TSMA:
+ ASSERT(0);
+ break;
+ case TSDB_TYPE_RSMA_L0:
+ SMA_SET_KEEP_CFG(0);
+ break;
+ case TSDB_TYPE_RSMA_L1:
+ SMA_SET_KEEP_CFG(1);
+ break;
+ case TSDB_TYPE_RSMA_L2:
+ SMA_SET_KEEP_CFG(2);
+ break;
+ default:
+ ASSERT(0);
+ break;
+ }
+ return 0;
+}
+
+int32_t smaOpen(SVnode *pVnode) {
+ STsdbCfg *pCfg = &pVnode->config.tsdbCfg;
+
+ ASSERT(!pVnode->pSma);
+
+ SSma *pSma = taosMemoryCalloc(1, sizeof(SSma));
+ if (!pSma) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+ pSma->pVnode = pVnode;
+ taosThreadMutexInit(&pSma->mutex, NULL);
+ pSma->locked = false;
+
+ if (vnodeIsRollup(pVnode)) {
+ STsdbKeepCfg keepCfg = {0};
+ for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
+ if (i == TSDB_RETENTION_L0) {
+ SMA_OPEN_RSMA_IMPL(pVnode, 0);
+ } else if (i == TSDB_RETENTION_L1) {
+ SMA_OPEN_RSMA_IMPL(pVnode, 1);
+ } else if (i == TSDB_RETENTION_L2) {
+ SMA_OPEN_RSMA_IMPL(pVnode, 2);
+ } else {
+ ASSERT(0);
+ }
+ }
+ }
+
+ pVnode->pSma = pSma;
+ return 0;
+_err:
+ taosMemoryFreeClear(pSma);
+ return -1;
+}
+
+int32_t smaClose(SSma *pSma) {
+ if (pSma) {
+ taosThreadMutexDestroy(&pSma->mutex);
+ if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
+ if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
+ if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));
+ }
+ return 0;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
new file mode 100644
index 0000000000..f9cb5a1a09
--- /dev/null
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -0,0 +1,484 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "sma.h"
+
+static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo,
+ STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level);
+
+struct SRSmaInfo {
+ void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t
+};
+
+static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) {
+ // Note: free/kill may in RC
+ qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
+ if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
+ qDestroyTask(otaskHandle);
+ }
+}
+
+void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
+ for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
+ if (pInfo->taskInfo[i]) {
+ tdFreeTaskHandle(pInfo->taskInfo[i]);
+ }
+ }
+ return NULL;
+}
+
+static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
+ ASSERT(*pStore == NULL);
+ *pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
+ if (*pStore == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
+ SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+ SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SRSmaInfo *pRSmaInfo = NULL;
+
+ if (!suid || !tbUids) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ smaError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+ return TSDB_CODE_FAILED;
+ }
+
+ pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
+ if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+ smaError("vgId:%d failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
+ terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+ return TSDB_CODE_FAILED;
+ }
+
+ if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) {
+ smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+ return TSDB_CODE_FAILED;
+ } else {
+ smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
+ pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+ }
+
+ if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) {
+ smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno));
+ return TSDB_CODE_FAILED;
+ } else {
+ smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma),
+ pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0));
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) {
+ if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) {
+ return TSDB_CODE_FAILED;
+ }
+
+ void *pIter = taosHashIterate(pStore->uidHash, NULL);
+ while (pIter) {
+ tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
+ SArray *pTbUids = *(SArray **)pIter;
+
+ if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) {
+ taosHashCancelIterate(pStore->uidHash, pIter);
+ return TSDB_CODE_FAILED;
+ }
+
+ pIter = taosHashIterate(pStore->uidHash, pIter);
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief fetch suid/uids when create child tables of rollup SMA
+ *
+ * @param pTsdb
+ * @param ppStore
+ * @param suid
+ * @param uid
+ * @return int32_t
+ */
+int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_uid_t uid) {
+ SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+
+ // only applicable to rollup SMA ctables
+ if (!pEnv) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SHashObj *infoHash = NULL;
+ if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) {
+ terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+ return TSDB_CODE_FAILED;
+ }
+
+ // info cached when create rsma stable and return directly for non-rsma ctables
+ if (!taosHashGet(infoHash, &suid, sizeof(tb_uid_t))) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ ASSERT(ppStore != NULL);
+
+ if (!(*ppStore)) {
+ if (tdUidStoreInit(ppStore) != 0) {
+ return TSDB_CODE_FAILED;
+ }
+ }
+
+ if (tdUidStorePut(*ppStore, suid, &uid) != 0) {
+ *ppStore = tdUidStoreFree(*ppStore);
+ return TSDB_CODE_FAILED;
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
+ *
+ * @param pTsdb
+ * @param pMeta
+ * @param pReq
+ * @return int32_t
+ */
+int32_t tdProcessRSmaCreate(SSma *pSma, SMeta *pMeta, SVCreateStbReq *pReq, SMsgCb *pMsgCb) {
+ if (!pReq->rollup) {
+ smaTrace("vgId:%d return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ SRSmaParam *param = &pReq->pRSmaParam;
+
+ if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) {
+ smaWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) {
+ terrno = TSDB_CODE_TDB_INIT_FAILED;
+ return TSDB_CODE_FAILED;
+ }
+
+ SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+ SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SRSmaInfo *pRSmaInfo = NULL;
+
+ pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
+ if (pRSmaInfo) {
+ smaWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo));
+ if (!pRSmaInfo) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+
+ STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta);
+ if (!pReadHandle) {
+ taosMemoryFree(pRSmaInfo);
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+
+ SReadHandle handle = {
+ .reader = pReadHandle,
+ .meta = pMeta,
+ .pMsgCb = pMsgCb,
+ };
+
+ if (param->qmsg1) {
+ pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle);
+ if (!pRSmaInfo->taskInfo[0]) {
+ taosMemoryFree(pRSmaInfo);
+ taosMemoryFree(pReadHandle);
+ return TSDB_CODE_FAILED;
+ }
+ }
+
+ if (param->qmsg2) {
+ pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle);
+ if (!pRSmaInfo->taskInfo[1]) {
+ taosMemoryFree(pRSmaInfo);
+ taosMemoryFree(pReadHandle);
+ return TSDB_CODE_FAILED;
+ }
+ }
+
+ if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) !=
+ TSDB_CODE_SUCCESS) {
+ return TSDB_CODE_FAILED;
+ } else {
+ smaDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief store suid/[uids], prefer to use array and then hash
+ *
+ * @param pStore
+ * @param suid
+ * @param uid
+ * @return int32_t
+ */
+static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) {
+ // prefer to store suid/uids in array
+ if ((suid == pStore->suid) || (pStore->suid == 0)) {
+ if (pStore->suid == 0) {
+ pStore->suid = suid;
+ }
+ if (uid) {
+ if (!pStore->tbUids) {
+ if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+ }
+ if (!taosArrayPush(pStore->tbUids, uid)) {
+ return TSDB_CODE_FAILED;
+ }
+ }
+ } else {
+ // store other suid/uids in hash when multiple stable/table included in 1 batch of request
+ if (!pStore->uidHash) {
+ pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
+ if (!pStore->uidHash) {
+ return TSDB_CODE_FAILED;
+ }
+ }
+ if (uid) {
+ SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t));
+ if (uidArray && ((uidArray = *(SArray **)uidArray))) {
+ taosArrayPush(uidArray, uid);
+ } else {
+ SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t));
+ if (!pUidArray) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+ if (!taosArrayPush(pUidArray, uid)) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+ if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) {
+ return TSDB_CODE_FAILED;
+ }
+ }
+ } else {
+ if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) {
+ return TSDB_CODE_FAILED;
+ }
+ }
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+void tdUidStoreDestory(STbUidStore *pStore) {
+ if (pStore) {
+ if (pStore->uidHash) {
+ if (pStore->tbUids) {
+ // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys.
+ void *pIter = taosHashIterate(pStore->uidHash, NULL);
+ while (pIter) {
+ SArray *arr = *(SArray **)pIter;
+ taosArrayDestroy(arr);
+ pIter = taosHashIterate(pStore->uidHash, pIter);
+ }
+ }
+ taosHashCleanup(pStore->uidHash);
+ }
+ taosArrayDestroy(pStore->tbUids);
+ }
+}
+
+void *tdUidStoreFree(STbUidStore *pStore) {
+ if (pStore) {
+ tdUidStoreDestory(pStore);
+ taosMemoryFree(pStore);
+ }
+ return NULL;
+}
+
+static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
+ if (!pReq) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ return TSDB_CODE_FAILED;
+ }
+
+ SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
+
+ if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) {
+ return TSDB_CODE_FAILED;
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
+ ASSERT(pMsg != NULL);
+ SSubmitMsgIter msgIter = {0};
+ SSubmitBlk *pBlock = NULL;
+ SSubmitBlkIter blkIter = {0};
+ STSRow *row = NULL;
+
+ terrno = TSDB_CODE_SUCCESS;
+
+ if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+ while (true) {
+ if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+
+ if (!pBlock) break;
+ tdUidStorePut(pStore, msgIter.suid, NULL);
+ pStore->uid = msgIter.uid; // TODO: remove, just for debugging
+ }
+
+ if (terrno != TSDB_CODE_SUCCESS) return -1;
+ return 0;
+}
+
+static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, qTaskInfo_t *taskInfo,
+ STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level) {
+ SArray *pResult = NULL;
+
+ if (!taskInfo) {
+ smaDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ smaDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, taskInfo, suid);
+
+ qSetStreamInput(taskInfo, pMsg, inputType);
+ while (1) {
+ SSDataBlock *output = NULL;
+ uint64_t ts;
+ if (qExecTask(taskInfo, &output, &ts) < 0) {
+ ASSERT(false);
+ }
+ if (!output) {
+ break;
+ }
+ if (!pResult) {
+ pResult = taosArrayInit(0, sizeof(SSDataBlock));
+ if (!pResult) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return TSDB_CODE_FAILED;
+ }
+ }
+
+ taosArrayPush(pResult, output);
+ }
+
+ if (taosArrayGetSize(pResult) > 0) {
+ blockDebugShowData(pResult);
+ STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
+ SSubmitReq *pReq = NULL;
+ if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), uid, suid) != 0) {
+ taosArrayDestroy(pResult);
+ return TSDB_CODE_FAILED;
+ }
+ if (tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) {
+ taosArrayDestroy(pResult);
+ taosMemoryFreeClear(pReq);
+ return TSDB_CODE_FAILED;
+ }
+ taosMemoryFreeClear(pReq);
+ } else {
+ smaWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno));
+ }
+
+ taosArrayDestroy(pResult);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) {
+ SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+ if (!pEnv) {
+ // only applicable when rsma env exists
+ return TSDB_CODE_SUCCESS;
+ }
+
+ ASSERT(uid != 0); // TODO: remove later
+
+ SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SRSmaInfo *pRSmaInfo = NULL;
+
+ pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+
+ if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+ smaDebug("vgId:%d no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
+ return TSDB_CODE_SUCCESS;
+ }
+ if (!pRSmaInfo->taskInfo[0]) {
+ smaDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+ // TODO: use the proper schema instead of 0, and cache STSchema in cache
+ STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, 0);
+ if (!pTSchema) {
+ terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+ return TSDB_CODE_FAILED;
+ }
+ tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1);
+ tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2);
+ taosMemoryFree(pTSchema);
+ }
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
+ SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+ if (!pEnv) {
+ // only applicable when rsma env exists
+ return TSDB_CODE_SUCCESS;
+ }
+
+ if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
+ STbUidStore uidStore = {0};
+ tdFetchSubmitReqSuids(pMsg, &uidStore);
+
+ if (uidStore.suid != 0) {
+ tdExecuteRSma(pSma, pMsg, inputType, uidStore.suid, uidStore.uid);
+
+ void *pIter = taosHashIterate(uidStore.uidHash, NULL);
+ while (pIter) {
+ tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
+ tdExecuteRSma(pSma, pMsg, inputType, *pTbSuid, 0);
+ pIter = taosHashIterate(uidStore.uidHash, pIter);
+ }
+
+ tdUidStoreDestory(&uidStore);
+ }
+ }
+ return TSDB_CODE_SUCCESS;
+}
diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c
new file mode 100644
index 0000000000..821ec44aa5
--- /dev/null
+++ b/source/dnode/vnode/src/sma/smaTDBImpl.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#define ALLOW_FORBID_FUNC
+
+#include "sma.h"
+
+int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) {
+ int ret = 0;
+
+ if (path == NULL) return -1;
+
+ ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param
+
+ if (ret != 0) {
+ smaError("failed to create tsdb db env, ret = %d", ret);
+ return -1;
+ }
+
+ return 0;
+}
+
+int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); }
+
+static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
+ const SSmaKey *pKey1 = (const SSmaKey *)arg1;
+ const SSmaKey *pKey2 = (const SSmaKey *)arg2;
+
+ ASSERT(len1 == len2 && len1 == sizeof(SSmaKey));
+
+ if (pKey1->skey < pKey2->skey) {
+ return -1;
+ } else if (pKey1->skey > pKey2->skey) {
+ return 1;
+ }
+ if (pKey1->groupId < pKey2->groupId) {
+ return -1;
+ } else if (pKey1->groupId > pKey2->groupId) {
+ return 1;
+ }
+
+ return 0;
+}
+
+static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) {
+ int ret;
+ tdb_cmpr_fn_t compFunc;
+
+ // Create a database
+ compFunc = tdSmaKeyCmpr;
+ ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
+
+ return 0;
+}
+
+static int32_t smaCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); }
+
+int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) {
+ // TEnv is shared by a group of SDBFile
+ if (!pEnv || !pDBF) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ return -1;
+ }
+
+ // Open DBF
+ if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
+ terrno = TSDB_CODE_TDB_INIT_FAILED;
+ smaCloseDBDb(pDBF->pDB);
+ return -1;
+ }
+
+ return 0;
+}
+
+int32_t smaCloseDBF(SDBFile *pDBF) {
+ int32_t ret = 0;
+ if (pDBF->pDB) {
+ ret = smaCloseDBDb(pDBF->pDB);
+ pDBF->pDB = NULL;
+ }
+ taosMemoryFreeClear(pDBF->path);
+ return ret;
+}
+
+int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
+ int32_t ret;
+
+ ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
+ if (ret < 0) {
+ smaError("failed to create insert sma data into db, ret = %d", ret);
+ return -1;
+ }
+
+ return 0;
+}
+
+void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) {
+ void *pVal = NULL;
+ int ret;
+
+ ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
+
+ if (ret < 0) {
+ smaError("failed to get sma data from db, ret = %d", ret);
+ return NULL;
+ }
+
+ ASSERT(*valLen >= 0);
+
+ // TODO: lock?
+ // TODO: Would the key/value be destoryed during return the data?
+ // TODO: How about the key is updated while value length is changed? The original value buffer would be freed
+ // automatically?
+
+ return pVal;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c
new file mode 100644
index 0000000000..b04885c5f0
--- /dev/null
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@@ -0,0 +1,1103 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "sma.h"
+#include "tsdb.h"
+
+#undef _TEST_SMA_PRINT_DEBUG_LOG_
+#define SMA_STORAGE_TSDB_DAYS 30
+#define SMA_STORAGE_TSDB_TIMES 10
+#define SMA_STORAGE_SPLIT_HOURS 24
+#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8
+#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds
+
+#define SMA_STATE_ITEM_HASH_SLOT 32
+
+
+typedef struct {
+ SSma *pSma;
+ SDBFile dFile;
+ const SArray *pDataBlocks; // sma data
+ int32_t interval; // interval with the precision of DB
+} STSmaWriteH;
+
+typedef struct {
+ int32_t iter;
+ int32_t fid;
+} SmaFsIter;
+
+typedef struct {
+ STsdb *pTsdb;
+ SSma *pSma;
+ SDBFile dFile;
+ int32_t interval; // interval with the precision of DB
+ int32_t blockSize; // size of SMA block item
+ int8_t storageLevel;
+ int8_t days;
+ SmaFsIter smaFsIter;
+} STSmaReadH;
+
+typedef enum {
+ SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma
+ SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma
+} ESmaStorageLevel;
+
+
+// static func
+
+static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
+static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit);
+static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
+ int8_t intervalUnit);
+static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit);
+static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH);
+static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel);
+static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid);
+static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey);
+static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
+static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
+ TXN *txn);
+// expired window
+static int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version);
+static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
+ int64_t version);
+static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
+static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
+
+// read data
+// TODO: This is the basic params, and should wrap the params to a queryHandle.
+static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
+
+// implementation
+
+/**
+ * @brief
+ *
+ * @param pSmaH
+ * @param pSma
+ * @param interval
+ * @param intervalUnit
+ * @return int32_t
+ */
+static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) {
+ pSmaH->pSma = pSma;
+ pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
+ pSmaH->storageLevel = tdGetSmaStorageLevel(interval, intervalUnit);
+ pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel);
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Init of tSma FS
+ *
+ * @param pReadH
+ * @param indexUid
+ * @param skey
+ * @return int32_t
+ */
+static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) {
+ SSma *pSma = pSmaH->pSma;
+
+ int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision));
+ char tSmaFile[TSDB_FILENAME_LEN] = {0};
+ snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid);
+ pSmaH->dFile.path = strdup(tSmaFile);
+ pSmaH->smaFsIter.iter = 0;
+ pSmaH->smaFsIter.fid = fid;
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Set and open tSma file if it has key locates in queryWin.
+ *
+ * @param pReadH
+ * @param param
+ * @param queryWin
+ * @return true
+ * @return false
+ */
+static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
+ // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf;
+ // int32_t nSmaFs = taosArrayGetSize(smaFs);
+
+ smaCloseDBF(&pReadH->dFile);
+
+#if 0
+ while (pReadH->smaFsIter.iter < nSmaFs) {
+ void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter);
+ if (pSmaFile) { // match(indexName, queryWindow)
+ // TODO: select the file by index_name ...
+ pReadH->dFile = pSmaFile;
+ ++pReadH->smaFsIter.iter;
+ break;
+ }
+ ++pReadH->smaFsIter.iter;
+ }
+
+ if (pReadH->pDFile) {
+ tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]");
+ return true;
+ }
+#endif
+
+ return false;
+}
+
+
+/**
+ * @brief Approximate value for week/month/year.
+ *
+ * @param interval
+ * @param intervalUnit
+ * @param precision
+ * @param adjusted Interval already adjusted according to DB precision
+ * @return int64_t
+ */
+static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) {
+ if (adjusted) {
+ return interval;
+ }
+
+ switch (intervalUnit) {
+ case TIME_UNIT_YEAR: // approximate value
+ interval *= 365 * 86400 * 1e3;
+ break;
+ case TIME_UNIT_MONTH: // approximate value
+ interval *= 30 * 86400 * 1e3;
+ break;
+ case TIME_UNIT_WEEK: // approximate value
+ interval *= 7 * 86400 * 1e3;
+ break;
+ case TIME_UNIT_DAY: // the interval for tSma calculation must <= day
+ interval *= 86400 * 1e3;
+ break;
+ case TIME_UNIT_HOUR:
+ interval *= 3600 * 1e3;
+ break;
+ case TIME_UNIT_MINUTE:
+ interval *= 60 * 1e3;
+ break;
+ case TIME_UNIT_SECOND:
+ interval *= 1e3;
+ break;
+ default:
+ break;
+ }
+
+ switch (precision) {
+ case TSDB_TIME_PRECISION_MILLI:
+ if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
+ return interval / 1e3;
+ } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second
+ return interval / 1e6;
+ } else { // ms
+ return interval;
+ }
+ break;
+ case TSDB_TIME_PRECISION_MICRO:
+ if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
+ return interval;
+ } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
+ return interval / 1e3;
+ } else { // ms
+ return interval * 1e3;
+ }
+ break;
+ case TSDB_TIME_PRECISION_NANO:
+ if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
+ return interval * 1e3;
+ } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
+ return interval;
+ } else { // ms
+ return interval * 1e6;
+ }
+ break;
+ default: // ms
+ if (TIME_UNIT_MICROSECOND == intervalUnit) { // us
+ return interval / 1e3;
+ } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns
+ return interval / 1e6;
+ } else { // ms
+ return interval;
+ }
+ break;
+ }
+ return interval;
+}
+
+
+static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
+ int8_t intervalUnit) {
+ pSmaH->pSma = pSma;
+ pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
+ pSmaH->pDataBlocks = pDataBlocks;
+ pSmaH->dFile.fid = SMA_IVLD_FID;
+ return TSDB_CODE_SUCCESS;
+}
+
+static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
+ if (pSmaH) {
+ smaCloseDBF(&pSmaH->dFile);
+ }
+}
+
+static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) {
+ SSma *pSma = pSmaH->pSma;
+ ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB);
+
+ pSmaH->dFile.fid = fid;
+ char tSmaFile[TSDB_FILENAME_LEN] = {0};
+ snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid);
+ pSmaH->dFile.path = strdup(tSmaFile);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief
+ *
+ * @param pSma
+ * @param interval Interval calculated by DB's precision
+ * @param storageLevel
+ * @return int32_t
+ */
+static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) {
+ STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
+ int32_t daysPerFile = pCfg->days;
+
+ if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
+ int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
+ daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS;
+ }
+
+ return daysPerFile;
+}
+
+/**
+ * @brief Judge the tSma storage level
+ *
+ * @param interval
+ * @param intervalUnit
+ * @return int32_t
+ */
+static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
+ // TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
+ switch (intervalUnit) {
+ case TIME_UNIT_HOUR:
+ if (interval < SMA_STORAGE_SPLIT_HOURS) {
+ return SMA_STORAGE_LEVEL_DFILESET;
+ }
+ break;
+ case TIME_UNIT_MINUTE:
+ if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) {
+ return SMA_STORAGE_LEVEL_DFILESET;
+ }
+ break;
+ case TIME_UNIT_SECOND:
+ if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) {
+ return SMA_STORAGE_LEVEL_DFILESET;
+ }
+ break;
+ case TIME_UNIT_MILLISECOND:
+ if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) {
+ return SMA_STORAGE_LEVEL_DFILESET;
+ }
+ break;
+ case TIME_UNIT_MICROSECOND:
+ if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) {
+ return SMA_STORAGE_LEVEL_DFILESET;
+ }
+ break;
+ case TIME_UNIT_NANOSECOND:
+ if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) {
+ return SMA_STORAGE_LEVEL_DFILESET;
+ }
+ break;
+ default:
+ break;
+ }
+ return SMA_STORAGE_LEVEL_TSDB;
+}
+
+/**
+ * @brief Insert/Update Time-range-wise SMA data.
+ * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g.
+ * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files.
+ * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The
+ * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d).
+ * - The destination file of one data block for some interval is determined by its start TS key.
+ *
+ * @param pSma
+ * @param msg
+ * @return int32_t
+ */
+int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
+ STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
+ const SArray *pDataBlocks = (const SArray *)msg;
+
+ // TODO: destroy SSDataBlocks(msg)
+
+ // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus
+ // the sma data would arrive ahead of the update-expired-window msg.
+ if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) {
+ terrno = TSDB_CODE_TDB_INIT_FAILED;
+ return TSDB_CODE_FAILED;
+ }
+
+ if (!pDataBlocks) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma));
+ return terrno;
+ }
+
+ if (taosArrayGetSize(pDataBlocks) <= 0) {
+ terrno = TSDB_CODE_INVALID_PARA;
+ smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma));
+ return TSDB_CODE_FAILED;
+ }
+
+ SSmaEnv *pEnv = SMA_TSMA_ENV(pSma);
+ SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SSmaStatItem *pItem = NULL;
+
+ tdRefSmaStat(pSma, pStat);
+
+ if (pStat && SMA_STAT_ITEMS(pStat)) {
+ pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid));
+ }
+
+ if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) {
+ terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+ tdUnRefSmaStat(pSma, pStat);
+ return TSDB_CODE_FAILED;
+ }
+
+ STSma *pTSma = pItem->pTSma;
+ STSmaWriteH tSmaH = {0};
+
+ if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) {
+ return TSDB_CODE_FAILED;
+ }
+
+ char rPath[TSDB_FILENAME_LEN] = {0};
+ char aPath[TSDB_FILENAME_LEN] = {0};
+ snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
+ tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath);
+ if (!taosCheckExistFile(aPath)) {
+ if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
+ tdUnRefSmaStat(pSma, pStat);
+ return TSDB_CODE_FAILED;
+ }
+ }
+
+ // Step 1: Judge the storage level and days
+ int32_t storageLevel = tdGetSmaStorageLevel(pTSma->interval, pTSma->intervalUnit);
+ int32_t daysPerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel);
+
+ char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId
+ char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer?
+ void *pDataBuf = NULL;
+ int32_t sz = taosArrayGetSize(pDataBlocks);
+ for (int32_t i = 0; i < sz; ++i) {
+ SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i);
+ int32_t colNum = pDataBlock->info.numOfCols;
+ int32_t rows = pDataBlock->info.rows;
+ int32_t rowSize = pDataBlock->info.rowSize;
+ int64_t groupId = pDataBlock->info.groupId;
+ for (int32_t j = 0; j < rows; ++j) {
+ printf("|");
+ TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval
+ void *pSmaKey = &smaKey;
+ bool isStartKey = false;
+
+ int32_t tlen = 0; // reset the len
+ pDataBuf = &dataBuf; // reset the buf
+ for (int32_t k = 0; k < colNum; ++k) {
+ SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
+ void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+ switch (pColInfoData->info.type) {
+ case TSDB_DATA_TYPE_TIMESTAMP:
+ if (!isStartKey) {
+ isStartKey = true;
+ skey = *(TSKEY *)var;
+ printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
+ tdEncodeTSmaKey(groupId, skey, &pSmaKey);
+ } else {
+ printf(" %" PRIi64 " |", *(int64_t *)var);
+ tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
+ break;
+ }
+ break;
+ case TSDB_DATA_TYPE_BOOL:
+ case TSDB_DATA_TYPE_UTINYINT:
+ printf(" %15d |", *(uint8_t *)var);
+ tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var);
+ break;
+ case TSDB_DATA_TYPE_TINYINT:
+ printf(" %15d |", *(int8_t *)var);
+ tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var);
+ break;
+ case TSDB_DATA_TYPE_SMALLINT:
+ printf(" %15d |", *(int16_t *)var);
+ tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var);
+ break;
+ case TSDB_DATA_TYPE_USMALLINT:
+ printf(" %15d |", *(uint16_t *)var);
+ tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var);
+ break;
+ case TSDB_DATA_TYPE_INT:
+ printf(" %15d |", *(int32_t *)var);
+ tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var);
+ break;
+ case TSDB_DATA_TYPE_FLOAT:
+ printf(" %15f |", *(float *)var);
+ tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float));
+ break;
+ case TSDB_DATA_TYPE_UINT:
+ printf(" %15u |", *(uint32_t *)var);
+ tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var);
+ break;
+ case TSDB_DATA_TYPE_BIGINT:
+ printf(" %15ld |", *(int64_t *)var);
+ tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
+ break;
+ case TSDB_DATA_TYPE_DOUBLE:
+ printf(" %15lf |", *(double *)var);
+ tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double));
+ case TSDB_DATA_TYPE_UBIGINT:
+ printf(" %15lu |", *(uint64_t *)var);
+ tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var);
+ break;
+ case TSDB_DATA_TYPE_NCHAR: {
+ char tmpChar[100] = {0};
+ strncpy(tmpChar, varDataVal(var), varDataLen(var));
+ printf(" %s |", tmpChar);
+ tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var));
+ break;
+ }
+ case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
+ char tmpChar[100] = {0};
+ strncpy(tmpChar, varDataVal(var), varDataLen(var));
+ printf(" %s |", tmpChar);
+ tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var));
+ break;
+ }
+ case TSDB_DATA_TYPE_VARBINARY:
+ // TODO: add binary/varbinary
+ TASSERT(0);
+ default:
+ printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
+ TASSERT(0);
+ break;
+ }
+ }
+ // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) {
+ if (tlen > 0) {
+ int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision));
+
+ // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index
+ // file
+ // - Set and open the DFile or the B+Tree file
+ // TODO: tsdbStartTSmaCommit();
+ if (fid != tSmaH.dFile.fid) {
+ if (tSmaH.dFile.fid != SMA_IVLD_FID) {
+ tdSmaEndCommit(pEnv);
+ smaCloseDBF(&tSmaH.dFile);
+ }
+ tdSetTSmaDataFile(&tSmaH, indexUid, fid);
+ if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
+ smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma),
+ tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
+ tdDestroyTSmaWriteH(&tSmaH);
+ tdUnRefSmaStat(pSma, pStat);
+ return TSDB_CODE_FAILED;
+ }
+ tdSmaBeginCommit(pEnv);
+ }
+
+ if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
+ smaWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
+ " since %s",
+ SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno));
+ tdSmaEndCommit(pEnv);
+ tdDestroyTSmaWriteH(&tSmaH);
+ tdUnRefSmaStat(pSma, pStat);
+ return TSDB_CODE_FAILED;
+ }
+ smaDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
+ SMA_VID(pSma), indexUid, skey, groupId);
+ // TODO:tsdbEndTSmaCommit();
+
+ // Step 3: reset the SSmaStat
+ tdResetExpiredWindow(pSma, pStat, indexUid, skey);
+ } else {
+ smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64,
+ SMA_VID(pSma), skey, tlen, indexUid);
+ }
+
+ printf("\n");
+ }
+ }
+ tdSmaEndCommit(pEnv); // TODO: not commit for every insert
+ tdDestroyTSmaWriteH(&tSmaH);
+ tdUnRefSmaStat(pSma, pStat);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) {
+ smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
+ }
+ return code;
+}
+
+/**
+ * @brief Insert TSma data blocks to DB File build by B+Tree
+ *
+ * @param pSmaH
+ * @param smaKey tableUid-colId-skeyOfWindow(8-2-8)
+ * @param keyLen
+ * @param pData
+ * @param dataLen
+ * @return int32_t
+ */
+static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
+ TXN *txn) {
+ SDBFile *pDBFile = &pSmaH->dFile;
+
+ // TODO: insert sma data blocks into B+Tree(TDB)
+ if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
+ smaWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
+ SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
+ return TSDB_CODE_FAILED;
+ }
+ smaDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
+ SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
+
+#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
+ uint32_t valueSize = 0;
+ void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize);
+ ASSERT(data != NULL);
+ for (uint32_t v = 0; v < valueSize; v += 8) {
+ smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v));
+ }
+#endif
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief When sma data received from stream computing, make the relative expired window valid.
+ *
+ * @param pSma
+ * @param pStat
+ * @param indexUid
+ * @param skey
+ * @return int32_t
+ */
+static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) {
+ SSmaStatItem *pItem = NULL;
+
+ tdRefSmaStat(pSma, pStat);
+
+ if (pStat && SMA_STAT_ITEMS(pStat)) {
+ pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid));
+ }
+ if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) {
+ // pItem resides in hash buffer all the time unless drop sma index
+ // TODO: multithread protect
+ if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
+ // error handling
+ tdUnRefSmaStat(pSma, pStat);
+ smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma),
+ skey, indexUid);
+ return TSDB_CODE_FAILED;
+ }
+ smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma),
+ skey, indexUid);
+ // TODO: use a standalone interface to received state upate notification from stream computing module.
+ /**
+ * @brief state
+ * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK.
+ * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g.
+ * when batch data caculation not finised)
+ * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB.
+ */
+ pItem->state = TSDB_SMA_STAT_OK;
+ } else {
+ // error handling
+ tdUnRefSmaStat(pSma, pStat);
+ smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid);
+ return TSDB_CODE_FAILED;
+ }
+
+ tdUnRefSmaStat(pSma, pStat);
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief Drop tSma data and local cache
+ * - insert/query reference
+ * @param pSma
+ * @param msg
+ * @return int32_t
+ */
+static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) {
+ SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma));
+
+ // clear local cache
+ if (pEnv) {
+ smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid);
+
+ SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
+ if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) {
+ if (tdSmaStatIsDropped(pItem)) {
+ smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid);
+ return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode
+ }
+
+ tdWLockSmaEnv(pEnv);
+ if (tdSmaStatIsDropped(pItem)) {
+ tdUnLockSmaEnv(pEnv);
+ smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid);
+ return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode
+ }
+ tdSmaStatSetDropped(pItem);
+ tdUnLockSmaEnv(pEnv);
+
+ int32_t nSleep = 0;
+ int32_t refVal = INT32_MAX;
+ while (true) {
+ if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) {
+ smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
+ break;
+ }
+ smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
+ taosSsleep(1);
+ if (++nSleep > SMA_DROP_EXPIRED_TIME) {
+ smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep,
+ refVal);
+ break;
+ };
+ }
+
+ tdFreeSmaStatItem(pItem);
+ smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid);
+ }
+ }
+ // clear sma data files
+ // TODO:
+ return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief
+ *
+ * @param pSma Return the data between queryWin and fill the pData.
+ * @param pData
+ * @param indexUid
+ * @param pQuerySKey
+ * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
+ * @return int32_t
+ */
+static int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
+ SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma));
+ SSmaStat *pStat = NULL;
+
+ if (!pEnv) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma));
+ return TSDB_CODE_FAILED;
+ }
+
+ pStat = SMA_ENV_STAT(pEnv);
+
+ tdRefSmaStat(pSma, pStat);
+ SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
+ if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) {
+ // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
+ // it's NULL.
+ tdUnRefSmaStat(pSma, pStat);
+ terrno = TSDB_CODE_TDB_INVALID_ACTION;
+ smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid);
+ return TSDB_CODE_FAILED;
+ }
+
+#if 0
+ int32_t nQueryWin = taosArrayGetSize(pQuerySKey);
+ for (int32_t n = 0; n < nQueryWin; ++n) {
+ TSKEY skey = taosArrayGet(pQuerySKey, n);
+ if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) {
+ // TODO: mark this window as expired.
+ }
+ }
+#endif
+
+#if 1
+ int8_t smaStat = 0;
+ if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query
+ tdUnRefSmaStat(pSma, pStat);
+ terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+ smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid,
+ tstrerror(terrno), smaStat);
+ return TSDB_CODE_FAILED;
+ }
+
+ if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) {
+ // TODO: mark this window as expired.
+ smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma),
+ querySKey, indexUid);
+ } else {
+ smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
+ indexUid);
+ }
+
+ STSma *pTSma = pItem->pTSma;
+#endif
+
+#if 1
+ STSmaReadH tReadH = {0};
+ tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit);
+ smaCloseDBF(&tReadH.dFile);
+
+ tdUnRefSmaStat(pSma, pStat);
+
+ tdInitTSmaFile(&tReadH, indexUid, querySKey);
+ if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) {
+ smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno));
+ return TSDB_CODE_FAILED;
+ }
+
+ char smaKey[SMA_KEY_LEN] = {0};
+ void *pSmaKey = &smaKey;
+ int64_t queryGroupId = 1;
+ tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
+
+ smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma),
+ tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN);
+
+ void *result = NULL;
+ int32_t valueSize = 0;
+ if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) {
+ smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s",
+ SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno));
+ smaCloseDBF(&tReadH.dFile);
+ return TSDB_CODE_FAILED;
+ }
+ #endif
+
+#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
+ for (uint32_t v = 0; v < valueSize; v += 8) {
+ smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v));
+ }
+#endif
+ taosMemoryFreeClear(result); // TODO: fill the result to output
+
+#if 0
+ int32_t nResult = 0;
+ int64_t lastKey = 0;
+
+ while (true) {
+ if (nResult >= nMaxResult) {
+ break;
+ }
+
+ // set and open the file according to the STSma param
+ if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) {
+ char bTree[100] = "\0";
+ while (strncmp(bTree, "has more nodes", 100) == 0) {
+ if (nResult >= nMaxResult) {
+ break;
+ }
+ // tdGetDataFromBTree(bTree, queryWin, lastKey)
+ // fill the pData
+ ++nResult;
+ }
+ }
+ }
+#endif
+ // read data from file and fill the result
+ smaCloseDBF(&tReadH.dFile);
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdProcessTSmaCreate(SSma *pSma, char *pMsg) {
+ #if 0
+ SSmaCfg vCreateSmaReq = {0};
+ if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ smaWarn("vgId:%d tsma create msg received but deserialize failed since %s", SMA_VID(pSma), terrstr(terrno));
+ return -1;
+ }
+
+ smaDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", SMA_VID(pSma),
+ vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid);
+
+ // record current timezone of server side
+ vCreateSmaReq.tSma.timezoneInt = tsTimezone;
+
+ if (metaCreateTSma(SMA_META(pSma), &vCreateSmaReq) < 0) {
+ // TODO: handle error
+ smaWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", SMA_VID(pSma),
+ vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno));
+ tdDestroyTSma(&vCreateSmaReq.tSma);
+ return -1;
+ }
+
+ tdTSmaAdd(pSma, 1);
+
+ tdDestroyTSma(&vCreateSmaReq.tSma);
+ // TODO: return directly or go on follow steps?
+#endif
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tdDropTSma(SSma *pSma, char *pMsg) {
+#if 0
+ SVDropTSmaReq vDropSmaReq = {0};
+ if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ // TODO: send msg to stream computing to drop tSma
+ // if ((send msg to stream computing) < 0) {
+ // tdDestroyTSma(&vCreateSmaReq);
+ // return -1;
+ // }
+ //
+
+ if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) {
+ // TODO: handle error
+ return -1;
+ }
+
+ if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) {
+ // TODO: handle error
+ return -1;
+ }
+
+ tdTSmaSub(pSma, 1);
+#endif
+
+ // TODO: return directly or go on follow steps?
+ return TSDB_CODE_SUCCESS;
+}
+
+static SSmaStatItem *tdNewSmaStatItem(int8_t state) {
+ SSmaStatItem *pItem = NULL;
+
+ pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem));
+ if (!pItem) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return NULL;
+ }
+
+ pItem->state = state;
+ pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP),
+ true, HASH_ENTRY_LOCK);
+ if (!pItem->expiredWindows) {
+ taosMemoryFreeClear(pItem);
+ return NULL;
+ }
+
+ return pItem;
+}
+
+static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
+ int64_t version) {
+ SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
+ if (!pItem) {
+ // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later
+ pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state
+ if (!pItem) {
+ // Response to stream computing: OOM
+ // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data.
+ return TSDB_CODE_FAILED;
+ }
+
+ // cache smaMeta
+ STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid, true);
+ if (!pTSma) {
+ terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
+ taosHashCleanup(pItem->expiredWindows);
+ taosMemoryFree(pItem);
+ smaWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid,
+ tstrerror(terrno));
+ return TSDB_CODE_FAILED;
+ }
+ pItem->pTSma = pTSma;
+
+ if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) {
+ // If error occurs during put smaStatItem, free the resources of pItem
+ taosHashCleanup(pItem->expiredWindows);
+ taosMemoryFree(pItem);
+ return TSDB_CODE_FAILED;
+ }
+ } else if (!(pItem = *(SSmaStatItem **)pItem)) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ return TSDB_CODE_FAILED;
+ }
+
+ if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) {
+ // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would
+ // tell query module to query raw TS data.
+ // N.B.
+ // 1) It is assumed to be extemely little probability event of fail to taosHashPut.
+ // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired
+ // windows failed to put into hash table.
+ taosHashCleanup(pItem->expiredWindows);
+ taosMemoryFreeClear(pItem->pTSma);
+ taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid));
+ smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid,
+ winSKey);
+ return TSDB_CODE_FAILED;
+ }
+
+ smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid,
+ winSKey);
+ return TSDB_CODE_SUCCESS;
+}
+
+
+
+/**
+ * @brief Update expired window according to msg from stream computing module.
+ *
+ * @param pSma
+ * @param msg SSubmitReq
+ * @return int32_t
+ */
+int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) {
+ // no time-range-sma, just return success
+ if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) {
+ smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma));
+ return TSDB_CODE_SUCCESS;
+ }
+
+ if (!SMA_META(pSma)) {
+ terrno = TSDB_CODE_INVALID_PTR;
+ smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma));
+ return TSDB_CODE_FAILED;
+ }
+
+ if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) {
+ smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno));
+ terrno = TSDB_CODE_TDB_INIT_FAILED;
+ return TSDB_CODE_FAILED;
+ }
+
+ // Firstly, assume that tSma can only be created on super table/normal table.
+ // getActiveTimeWindow
+
+ SSmaEnv *pEnv = SMA_TSMA_ENV(pSma);
+ SSmaStat *pStat = SMA_ENV_STAT(pEnv);
+ SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
+
+ TASSERT(pEnv && pStat && pItemsHash);
+
+ // basic procedure
+ // TODO: optimization
+ tdRefSmaStat(pSma, pStat);
+
+ SSubmitMsgIter msgIter = {0};
+ SSubmitBlk *pBlock = NULL;
+ SInterval interval = {0};
+ TSKEY lastWinSKey = INT64_MIN;
+
+ if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
+ return TSDB_CODE_FAILED;
+ }
+
+ while (true) {
+ tGetSubmitMsgNext(&msgIter, &pBlock);
+ if (!pBlock) break;
+
+ STSmaWrapper *pSW = NULL;
+ STSma *pTSma = NULL;
+
+ SSubmitBlkIter blkIter = {0};
+ if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) {
+ pSW = tdFreeTSmaWrapper(pSW);
+ break;
+ }
+
+ while (true) {
+ STSRow *row = tGetSubmitBlkNext(&blkIter);
+ if (!row) {
+ tdFreeTSmaWrapper(pSW);
+ break;
+ }
+ if (!pSW || (pTSma->tableUid != pBlock->suid)) {
+ if (pSW) {
+ pSW = tdFreeTSmaWrapper(pSW);
+ }
+ if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), pBlock->suid))) {
+ break;
+ }
+ if ((pSW->number) <= 0 || !pSW->tSma) {
+ pSW = tdFreeTSmaWrapper(pSW);
+ break;
+ }
+
+ pTSma = pSW->tSma;
+
+ interval.interval = pTSma->interval;
+ interval.intervalUnit = pTSma->intervalUnit;
+ interval.offset = pTSma->offset;
+ interval.precision = SMA_TSDB_CFG(pSma)->precision;
+ interval.sliding = pTSma->sliding;
+ interval.slidingUnit = pTSma->slidingUnit;
+ }
+
+ TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
+
+ if (lastWinSKey != winSKey) {
+ lastWinSKey = winSKey;
+ if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) {
+ tdUnRefSmaStat(pSma, pStat);
+ return TSDB_CODE_FAILED;
+ }
+ } else {
+ smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated",
+ SMA_VID(pSma), pTSma->indexUid, winSKey);
+ }
+ }
+ }
+
+ tdUnRefSmaStat(pSma, pStat);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t tdUpdateExpireWindow(SSma *pSma, SSubmitReq *pMsg, int64_t version) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) {
+ smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno));
+ }
+ return code;
+}
+
+int32_t tdGetTSmaData(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
+ int32_t code = TSDB_CODE_SUCCESS;
+ if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) {
+ smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
+ }
+ return code;
+}
+
+
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 9526451907..873db62dd8 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -14,6 +14,7 @@
*/
#include "tq.h"
+#include "tqueue.h"
int32_t tqInit() {
//
@@ -234,7 +235,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
if (msgType != TDMT_VND_SUBMIT) return 0;
// make sure msgType == TDMT_VND_SUBMIT
- if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
+ if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
return -1;
}
@@ -1031,3 +1032,90 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId)
}
return 0;
}
+
+int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
+ void* pIter = NULL;
+ bool failed = false;
+
+ SStreamDataSubmit* pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
+ if (pSubmit == NULL) {
+ failed = true;
+ }
+ pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t));
+ if (pSubmit->dataRef == NULL) {
+ failed = true;
+ }
+
+ pSubmit->type = STREAM_DATA_TYPE_SUBMIT_BLOCK;
+ pSubmit->sourceVer = ver;
+ pSubmit->sourceVg = pTq->pVnode->config.vgId;
+ pSubmit->data = pReq;
+ *pSubmit->dataRef = 1;
+
+ while (1) {
+ pIter = taosHashIterate(pTq->pStreamTasks, pIter);
+ if (pIter == NULL) break;
+ SStreamTask* pTask = (SStreamTask*)pIter;
+ if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue;
+
+ int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
+ if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
+ if (failed) {
+ atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
+ continue;
+ }
+
+ streamDataSubmitRefInc(pSubmit);
+ taosWriteQitem(pTask->inputQ, pSubmit);
+
+ int8_t execStatus = atomic_load_8(&pTask->status);
+ if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
+ // TODO dispatch task launch msg to fetch queue
+ }
+
+ } else {
+ // blocked or stopped, do nothing
+ }
+ }
+
+ if (!failed) {
+ streamDataSubmitRefDec(pSubmit);
+ return 0;
+ } else {
+ return -1;
+ }
+}
+
+int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) {
+ SStreamTaskExecReq req = {0};
+ tDecodeSStreamTaskExecReq(msg, &req);
+ int32_t taskId = req.taskId;
+
+ SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+ ASSERT(pTask);
+ ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK);
+
+ // enqueue
+ int32_t inputStatus = streamEnqueueDataBlk(pTask, (SStreamDataBlock*)req.data);
+ if (inputStatus == TASK_INPUT_STATUS__BLOCKED) {
+ // TODO rsp blocked
+ return 0;
+ }
+
+ // try exec
+ int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
+ if (execStatus == TASK_STATUS__IDLE) {
+ if (streamTaskRun(pTask) < 0) {
+ atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
+
+ goto FAIL;
+ }
+ } else if (execStatus == TASK_STATUS__EXECUTING) {
+ return 0;
+ }
+
+ // TODO rsp success
+ return 0;
+FAIL:
+ return -1;
+}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index 585ef63531..844cfc094b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -16,6 +16,8 @@
#include "tsdb.h"
int tsdbBegin(STsdb *pTsdb) {
+ if (!pTsdb) return 0;
+
STsdbMemTable *pMem;
if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c
index 52b466d0f6..6dfd73158e 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS.c
@@ -37,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
// static int tsdbProcessExpiredFS(STsdb *pRepo);
// static int tsdbCreateMeta(STsdb *pRepo);
-static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) {
- snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]);
+static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) {
+ snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir);
}
-static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) {
- snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]);
+static void tsdbGetDataDir(int repoid, const char* dir, char dirName[]) {
+ snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir);
}
// For backward compatibility
@@ -591,7 +591,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
- TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]);
+ pRepo->dir, tsdbTxnFname[ftype]);
}
static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
@@ -721,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) {
STsdbFS *pfs = REPO_FS(pRepo);
const STfsFile *pf;
- tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir);
+ tsdbGetRootDir(REPO_ID(pRepo), pRepo->dir, rootDir);
STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir);
if (tdir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
@@ -755,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) {
STsdbFS *pfs = REPO_FS(pRepo);
const STfsFile *pf;
- tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir);
+ tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir);
STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir);
if (tdir == NULL) {
tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno));
@@ -803,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
regex_t regex;
STsdbFS *pfs = REPO_FS(pRepo);
- tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir);
+ tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir);
// Resource allocation and init
regcomp(®ex, pattern, REG_EXTENDED);
diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c
index 7f024786de..04be2a48de 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFile.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFile.c
@@ -23,14 +23,6 @@ static const char *TSDB_FNAME_SUFFIX[] = {
"smal", // TSDB_FILE_SMAL
"", // TSDB_FILE_MAX
"meta", // TSDB_FILE_META
- "tsma", // TSDB_FILE_TSMA
- "rsma", // TSDB_FILE_RSMA
-};
-
-const char *TSDB_LEVEL_DNAME[] = {
- "tsdb",
- "rsma1",
- "rsma2",
};
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname);
@@ -51,7 +43,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t
pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
pDFile->info.fver = tsdbGetDFSVersion(ftype);
- tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname);
+ tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, pRepo->dir, fname);
tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c
index 807ee95b03..8e689fc185 100644
--- a/source/dnode/vnode/src/tsdb/tsdbOpen.c
+++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c
@@ -15,100 +15,17 @@
#include "tsdb.h"
-#define TSDB_OPEN_RSMA_IMPL(v, l) \
- do { \
- SRetention *r = VND_RETENTIONS(v)[0]; \
- if (RETENTION_VALID(r)) { \
- return tsdbOpenImpl((v), type, &VND_RSMA##l(v), VNODE_RSMA##l##_DIR, TSDB_RETENTION_L##l); \
- } \
- } while (0)
+static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg);
-#define TSDB_SET_KEEP_CFG(l) \
- do { \
- SRetention *r = &pCfg->retentions[l]; \
- pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
- pKeepCfg->keep0 = pKeepCfg->keep2; \
- pKeepCfg->keep1 = pKeepCfg->keep2; \
- pKeepCfg->days = tsdbEvalDays(r, pCfg->precision); \
- } while (0)
-#define RETENTION_DAYS_SPLIT_RATIO 10
-#define RETENTION_DAYS_SPLIT_MIN 1
-#define RETENTION_DAYS_SPLIT_MAX 30
+// implementation
-static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type);
-static int32_t tsdbEvalDays(SRetention *r, int8_t precision);
-static int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level);
-
-int tsdbOpen(SVnode *pVnode, int8_t type) {
- switch (type) {
- case TSDB_TYPE_TSDB:
- return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0);
- case TSDB_TYPE_TSMA:
- ASSERT(0);
- break;
- case TSDB_TYPE_RSMA_L0:
- TSDB_OPEN_RSMA_IMPL(pVnode, 0);
- break;
- case TSDB_TYPE_RSMA_L1:
- TSDB_OPEN_RSMA_IMPL(pVnode, 1);
- break;
- case TSDB_TYPE_RSMA_L2:
- TSDB_OPEN_RSMA_IMPL(pVnode, 2);
- break;
- default:
- ASSERT(0);
- break;
- }
- return 0;
-}
-
-static int32_t tsdbEvalDays(SRetention *r, int8_t precision) {
- int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY);
- int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY);
-
- int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO;
- if (days <= RETENTION_DAYS_SPLIT_MIN) {
- days = RETENTION_DAYS_SPLIT_MIN;
- if (days < freqDays) {
- days = freqDays + 1;
- }
- } else {
- if (days > RETENTION_DAYS_SPLIT_MAX) {
- days = RETENTION_DAYS_SPLIT_MAX;
- }
- if (days < freqDays) {
- days = freqDays + 1;
- }
- }
- return days * 1440;
-}
-
-static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t type) {
+static int tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg) {
pKeepCfg->precision = pCfg->precision;
- switch (type) {
- case TSDB_TYPE_TSDB:
- pKeepCfg->days = pCfg->days;
- pKeepCfg->keep0 = pCfg->keep0;
- pKeepCfg->keep1 = pCfg->keep1;
- pKeepCfg->keep2 = pCfg->keep2;
- break;
- case TSDB_TYPE_TSMA:
- ASSERT(0);
- break;
- case TSDB_TYPE_RSMA_L0:
- TSDB_SET_KEEP_CFG(0);
- break;
- case TSDB_TYPE_RSMA_L1:
- TSDB_SET_KEEP_CFG(1);
- break;
- case TSDB_TYPE_RSMA_L2:
- TSDB_SET_KEEP_CFG(2);
- break;
- default:
- ASSERT(0);
- break;
- }
+ pKeepCfg->days = pCfg->days;
+ pKeepCfg->keep0 = pCfg->keep0;
+ pKeepCfg->keep1 = pCfg->keep1;
+ pKeepCfg->keep2 = pCfg->keep2;
return 0;
}
@@ -116,18 +33,16 @@ static int32_t tsdbSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int8_t typ
* @brief
*
* @param pVnode
- * @param type
* @param ppTsdb
* @param dir
- * @param level retention level
* @return int
*/
-int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) {
+int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg) {
STsdb *pTsdb = NULL;
int slen = 0;
*ppTsdb = NULL;
- slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3;
+ slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + TSDB_DATA_DIR_LEN + 3;
// create handle
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
@@ -136,13 +51,18 @@ int32_t tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *di
return -1;
}
+ ASSERT(strlen(dir) < TSDB_DATA_DIR_LEN);
+ memcpy(pTsdb->dir, dir, strlen(dir));
pTsdb->path = (char *)&pTsdb[1];
sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir);
pTsdb->pVnode = pVnode;
- pTsdb->level = level;
pTsdb->repoLocked = false;
taosThreadMutexInit(&pTsdb->mutex, NULL);
- tsdbSetKeepCfg(REPO_KEEP_CFG(pTsdb), REPO_CFG(pTsdb), type);
+ if (!pKeepCfg) {
+ tsdbSetKeepCfg(&pTsdb->keepCfg, &pVnode->config.tsdbCfg);
+ } else {
+ memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg));
+ }
pTsdb->fs = tsdbNewFS(REPO_KEEP_CFG(pTsdb));
// create dir (TODO: use tfsMkdir)
@@ -163,12 +83,13 @@ _err:
return -1;
}
-int tsdbClose(STsdb *pTsdb) {
- if (pTsdb) {
+int tsdbClose(STsdb **pTsdb) {
+ if (*pTsdb) {
// TODO: destroy mem/imem
- tsdbCloseFS(pTsdb);
- tsdbFreeFS(pTsdb->fs);
- taosMemoryFree(pTsdb);
+ taosThreadMutexDestroy(&(*pTsdb)->mutex);
+ tsdbCloseFS(*pTsdb);
+ tsdbFreeFS((*pTsdb)->fs);
+ taosMemoryFreeClear(*pTsdb);
}
return 0;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index e7bee3342a..b4fbd01c63 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -47,7 +47,7 @@ int vnodeBegin(SVnode *pVnode) {
}
// begin tsdb
- if (vnodeIsRollup(pVnode)) {
+ if (pVnode->pSma) {
if (tsdbBegin(VND_RSMA0(pVnode)) < 0) {
vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 7476da2a0f..d44e30988d 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -96,26 +96,15 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
}
// open tsdb
- if (vnodeIsRollup(pVnode)) {
- if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) {
- vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
- goto _err;
- }
+ if (!vnodeIsRollup(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_TYPE_TSDB) < 0) {
+ vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
+ }
- if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) {
- vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno));
- goto _err;
- }
-
- if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) {
- vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
- goto _err;
- }
- } else {
- if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) {
- vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
- goto _err;
- }
+ // open sma
+ if (smaOpen(pVnode)) {
+ vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
+ goto _err;
}
// open wal
@@ -161,10 +150,10 @@ _err:
if (pVnode->pQuery) vnodeQueryClose(pVnode);
if (pVnode->pTq) tqClose(pVnode->pTq);
if (pVnode->pWal) walClose(pVnode->pWal);
- if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb);
+ if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
if (pVnode->pMeta) metaClose(pVnode->pMeta);
- tsdbClose(VND_RSMA1(pVnode));
- tsdbClose(VND_RSMA2(pVnode));
+ if (pVnode->pSma) smaClose(pVnode->pSma);
+
tsem_destroy(&(pVnode->canCommit));
taosMemoryFree(pVnode);
return NULL;
@@ -177,9 +166,8 @@ void vnodeClose(SVnode *pVnode) {
vnodeQueryClose(pVnode);
walClose(pVnode->pWal);
tqClose(pVnode->pTq);
- tsdbClose(VND_TSDB(pVnode));
- tsdbClose(VND_RSMA1(pVnode));
- tsdbClose(VND_RSMA2(pVnode));
+ if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
+ smaClose(pVnode->pSma);
metaClose(pVnode->pMeta);
vnodeCloseBufPool(pVnode);
// destroy handle
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 4f76bc5386..f638ac056a 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -22,6 +22,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
#if 0
@@ -86,10 +87,8 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
case TDMT_VND_DROP_TABLE:
if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
- case TDMT_VND_CREATE_SMA: { // timeRangeSMA
- if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
- // TODO
- }
+ case TDMT_VND_CREATE_SMA: {
+ if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
} break;
/* TSDB */
case TDMT_VND_SUBMIT:
@@ -195,7 +194,7 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
// TODO
// blockDebugShowData(data);
- tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data);
+ tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data);
}
int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
@@ -305,7 +304,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq,
goto _err;
}
- tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &req, &pVnode->msgCb);
+ tdProcessRSmaCreate(pVnode->pSma, pVnode->pMeta, &req, &pVnode->msgCb);
tDecoderClear(&coder);
return 0;
@@ -366,7 +365,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
}
} else {
cRsp.code = TSDB_CODE_SUCCESS;
- tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
+ tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
}
taosArrayPush(rsp.pArray, &cRsp);
@@ -374,8 +373,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
tDecoderClear(&decoder);
- tsdbUpdateTbUidList(pVnode->pTsdb, pStore);
- tsdbUidStoreFree(pStore);
+ tdUpdateTbUidList(pVnode->pSma, pStore);
+ tdUidStoreFree(pStore);
// prepare rsp
SEncoder encoder = {0};
@@ -649,8 +648,38 @@ _exit:
// TODO: refactor
if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) &&
(pRsp->code == TSDB_CODE_SUCCESS)) {
- tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+ tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
}
return 0;
}
+
+static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
+ SVCreateTSmaReq req = {0};
+ SDecoder coder;
+
+ pRsp->msgType = TDMT_VND_CREATE_SMA_RSP;
+ pRsp->code = TSDB_CODE_SUCCESS;
+ pRsp->pCont = NULL;
+ pRsp->contLen = 0;
+
+ // decode and process req
+ tDecoderInit(&coder, pReq, len);
+
+ if (tDecodeSVCreateTSmaReq(&coder, &req) < 0) {
+ pRsp->code = terrno;
+ goto _err;
+ }
+
+ if (metaCreateTSma(pVnode->pMeta, version, &req) < 0) {
+ pRsp->code = terrno;
+ goto _err;
+ }
+
+ tDecoderClear(&coder);
+ return 0;
+
+_err:
+ tDecoderClear(&coder);
+ return -1;
+}
diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c
index 3cc75a815d..354f4d8752 100644
--- a/source/libs/executor/src/executorMain.c
+++ b/source/libs/executor/src/executorMain.c
@@ -21,13 +21,14 @@
#include "tcache.h"
#include "tglobal.h"
#include "tmsg.h"
+#include "tudf.h"
-#include "thash.h"
-#include "executorimpl.h"
#include "executor.h"
+#include "executorimpl.h"
+#include "query.h"
+#include "thash.h"
#include "tlosertree.h"
#include "ttypes.h"
-#include "query.h"
typedef struct STaskMgmt {
TdThreadMutex lock;
@@ -156,6 +157,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0;
pTaskInfo->totalRows += current;
+ cleanUpUdfs();
qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0);
diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index f2a0b4ec2c..4841e05267 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -25,8 +25,6 @@
#include "functionMgt.h"
//TODO: add unit test
-//TODO: include all global variable under context struct
-
typedef struct SUdfdData {
bool startCalled;
bool needCleanUp;
@@ -313,6 +311,7 @@ int64_t gUdfTaskSeqNum = 0;
typedef struct SUdfcFuncStub {
char udfName[TSDB_FUNC_NAME_LEN];
UdfcFuncHandle handle;
+ int32_t refCount;
} SUdfcFuncStub;
typedef struct SUdfcProxy {
@@ -338,7 +337,7 @@ typedef struct SUdfcProxy {
SUdfcProxy gUdfdProxy = {0};
-typedef struct SClientUdfUvSession {
+typedef struct SUdfcUvSession {
SUdfcProxy *udfc;
int64_t severHandle;
uv_pipe_t *udfUvPipe;
@@ -346,7 +345,9 @@ typedef struct SClientUdfUvSession {
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
-} SClientUdfUvSession;
+
+ char udfName[TSDB_FUNC_NAME_LEN];
+} SUdfcUvSession;
typedef struct SClientUvTaskNode {
SUdfcProxy *udfc;
@@ -369,7 +370,7 @@ typedef struct SClientUvTaskNode {
typedef struct SClientUdfTask {
int8_t type;
- SClientUdfUvSession *session;
+ SUdfcUvSession *session;
int32_t errCode;
@@ -401,7 +402,7 @@ typedef struct SClientUvConn {
uv_pipe_t *pipe;
QUEUE taskQueue;
SClientConnBuf readBuf;
- SClientUdfUvSession *session;
+ SUdfcUvSession *session;
} SClientUvConn;
enum {
@@ -1140,7 +1141,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
return code;
}
-void udfClientAsyncCb(uv_async_t *async) {
+void udfcAsyncTaskCb(uv_async_t *async) {
SUdfcProxy *udfc = async->data;
QUEUE wq;
@@ -1204,7 +1205,7 @@ void constructUdfService(void *argsThread) {
SUdfcProxy *udfc = (SUdfcProxy *)argsThread;
uv_loop_init(&udfc->uvLoop);
- uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfClientAsyncCb);
+ uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfcAsyncTaskCb);
udfc->loopTaskAync.data = udfc;
uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb);
udfc->loopStopAsync.data = udfc;
@@ -1272,13 +1273,12 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
}
int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
- fnInfo("udfc setup udf. udfName: %s", udfName);
if (gUdfdProxy.udfcState != UDFC_STATE_READY) {
return TSDB_CODE_UDF_INVALID_STATE;
}
SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask));
task->errCode = 0;
- task->session = taosMemoryCalloc(1, sizeof(SClientUdfUvSession));
+ task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession));
task->session->udfc = &gUdfdProxy;
task->type = UDF_TASK_SETUP;
@@ -1298,10 +1298,11 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
task->session->outputType = rsp->outputType;
task->session->outputLen = rsp->outputLen;
task->session->bufSize = rsp->bufSize;
+ strcpy(task->session->udfName, udfName);
if (task->errCode != 0) {
fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode)
} else {
- fnInfo("sucessfully setup udf func handle. handle: %p", task->session);
+ fnInfo("sucessfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session);
*funcHandle = task->session;
}
int32_t err = task->errCode;
@@ -1312,14 +1313,14 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
SSDataBlock* output, SUdfInterBuf *newState) {
fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
- SClientUdfUvSession *session = (SClientUdfUvSession *) handle;
+ SUdfcUvSession *session = (SUdfcUvSession *) handle;
if (session->udfUvPipe == NULL) {
fnError("No pipe to udfd");
return TSDB_CODE_UDF_PIPE_NO_PIPE;
}
SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
task->errCode = 0;
- task->session = (SClientUdfUvSession *) handle;
+ task->session = (SUdfcUvSession *) handle;
task->type = UDF_TASK_CALL;
SUdfCallRequest *req = &task->_call.req;
@@ -1435,7 +1436,7 @@ int compareUdfcFuncSub(const void* elem1, const void* elem2) {
return strcmp(stub1->udfName, stub2->udfName);
}
-int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
+int32_t accquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) {
int32_t code = 0;
uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
@@ -1444,6 +1445,7 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
if (foundStub != NULL) {
uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
*pHandle = foundStub->handle;
+ ++foundStub->refCount;
return 0;
}
*pHandle = NULL;
@@ -1452,6 +1454,7 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
SUdfcFuncStub stub = {0};
strcpy(stub.udfName, udfName);
stub.handle = *pHandle;
+ ++stub.refCount;
taosArrayPush(gUdfdProxy.udfStubs, &stub);
taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub);
} else {
@@ -1462,22 +1465,33 @@ int32_t setupUdf(char* udfName, UdfcFuncHandle* pHandle) {
return code;
}
+void releaseUdfFuncHandle(char* udfName) {
+ uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
+ SUdfcFuncStub key = {0};
+ strcpy(key.udfName, udfName);
+ SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
+ ASSERT(foundStub);
+ --foundStub->refCount;
+ ASSERT(foundStub->refCount>=0);
+ uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
+}
+
int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) {
UdfcFuncHandle handle = NULL;
- int32_t code = setupUdf(udfName, &handle);
+ int32_t code = accquireUdfFuncHandle(udfName, &handle);
if (code != 0) {
return code;
}
code = doCallUdfScalarFunc(handle, input, numOfCols, output);
+ releaseUdfFuncHandle(udfName);
return code;
}
int32_t doTeardownUdf(UdfcFuncHandle handle) {
- fnInfo("tear down udf. udf func handle: %p", handle);
+ SUdfcUvSession *session = (SUdfcUvSession *) handle;
- SClientUdfUvSession *session = (SClientUdfUvSession *) handle;
if (session->udfUvPipe == NULL) {
- fnError("pipe to udfd does not exist");
+ fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName);
return TSDB_CODE_UDF_PIPE_NO_PIPE;
}
@@ -1492,7 +1506,6 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
SUdfTeardownResponse *rsp = &task->_teardown.rsp;
-
int32_t err = task->errCode;
udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
@@ -1500,12 +1513,14 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
taosMemoryFree(task->session);
taosMemoryFree(task);
+ fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
+
return err;
}
//memory layout |---SUdfAggRes----|-----final result-----|---inter result----|
typedef struct SUdfAggRes {
- SClientUdfUvSession *session;
+ SUdfcUvSession *session;
int8_t finalResNum;
int8_t interResNum;
char* finalResBuf;
@@ -1526,11 +1541,11 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
}
UdfcFuncHandle handle;
int32_t udfCode = 0;
- if ((udfCode = setupUdf((char *)pCtx->udfName, &handle)) != 0) {
+ if ((udfCode = accquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode);
return false;
}
- SClientUdfUvSession *session = (SClientUdfUvSession *)handle;
+ SUdfcUvSession *session = (SUdfcUvSession *)handle;
SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo);
int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize;
memset(udfRes, 0, envSize);
@@ -1538,7 +1553,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
- udfRes->session = (SClientUdfUvSession *)handle;
+ udfRes->session = (SUdfcUvSession *)handle;
SUdfInterBuf buf = {0};
if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) {
fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode);
@@ -1554,7 +1569,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
int32_t numOfCols = pInput->numOfInputCols;
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
- SClientUdfUvSession *session = udfRes->session;
+ SUdfcUvSession *session = udfRes->session;
if (session == NULL) {
return TSDB_CODE_UDF_NO_FUNC_HANDLE;
}
@@ -1609,7 +1624,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
- SClientUdfUvSession *session = udfRes->session;
+ SUdfcUvSession *session = udfRes->session;
if (session == NULL) {
return TSDB_CODE_UDF_NO_FUNC_HANDLE;
}
@@ -1632,11 +1647,28 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
}
-// int32_t code = doTeardownUdf(session);
-// if (code != 0) {
-// fnError("udfAggFinalize error. doTeardownUdf step. udf code: %d", code);
-// }
-
int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf);
+ releaseUdfFuncHandle(pCtx->udfName);
return udfCallCode == 0 ? numOfResults : udfCallCode;
+}
+
+int32_t cleanUpUdfs() {
+ uv_mutex_lock(&gUdfdProxy.udfStubsMutex);
+ int32_t i = 0;
+ SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
+ while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) {
+ SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i);
+ if (stub->refCount == 0) {
+ fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle);
+ doTeardownUdf(stub->handle);
+ } else {
+ fnInfo("udf still in use. udf name: %s, ref count: %d, handle: %p", stub->udfName, stub->refCount, stub->handle);
+ taosArrayPush(udfStubs, stub);
+ }
+ ++i;
+ }
+ taosArrayDestroy(gUdfdProxy.udfStubs);
+ gUdfdProxy.udfStubs = udfStubs;
+ uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
+ return 0;
}
\ No newline at end of file
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 6427e13ae9..1639abceba 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -2949,6 +2949,8 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p
}
static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) {
+ SEncoder encoder = {0};
+ int32_t contLen = 0;
SVDropTSmaReq dropSmaReq = {0};
strcpy(dropSmaReq.indexName, pStmt->indexName);
@@ -2956,16 +2958,26 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt
if (NULL == pCxt->pCmdMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
+
+ int32_t ret = 0;
+ tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret);
+ if (ret < 0) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA;
- pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq);
+ pCxt->pCmdMsg->msgLen = contLen;
pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
if (NULL == pCxt->pCmdMsg->pMsg) {
return TSDB_CODE_OUT_OF_MEMORY;
}
void* pBuf = pCxt->pCmdMsg->pMsg;
- tSerializeSVDropTSmaReq(&pBuf, &dropSmaReq);
-
+ if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) {
+ tEncoderClear(&encoder);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+ tEncoderClear(&encoder);
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c
index 08093c8b18..812874dafb 100644
--- a/source/libs/stream/src/tstream.c
+++ b/source/libs/stream/src/tstream.c
@@ -16,6 +16,25 @@
#include "tstream.h"
#include "executor.h"
+int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
+ int32_t tlen = 0;
+ tlen += taosEncodeFixedI8(buf, pOutput->type);
+ tlen += taosEncodeFixedI32(buf, pOutput->sourceVg);
+ tlen += taosEncodeFixedI64(buf, pOutput->sourceVer);
+ ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK);
+ tlen += tEncodeDataBlocks(buf, pOutput->blocks);
+ return tlen;
+}
+
+void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
+ buf = taosDecodeFixedI8(buf, &pInput->type);
+ buf = taosDecodeFixedI32(buf, &pInput->sourceVg);
+ buf = taosDecodeFixedI64(buf, &pInput->sourceVer);
+ ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
+ buf = tDecodeDataBlocks(buf, &pInput->blocks);
+ return (void*)buf;
+}
+
static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
SStreamTaskExecReq req = {
.streamId = pTask->streamId,
@@ -97,6 +116,363 @@ static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashOb
return 0;
}
+int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input) {
+ ASSERT(pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK);
+ int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
+ if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
+ streamDataSubmitRefInc(input);
+ taosWriteQitem(pTask->inputQ, input);
+ }
+ return inputStatus;
+}
+
+int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
+ ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK);
+ taosWriteQitem(pTask->inputQ, input);
+ int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
+ return inputStatus;
+}
+
+int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
+ void* exec = pTask->exec.runners[0].executor;
+
+ // set input
+ if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+ SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
+ ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT);
+
+ qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+ } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) {
+ SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
+ ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
+
+ SArray* blocks = pBlock->blocks;
+ qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
+ }
+
+ // exec
+ while (1) {
+ SSDataBlock* output;
+ uint64_t ts = 0;
+ if (qExecTask(exec, &output, &ts) < 0) {
+ ASSERT(false);
+ }
+ if (output == NULL) break;
+ taosArrayPush(pRes, &output);
+ }
+
+ // destroy
+ if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+ streamDataSubmitRefDec((SStreamDataSubmit*)data);
+ } else {
+ taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
+ }
+ return 0;
+}
+
+// TODO: handle version
+int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
+ SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
+ if (pRes == NULL) return -1;
+ while (1) {
+ int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
+ void* exec = pTask->exec.runners[0].executor;
+ if (execStatus == TASK_STATUS__IDLE) {
+ // first run, from qall, handle failure from last exec
+ while (1) {
+ void* data = NULL;
+ taosGetQitem(pTask->inputQAll, &data);
+ if (data == NULL) break;
+
+ streamTaskExecImpl(pTask, data, pRes);
+
+ taosFreeQitem(data);
+
+ if (taosArrayGetSize(pRes) != 0) {
+ SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+ resQ->type = STREAM_INPUT__DATA_BLOCK;
+ resQ->blocks = pRes;
+ taosWriteQitem(pTask->outputQ, resQ);
+ pRes = taosArrayInit(0, sizeof(SSDataBlock));
+ if (pRes == NULL) goto FAIL;
+ }
+ }
+ // second run, from inputQ
+ taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
+ while (1) {
+ void* data = NULL;
+ taosGetQitem(pTask->inputQAll, &data);
+ if (data == NULL) break;
+
+ streamTaskExecImpl(pTask, data, pRes);
+
+ taosFreeQitem(data);
+
+ if (taosArrayGetSize(pRes) != 0) {
+ SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+ resQ->type = STREAM_INPUT__DATA_BLOCK;
+ resQ->blocks = pRes;
+ taosWriteQitem(pTask->outputQ, resQ);
+ pRes = taosArrayInit(0, sizeof(SSDataBlock));
+ if (pRes == NULL) goto FAIL;
+ }
+ }
+ // set status closing
+ atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
+ // third run, make sure all inputQ is cleared
+ taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
+ while (1) {
+ void* data = NULL;
+ taosGetQitem(pTask->inputQAll, &data);
+ if (data == NULL) break;
+
+ streamTaskExecImpl(pTask, data, pRes);
+
+ taosFreeQitem(data);
+
+ if (taosArrayGetSize(pRes) != 0) {
+ SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+ resQ->type = STREAM_INPUT__DATA_BLOCK;
+ resQ->blocks = pRes;
+ taosWriteQitem(pTask->outputQ, resQ);
+ pRes = taosArrayInit(0, sizeof(SSDataBlock));
+ if (pRes == NULL) goto FAIL;
+ }
+ }
+ // set status closing
+ atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
+ // third run, make sure all inputQ is cleared
+ taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
+ while (1) {
+ void* data = NULL;
+ taosGetQitem(pTask->inputQAll, &data);
+ if (data == NULL) break;
+ }
+
+ atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
+ break;
+ } else if (execStatus == TASK_STATUS__CLOSING) {
+ continue;
+ } else if (execStatus == TASK_STATUS__EXECUTING) {
+ break;
+ } else {
+ ASSERT(0);
+ }
+ }
+ return 0;
+FAIL:
+ atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
+ return -1;
+}
+
+int32_t streamTaskDispatchDown(SStreamTask* pTask, SMsgCb* pMsgCb) {
+ //
+ return 0;
+}
+
+int32_t streamTaskSink(SStreamTask* pTask) {
+ //
+ return 0;
+}
+
+int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDataBlock* pBlock, SRpcMsg* pRsp) {
+ // 1. handle input
+ // 1.1 enqueue
+ taosWriteQitem(pTask->inputQ, pBlock);
+ // 1.2 calc back pressure
+ // 1.3 rsp by input status
+ int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
+ SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
+ pCont->status = inputStatus;
+ pRsp->pCont = pCont;
+ pRsp->contLen = sizeof(SStreamDispatchRsp);
+ tmsgSendRsp(pRsp);
+ // 2. try exec
+ // 2.1. idle: exec
+ // 2.2. executing: return
+ // 2.3. closing: keep trying
+ while (1) {
+ int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
+ if (execStatus == TASK_STATUS__IDLE) {
+ void* exec = pTask->exec.runners[0].executor;
+ SArray* pRes = taosArrayInit(0, sizeof(void*));
+ const SArray* blocks = pBlock->blocks;
+ qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
+ while (1) {
+ SSDataBlock* output;
+ uint64_t ts = 0;
+ if (qExecTask(exec, &output, &ts) < 0) {
+ ASSERT(false);
+ }
+ if (output == NULL) break;
+ taosArrayPush(pRes, &output);
+ }
+ // TODO: wrap destroy block
+ taosArrayDestroyP(pBlock->blocks, (FDelete)blockDataDestroy);
+
+ if (taosArrayGetSize(pRes) != 0) {
+ SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+ *resQ = pRes;
+ taosWriteQitem(pTask->outputQ, resQ);
+ }
+
+ } else if (execStatus == TASK_STATUS__CLOSING) {
+ continue;
+ } else if (execStatus == TASK_STATUS__EXECUTING)
+ break;
+ else {
+ ASSERT(0);
+ }
+ }
+ // 3. handle output
+ // 3.1 check and set status
+ // 3.2 dispatch / sink
+ STaosQall* qall = taosAllocateQall();
+ taosReadAllQitems(pTask->outputQ, qall);
+ SArray** ppRes = NULL;
+ while (1) {
+ taosGetQitem(qall, (void**)&ppRes);
+ if (ppRes == NULL) break;
+
+ SArray* pRes = *ppRes;
+ if (pTask->sinkType == TASK_SINK__TABLE) {
+ pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, pBlock->sourceVer, pRes);
+ } else if (pTask->sinkType == TASK_SINK__SMA) {
+ pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
+ } else {
+ }
+
+ // dispatch
+ if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+ SRpcMsg dispatchMsg = {0};
+ if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
+ ASSERT(0);
+ return -1;
+ }
+
+ int32_t qType;
+ if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) {
+ qType = FETCH_QUEUE;
+ } else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||
+ pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {
+ qType = MERGE_QUEUE;
+ } else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {
+ qType = WRITE_QUEUE;
+ } else {
+ ASSERT(0);
+ }
+ tmsgPutToQueue(pMsgCb, qType, &dispatchMsg);
+
+ } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
+ SRpcMsg dispatchMsg = {0};
+ SEpSet* pEpSet = NULL;
+ if (streamBuildDispatchMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
+ ASSERT(0);
+ return -1;
+ }
+
+ tmsgSendReq(pMsgCb, pEpSet, &dispatchMsg);
+
+ } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
+ SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+ if (pShuffleRes == NULL) {
+ return -1;
+ }
+
+ int32_t sz = taosArrayGetSize(pRes);
+ for (int32_t i = 0; i < sz; i++) {
+ SSDataBlock* pDataBlock = taosArrayGet(pRes, i);
+ SArray* pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t));
+ if (pArray == NULL) {
+ pArray = taosArrayInit(0, sizeof(SSDataBlock));
+ if (pArray == NULL) {
+ return -1;
+ }
+ taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*));
+ }
+ taosArrayPush(pArray, pDataBlock);
+ }
+
+ if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) {
+ return -1;
+ }
+
+ } else {
+ ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
+ }
+ }
+ //
+ return 0;
+}
+
+int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, char* msg, int32_t msgLen) {
+ //
+ return 0;
+}
+
+int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) {
+ //
+ return 0;
+}
+
+int32_t streamTaskRun(SStreamTask* pTask) {
+ SArray* pRes = NULL;
+ if (pTask->execType == TASK_EXEC__PIPE || pTask->execType == TASK_EXEC__MERGE) {
+ // TODO remove multi runner
+ void* exec = pTask->exec.runners[0].executor;
+
+ int8_t status = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
+ if (status == TASK_STATUS__IDLE) {
+ pRes = taosArrayInit(0, sizeof(void*));
+ if (pRes == NULL) {
+ return -1;
+ }
+
+ void* input = NULL;
+ taosWriteQitem(pTask->inputQ, &input);
+ if (input == NULL) return 0;
+
+ // TODO: fix type
+ if (pTask->sourceType == TASK_SOURCE__SCAN) {
+ SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input;
+ qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+ while (1) {
+ SSDataBlock* output;
+ uint64_t ts = 0;
+ if (qExecTask(exec, &output, &ts) < 0) {
+ ASSERT(false);
+ }
+ if (output == NULL) break;
+ taosArrayPush(pRes, &output);
+ }
+ streamDataSubmitRefDec(pSubmit);
+ } else {
+ SStreamDataBlock* pStreamBlock = (SStreamDataBlock*)input;
+ const SArray* blocks = pStreamBlock->blocks;
+ qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
+ while (1) {
+ SSDataBlock* output;
+ uint64_t ts = 0;
+ if (qExecTask(exec, &output, &ts) < 0) {
+ ASSERT(false);
+ }
+ if (output == NULL) break;
+ taosArrayPush(pRes, &output);
+ }
+ // TODO: wrap destroy block
+ taosArrayDestroyP(pStreamBlock->blocks, (FDelete)blockDataDestroy);
+ }
+
+ if (taosArrayGetSize(pRes) != 0) {
+ SArray** resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM);
+ *resQ = pRes;
+ taosWriteQitem(pTask->outputQ, resQ);
+ }
+ }
+ }
+ return 0;
+}
+
int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) {
SArray* pRes = NULL;
// source
@@ -251,15 +627,29 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
}
pTask->taskId = tGenIdPI32();
pTask->streamId = streamId;
- pTask->status = STREAM_TASK_STATUS__RUNNING;
- /*pTask->qmsg = NULL;*/
+ pTask->status = TASK_STATUS__IDLE;
+
+ pTask->inputQ = taosOpenQueue();
+ pTask->outputQ = taosOpenQueue();
+ pTask->inputQAll = taosAllocateQall();
+ pTask->outputQAll = taosAllocateQall();
+ if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
+ goto FAIL;
return pTask;
+FAIL:
+ if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
+ if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
+ if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
+ if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
+ if (pTask) taosMemoryFree(pTask);
+ return NULL;
}
int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
/*if (tStartEncode(pEncoder) < 0) return -1;*/
if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
+ if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
@@ -305,6 +695,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
/*if (tStartDecode(pDecoder) < 0) return -1;*/
if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
+ if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
@@ -349,10 +740,16 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
}
void tFreeSStreamTask(SStreamTask* pTask) {
+ taosCloseQueue(pTask->inputQ);
+ taosCloseQueue(pTask->outputQ);
// TODO
- /*taosMemoryFree(pTask->qmsg);*/
+ if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
+ for (int32_t i = 0; i < pTask->exec.numOfRunners; i++) {
+ qDestroyTask(pTask->exec.runners[i].executor);
+ }
+ taosMemoryFree(pTask->exec.runners);
/*taosMemoryFree(pTask->executor);*/
- /*taosMemoryFree(pTask);*/
+ taosMemoryFree(pTask);
}
#if 0
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index a2b0648a4b..8a2a60d3e2 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -354,6 +354,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META, "No sma index in meta")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT, "Invalid sma state")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST, "Tsma already exists")
// query
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index 8531f2c8ea..9970ac24d7 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -94,6 +94,7 @@ int32_t tqDebugFlag = 135;
int32_t fsDebugFlag = 135;
int32_t metaDebugFlag = 135;
int32_t fnDebugFlag = 135;
+int32_t smaDebugFlag = 135;
int64_t dbgEmptyW = 0;
int64_t dbgWN = 0;
@@ -755,6 +756,7 @@ void taosSetAllDebugFlag(int32_t flag) {
tqDebugFlag = flag;
fsDebugFlag = flag;
fnDebugFlag = flag;
+ smaDebugFlag = flag;
uInfo("all debug flag are set to %d", flag);
}
diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim
index 3bb5943b3b..37f9cb94c9 100644
--- a/tests/script/tsim/tstream/basic1.sim
+++ b/tests/script/tsim/tstream/basic1.sim
@@ -136,7 +136,7 @@ if $data35 != 3 then
endi
sql insert into t1 values(1648791223001,12,14,13,11.1);
-sleep 100
+sleep 500
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt;
if $rows != 4 then
diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py
new file mode 100644
index 0000000000..0a998aee2b
--- /dev/null
+++ b/tests/system-test/0-others/udfTest.py
@@ -0,0 +1,544 @@
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+import subprocess
+
+class TDTestCase:
+
+ def init(self, conn, logSql):
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+
+ def getBuildPath(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+
+ for root, dirs, files in os.walk(projPath):
+ if ("taosd" in files):
+ rootRealPath = os.path.dirname(os.path.realpath(root))
+ if ("packaging" not in rootRealPath):
+ buildPath = root[:len(root) - len("/build/bin")]
+ break
+ return buildPath
+
+ def prepare_udf_so(self):
+ selfPath = os.path.dirname(os.path.realpath(__file__))
+
+ if ("community" in selfPath):
+ projPath = selfPath[:selfPath.find("community")]
+ else:
+ projPath = selfPath[:selfPath.find("tests")]
+ print(projPath)
+
+ libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+ libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8")
+ os.system("mkdir /tmp/udf/")
+ os.system("sudo cp %s /tmp/udf/ "%libudf1.replace("\n" ,""))
+ os.system("sudo cp %s /tmp/udf/ "%libudf2.replace("\n" ,""))
+
+
+ def prepare_data(self):
+
+ tdSql.execute("use db")
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+ for i in range(9):
+ tdSql.execute(
+ f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+ )
+ tdSql.execute(
+ f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+ )
+ tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
+ tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+ tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
+ tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+
+ tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+ tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+ tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+
+ tdSql.execute(
+ f'''insert into t1 values
+ ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
+ ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
+ ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
+ ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
+ ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
+ ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
+ ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
+ ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
+ ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
+ ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ '''
+ )
+
+ tdSql.execute("create table tb (ts timestamp , num1 int , num2 int, num3 double , num4 binary(30))")
+ tdSql.execute(
+ f'''insert into tb values
+ ( '2020-04-21 01:01:01.000', NULL, 1, 1, "binary1" )
+ ( '2020-10-21 01:01:01.000', 1, 1, 1.11, "binary1" )
+ ( '2020-12-31 01:01:01.000', 2, 22222, 22, "binary1" )
+ ( '2021-01-01 01:01:06.000', 3, 33333, 33, "binary1" )
+ ( '2021-05-07 01:01:10.000', 4, 44444, 44, "binary1" )
+ ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, "binary1" )
+ ( '2021-09-30 01:01:16.000', 5, 55555, 55, "binary1" )
+ ( '2022-02-01 01:01:20.000', 6, 66666, 66, "binary1" )
+ ( '2022-10-28 01:01:26.000', 0, 00000, 00, "binary1" )
+ ( '2022-12-01 01:01:30.000', 8, -88888, -88, "binary1" )
+ ( '2022-12-31 01:01:36.000', 9, -9999999, -99, "binary1" )
+ ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, "binary1" )
+ '''
+ )
+
+
+ def create_udf_function(self):
+
+ for i in range(10):
+ # create scalar functions
+ tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+
+ # create aggregate functions
+
+ tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+
+ functions = tdSql.getResult("show functions")
+ function_nums = len(functions)
+ if function_nums == 2:
+ tdLog.info("create two udf functions success ")
+
+ # drop functions
+
+ tdSql.execute("drop function udf1")
+ tdSql.execute("drop function udf2")
+
+ functions = tdSql.getResult("show functions")
+ for function in functions:
+ if "udf1" in function[0] or "udf2" in function[0]:
+ tdLog.info("drop udf functions failed ")
+ tdLog.exit("drop udf functions failed")
+
+ tdLog.info("drop two udf functions success ")
+
+ # create scalar functions
+ tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;")
+
+ # create aggregate functions
+
+ tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;")
+
+ functions = tdSql.getResult("show functions")
+ function_nums = len(functions)
+ if function_nums == 2:
+ tdLog.info("create two udf functions success ")
+
+ def basic_udf_query(self):
+
+ # scalar functions
+
+ tdSql.execute("use db ")
+ tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb")
+ tdSql.checkData(0,0,None)
+ tdSql.checkData(0,1,None)
+ tdSql.checkData(0,2,1)
+ tdSql.checkData(0,3,88)
+ tdSql.checkData(0,4,1.000000000)
+ tdSql.checkData(0,5,88)
+ tdSql.checkData(0,6,"binary1")
+ tdSql.checkData(0,7,88)
+
+ tdSql.checkData(3,0,3)
+ tdSql.checkData(3,1,88)
+ tdSql.checkData(3,2,33333)
+ tdSql.checkData(3,3,88)
+ tdSql.checkData(3,4,33.000000000)
+ tdSql.checkData(3,5,88)
+ tdSql.checkData(3,6,"binary1")
+ tdSql.checkData(3,7,88)
+
+ tdSql.checkData(11,0,None)
+ tdSql.checkData(11,1,None)
+ tdSql.checkData(11,2,None)
+ tdSql.checkData(11,3,None)
+ tdSql.checkData(11,4,None)
+ tdSql.checkData(11,5,None)
+ tdSql.checkData(11,6,"binary1")
+ tdSql.checkData(11,7,88)
+
+ tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1")
+ tdSql.checkData(0,0,None)
+ tdSql.checkData(0,1,None)
+ tdSql.checkData(0,2,None)
+ tdSql.checkData(0,3,None)
+ tdSql.checkData(0,4,None)
+ tdSql.checkData(0,5,None)
+ tdSql.checkData(0,6,None)
+ tdSql.checkData(0,7,None)
+
+ tdSql.checkData(20,0,8)
+ tdSql.checkData(20,1,88)
+ tdSql.checkData(20,2,88888)
+ tdSql.checkData(20,3,88)
+ tdSql.checkData(20,4,888)
+ tdSql.checkData(20,5,88)
+ tdSql.checkData(20,6,88)
+ tdSql.checkData(20,7,88)
+
+
+ # aggregate functions
+ tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb")
+ tdSql.checkData(0,0,15.362291496)
+ tdSql.checkData(0,1,10000949.553189287)
+ tdSql.checkData(0,2,168.633425216)
+
+ # Arithmetic compute
+ tdSql.query("select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb")
+ tdSql.checkData(0,0,115.362291496)
+ tdSql.checkData(0,1,10000849.553189287)
+ tdSql.checkData(0,2,16863.342521576)
+ tdSql.checkData(0,3,1.686334252)
+
+ tdSql.query("select udf2(c1) ,udf2(c6) from stb1 ")
+ tdSql.checkData(0,0,25.514701644)
+ tdSql.checkData(0,1,265.247614504)
+
+ tdSql.query("select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 ")
+ tdSql.checkData(0,0,125.514701644)
+ tdSql.checkData(0,1,165.247614504)
+ tdSql.checkData(0,2,2551.470164435)
+ tdSql.checkData(0,3,2.652476145)
+
+ # # bug for crash when query sub table
+ tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1")
+ tdSql.checkData(0,0,378.215547010)
+ tdSql.checkData(0,1,353.808067460)
+ tdSql.checkData(0,2,2114.237451187)
+ tdSql.checkData(0,3,2.125468151)
+
+ tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ")
+ tdSql.checkData(0,0,490.358032462)
+ tdSql.checkData(0,1,400.460106627)
+ tdSql.checkData(0,2,2551.470164435)
+ tdSql.checkData(0,3,2.652476145)
+
+
+ # regular table with aggregate functions
+
+ tdSql.error("select udf1(num1) , count(num1) from tb;")
+ tdSql.error("select udf1(num1) , avg(num1) from tb;")
+ tdSql.error("select udf1(num1) , twa(num1) from tb;")
+ tdSql.error("select udf1(num1) , irate(num1) from tb;")
+ tdSql.error("select udf1(num1) , sum(num1) from tb;")
+ tdSql.error("select udf1(num1) , stddev(num1) from tb;")
+ tdSql.error("select udf1(num1) , mode(num1) from tb;")
+ tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;")
+ # stable
+ tdSql.error("select udf1(c1) , count(c1) from stb1;")
+ tdSql.error("select udf1(c1) , avg(c1) from stb1;")
+ tdSql.error("select udf1(c1) , twa(c1) from stb1;")
+ tdSql.error("select udf1(c1) , irate(c1) from stb1;")
+ tdSql.error("select udf1(c1) , sum(c1) from stb1;")
+ tdSql.error("select udf1(c1) , stddev(c1) from stb1;")
+ tdSql.error("select udf1(c1) , mode(c1) from stb1;")
+ tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;")
+
+ # regular table with select functions
+
+ tdSql.query("select udf1(num1) , max(num1) from tb;")
+ tdSql.checkRows(1)
+ tdSql.query("select floor(num1) , max(num1) from tb;")
+ tdSql.checkRows(1)
+ tdSql.query("select udf1(num1) , min(num1) from tb;")
+ tdSql.checkRows(1)
+ tdSql.query("select ceil(num1) , min(num1) from tb;")
+ tdSql.checkRows(1)
+ tdSql.error("select udf1(num1) , first(num1) from tb;")
+
+ tdSql.error("select abs(num1) , first(num1) from tb;")
+
+ tdSql.error("select udf1(num1) , last(num1) from tb;")
+
+ tdSql.error("select round(num1) , last(num1) from tb;")
+
+ tdSql.query("select udf1(num1) , top(num1,1) from tb;")
+ tdSql.checkRows(1)
+ tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
+ tdSql.checkRows(1)
+ tdSql.error("select udf1(num1) , last_row(num1) from tb;")
+
+ tdSql.error("select round(num1) , last_row(num1) from tb;")
+
+
+ # stable
+ tdSql.query("select udf1(c1) , max(c1) from stb1;")
+ tdSql.checkRows(1)
+ tdSql.query("select abs(c1) , max(c1) from stb1;")
+ tdSql.checkRows(1)
+ tdSql.query("select udf1(c1) , min(c1) from stb1;")
+ tdSql.checkRows(1)
+ tdSql.query("select floor(c1) , min(c1) from stb1;")
+ tdSql.checkRows(1)
+ tdSql.error("select udf1(c1) , first(c1) from stb1;")
+
+ tdSql.error("select udf1(c1) , last(c1) from stb1;")
+
+ tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;")
+ tdSql.checkRows(1)
+ tdSql.query("select abs(c1) , top(c1 ,1) from stb1;")
+ tdSql.checkRows(1)
+ tdSql.query("select udf1(c1) , bottom(c1,1) from stb1;")
+ tdSql.checkRows(1)
+ tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
+ tdSql.checkRows(1)
+
+ tdSql.error("select udf1(c1) , last_row(c1) from stb1;")
+ tdSql.error("select ceil(c1) , last_row(c1) from stb1;")
+
+ # regular table with compute functions
+
+ tdSql.query("select udf1(num1) , abs(num1) from tb;")
+ tdSql.checkRows(12)
+ tdSql.query("select floor(num1) , abs(num1) from tb;")
+ tdSql.checkRows(12)
+
+ # # bug need fix
+
+ tdSql.query("select udf1(num1) , csum(num1) from tb;")
+ tdSql.checkRows(9)
+ tdSql.query("select ceil(num1) , csum(num1) from tb;")
+ tdSql.checkRows(9)
+ tdSql.query("select udf1(c1) , csum(c1) from stb1;")
+ tdSql.checkRows(22)
+ tdSql.query("select floor(c1) , csum(c1) from stb1;")
+ tdSql.checkRows(22)
+
+ # stable with compute functions
+ tdSql.query("select udf1(c1) , abs(c1) from stb1;")
+ tdSql.checkRows(25)
+ tdSql.query("select abs(c1) , ceil(c1) from stb1;")
+ tdSql.checkRows(25)
+
+ # nest query
+ tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;")
+ tdSql.checkRows(25)
+ tdSql.checkData(0,0,None)
+ tdSql.checkData(0,1,None)
+ tdSql.checkData(1,0,88)
+ tdSql.checkData(1,1,8)
+
+ tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;")
+ tdSql.checkRows(13)
+ tdSql.checkData(0,0,88)
+ tdSql.checkData(0,1,8)
+ tdSql.checkData(1,0,88)
+ tdSql.checkData(1,1,7)
+
+ # bug fix for crash
+ # order by udf function result
+ for _ in range(50):
+ tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)")
+ print(tdSql.queryResult)
+
+ # udf functions with filter
+
+ tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 where c1 is null order by ts;")
+ tdSql.checkRows(3)
+ tdSql.checkData(0,0,None)
+ tdSql.checkData(0,1,None)
+
+ tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts")
+ tdSql.checkRows(3)
+ tdSql.checkData(0,0,9)
+ tdSql.checkData(0,1,88)
+ tdSql.checkData(0,2,-99.990000000)
+ tdSql.checkData(0,3,88)
+
+ # udf functions with join
+ ts_start = 1652517451000
+ tdSql.execute("create stable st (ts timestamp , c1 int , c2 int ,c3 double ,c4 double ) tags(ind int)")
+ tdSql.execute("create table sub1 using st tags(1)")
+ tdSql.execute("create table sub2 using st tags(2)")
+
+ for i in range(10):
+ ts = ts_start + i *1000
+ tdSql.execute(" insert into sub1 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0))
+ tdSql.execute(" insert into sub2 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0))
+
+ tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,0)
+ tdSql.checkData(0,1,0)
+ tdSql.checkData(1,0,1)
+ tdSql.checkData(1,1,10)
+
+ tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,88)
+ tdSql.checkData(0,1,88)
+ tdSql.checkData(1,0,88)
+ tdSql.checkData(1,1,88)
+
+ tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,0)
+ tdSql.checkData(0,1,88)
+ tdSql.checkData(0,2,0)
+ tdSql.checkData(0,3,88)
+ tdSql.checkData(1,0,1)
+ tdSql.checkData(1,1,88)
+ tdSql.checkData(1,2,10)
+ tdSql.checkData(1,3,88)
+
+ tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,16.881943016)
+ tdSql.checkData(0,1,168.819430161)
+ tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+
+ # udf functions with group by
+ tdSql.query("select udf1(c1) from ct1 group by c1")
+ tdSql.checkRows(10)
+ tdSql.query("select udf1(c1) from stb1 group by c1")
+ tdSql.checkRows(11)
+ tdSql.query("select c1,c2, udf1(c1,c2) from ct1 group by c1,c2")
+ tdSql.checkRows(10)
+ tdSql.query("select c1,c2, udf1(c1,c2) from stb1 group by c1,c2")
+ tdSql.checkRows(11)
+
+ tdSql.query("select udf2(c1) from ct1 group by c1")
+ tdSql.checkRows(10)
+ tdSql.query("select udf2(c1) from stb1 group by c1")
+ tdSql.checkRows(11)
+ tdSql.query("select c1,c2, udf2(c1,c6) from ct1 group by c1,c2")
+ tdSql.checkRows(10)
+ tdSql.query("select c1,c2, udf2(c1,c6) from stb1 group by c1,c2")
+ tdSql.checkRows(11)
+ tdSql.query("select udf2(c1) from stb1 group by udf1(c1)")
+ tdSql.checkRows(2)
+ tdSql.query("select udf2(c1) from stb1 group by floor(c1)")
+ tdSql.checkRows(11)
+
+ # udf mix with order by
+ tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)")
+ tdSql.checkRows(11)
+
+
+ def multi_cols_udf(self):
+ tdSql.query("select num1,num2,num3,udf1(num1,num2,num3) from tb")
+ tdSql.checkData(0,0,None)
+ tdSql.checkData(0,1,1)
+ tdSql.checkData(0,2,1.000000000)
+ tdSql.checkData(0,3,None)
+ tdSql.checkData(1,0,1)
+ tdSql.checkData(1,1,1)
+ tdSql.checkData(1,2,1.110000000)
+ tdSql.checkData(1,3,88)
+
+ tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts")
+ tdSql.checkData(1,0,8)
+ tdSql.checkData(1,1,88.880000000)
+ tdSql.checkData(1,2,88)
+
+ tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;")
+ tdSql.checkRows(22)
+
+ tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,169.661427555)
+ tdSql.checkData(0,1,169.661427555)
+
+
+ def unexpected_create(self):
+
+ tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+
+ def loop_kill_udfd(self):
+
+ buildPath = self.getBuildPath()
+ if (buildPath == ""):
+ tdLog.exit("taosd not found!")
+ else:
+ tdLog.info("taosd found in %s" % buildPath)
+
+ cfgPath = buildPath + "/../sim/dnode1/cfg"
+ udfdPath = buildPath +'/build/bin/udfd'
+
+ for i in range(5):
+
+ tdLog.info(" loop restart udfd %d_th" % i)
+
+ tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,169.661427555)
+ tdSql.checkData(0,1,169.661427555)
+ # stop udfd cmds
+ get_processID = "ps -ef | grep -w udfd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}'"
+ processID = subprocess.check_output(get_processID, shell=True).decode("utf-8")
+ stop_udfd = " kill -9 %s" % processID
+ os.system(stop_udfd)
+
+ time.sleep(2)
+
+ tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,169.661427555)
+ tdSql.checkData(0,1,169.661427555)
+
+ # # start udfd cmds
+ # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &"
+ # tdLog.info("start udfd : %s " % start_udfd)
+
+
+ def restart_taosd_query_udf(self):
+
+ for i in range(5):
+ time.sleep(5)
+ tdLog.info(" this is %d_th restart taosd " %i)
+ tdSql.execute("use db ")
+ tdSql.query("select count(*) from stb1")
+ tdSql.checkRows(1)
+ tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
+ tdSql.checkData(0,0,169.661427555)
+ tdSql.checkData(0,1,169.661427555)
+ tdDnodes.stop(1)
+ time.sleep(2)
+ tdDnodes.start(1)
+ time.sleep(5)
+
+
+
+ def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
+ tdSql.prepare()
+
+ self.prepare_udf_so()
+ self.prepare_data()
+ self.create_udf_function()
+ self.basic_udf_query()
+ self.loop_kill_udfd()
+ # self.restart_taosd_query_udf()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index c80206abbc..2b813f83ca 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -7,6 +7,7 @@ python3 ./test.py -f 0-others/taosShellError.py
python3 ./test.py -f 0-others/taosShellNetChk.py
python3 ./test.py -f 0-others/telemetry.py
python3 ./test.py -f 0-others/taosdMonitor.py
+python3 ./test.py -f 0-others/udfTest.py
python3 ./test.py -f 0-others/user_control.py