fix(gpt): 1. fix error in anomalywindow count; 2. update gpt model name; 3. update the expired test case. 4. support the user specified valid_code

source/libs/executor/src/anomalywindowoperator.c

@@ -593,7 +593,7 @@ static int32_t anomalyAggregateBlocks(SOperatorInfo* pOperator) {
 
     for (int32_t r = 0; r < pBlock->info.rows; ++r) {
       TSKEY key = tsList[r];
-      bool  keyInWin = (key >= pSupp->curWin.skey && key < pSupp->curWin.ekey);
+      bool  keyInWin = (key >= pSupp->curWin.skey && key <= pSupp->curWin.ekey);
       bool  lastRow = (r == pBlock->info.rows - 1);
 
       if (keyInWin) {

tests/script/tsim/analytics/basic0.sim

@@ -23,8 +23,8 @@ endi
 
 print =============== show info
 sql show anodes full
-if $rows != 8 then
+if $rows != 10 then
-  print expect 8 , actual $rows
+  print expect 10 , actual $rows
   return -1
 endi
 

tools/tdgpt/cfg/taosanode.ini

@@ -78,4 +78,4 @@ model-dir = /usr/local/taos/taosanode/model/
 log-level = DEBUG
 
 # draw the query results
-draw-result = 1
+draw-result = 0

tools/tdgpt/taosanalytics/algo/anomaly.py

@@ -5,6 +5,7 @@
 from matplotlib import pyplot as plt
 from taosanalytics.conf import app_logger, conf
 from taosanalytics.servicemgmt import loader
+from taosanalytics.util import convert_results_to_windows
 
 
 def do_ad_check(input_list, ts_list, algo_name, params):
@@ -22,17 +23,19 @@ def do_ad_check(input_list, ts_list, algo_name, params):
 
     res = s.execute()
 
-    n_error = abs(sum(filter(lambda x: x == -1, res)))
+    n_error = abs(sum(filter(lambda x: x != s.valid_code, res)))
     app_logger.log_inst.debug("There are %d in input, and %d anomaly points found: %s",
                               len(input_list),
                               n_error,
                               res)
 
-    draw_ad_results(input_list, res, algo_name)
+    # draw_ad_results(input_list, res, algo_name, s.valid_code)
-    return res
+
+    ano_window = convert_results_to_windows(res, ts_list, s.valid_code)
+    return res, ano_window
 
 
-def draw_ad_results(input_list, res, fig_name):
+def draw_ad_results(input_list, res, fig_name, valid_code):
     """ draw the detected anomaly points """
 
     # not in debug, do not visualize the anomaly detection result
@@ -41,9 +44,8 @@ def draw_ad_results(input_list, res, fig_name):
 
     plt.clf()
     for index, val in enumerate(res):
-        if val != -1:
+        if val != valid_code:
-            continue
+            plt.scatter(index, input_list[index], marker='o', color='r', alpha=0.5, s=100, zorder=3)
-        plt.scatter(index, input_list[index], marker='o', color='r', alpha=0.5, s=100, zorder=3)
 
     plt.plot(input_list, label='sample')
     plt.savefig(fig_name)

tools/tdgpt/taosanalytics/algo/fc/gpt.py

@@ -10,7 +10,7 @@ from taosanalytics.service import AbstractForecastService
 
 
 class _GPTService(AbstractForecastService):
-    name = 'td_gpt_fc'
+    name = 'TDtsfm_1'
     desc = "internal gpt forecast model based on transformer"
 
     def __init__(self):
@@ -23,7 +23,6 @@ class _GPTService(AbstractForecastService):
         self.std = None
         self.threshold = None
         self.time_interval = None
-        self.dir = 'internal-gpt'
 
 
     def execute(self):

tools/tdgpt/taosanalytics/app.py

@@ -91,9 +91,7 @@ def handle_ad_request():
 
     # 4. do anomaly detection
     try:
-        res_list = do_ad_check(payload[data_index], payload[ts_index], algo, params)
+        res_list, ano_window = do_ad_check(payload[data_index], payload[ts_index], algo, params)
-        ano_window = convert_results_to_windows(res_list, payload[ts_index])
-
         result = {"algo": algo, "option": options, "res": ano_window, "rows": len(ano_window)}
         app_logger.log_inst.debug("anomaly-detection result: %s", str(result))
 

tools/tdgpt/taosanalytics/service.py

@@ -51,6 +51,7 @@ class AbstractAnomalyDetectionService(AbstractAnalyticsService, ABC):
      inherent from this class"""
 
     def __init__(self):
+        self.valid_code = 1
         super().__init__()
         self.type = "anomaly-detection"
 
@@ -58,6 +59,12 @@ class AbstractAnomalyDetectionService(AbstractAnalyticsService, ABC):
         """ check if the input list is empty or None """
         return (self.list is None) or (len(self.list) == 0)
 
+    def set_params(self, params: dict) -> None:
+        super().set_params(params)
+
+        if "valid_code" in params:
+            self.valid_code = int(params["valid_code"])
+
 
 class AbstractForecastService(AbstractAnalyticsService, ABC):
     """abstract forecast service, all forecast algorithms class should be inherent from

tools/tdgpt/taosanalytics/test/anomaly_test.py

@@ -44,7 +44,7 @@ class AnomalyDetectionTest(unittest.TestCase):
         s.set_params({"k": 2})
 
         r = s.execute()
-        draw_ad_results(AnomalyDetectionTest.input_list, r, "ksigma")
+        draw_ad_results(AnomalyDetectionTest.input_list, r, "ksigma", s.valid_code)
 
         self.assertEqual(r[-1], -1)
         self.assertEqual(len(r), len(AnomalyDetectionTest.input_list))
@@ -64,7 +64,7 @@ class AnomalyDetectionTest(unittest.TestCase):
             self.assertEqual(1, 0, e)
 
         r = s.execute()
-        draw_ad_results(AnomalyDetectionTest.input_list, r, "iqr")
+        draw_ad_results(AnomalyDetectionTest.input_list, r, "iqr", s.valid_code)
 
         self.assertEqual(r[-1], -1)
         self.assertEqual(len(r), len(AnomalyDetectionTest.input_list))
@@ -82,7 +82,7 @@ class AnomalyDetectionTest(unittest.TestCase):
         s.set_params({"alpha": 0.95})
 
         r = s.execute()
-        draw_ad_results(AnomalyDetectionTest.input_list, r, "grubbs")
+        draw_ad_results(AnomalyDetectionTest.input_list, r, "grubbs", s.valid_code)
 
         self.assertEqual(r[-1], -1)
         self.assertEqual(len(r), len(AnomalyDetectionTest.input_list))
@@ -100,7 +100,7 @@ class AnomalyDetectionTest(unittest.TestCase):
         s.set_input_list(AnomalyDetectionTest.input_list, None)
 
         r = s.execute()
-        draw_ad_results(AnomalyDetectionTest.input_list, r, "shesd")
+        draw_ad_results(AnomalyDetectionTest.input_list, r, "shesd", s.valid_code)
 
         self.assertEqual(r[-1], -1)
 
@@ -116,7 +116,7 @@ class AnomalyDetectionTest(unittest.TestCase):
         s.set_input_list(AnomalyDetectionTest.input_list, None)
 
         r = s.execute()
-        draw_ad_results(AnomalyDetectionTest.input_list, r, "lof")
+        draw_ad_results(AnomalyDetectionTest.input_list, r, "lof", s.valid_code)
 
         self.assertEqual(r[-1], -1)
         self.assertEqual(r[-2], -1)

tools/tdgpt/taosanalytics/util.py

@@ -36,7 +36,7 @@ def validate_pay_load(json_obj):
         raise ValueError('invalid schema info, data column is missing')
 
 
-def convert_results_to_windows(result, ts_list):
+def convert_results_to_windows(result, ts_list, valid_code):
     """generate the window according to anomaly detection result"""
     skey, ekey = -1, -1
     wins = []
@@ -45,7 +45,7 @@ def convert_results_to_windows(result, ts_list):
         return wins
 
     for index, val in enumerate(result):
-        if val == -1:
+        if val != valid_code:
             ekey = ts_list[index]
             if skey == -1:
                 skey = ts_list[index]