enhance: add taospyudf test and run it in CI

tests/parallel_test/run_case.sh

@@ -51,6 +51,9 @@ if [ $ent -eq 0 ]; then
     ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
     ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null
     ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null
+    ln -s /home/TDengine/include/common/taosdef.h /usr/include/taosdef.h 2>/dev/null
+    ln -s /home/TDengine/include/util/taoserror.h /usr/include/taoserror.h 2>/dev/null
+    ln -s /home/TDengine/include/libs/function/taosudf.h /usr/include/taosudf.h 2>/dev/null
     CONTAINER_TESTDIR=/home/TDengine
 else
     export PATH=$PATH:/home/TDinternal/debug/build/bin
@@ -58,6 +61,9 @@ else
     ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
     ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null
     ln -s /home/TDinternal/community/include/client/taos.h /usr/include/taos.h 2>/dev/null
+    ln -s /home/TDinternal/community/include/common/taosdef.h /usr/include/taosdef.h 2>/dev/null
+    ln -s /home/TDinternal/community/include/util/taoserror.h /usr/include/taoserror.h 2>/dev/null
+    ln -s /home/TDinternal/community/include/libs/function/taosudf.h /usr/include/taosudf.h 2>/dev/null
     CONTAINER_TESTDIR=/home/TDinternal/community
 fi
 mkdir -p /var/lib/taos/subscribe
@@ -75,6 +81,8 @@ pip3 list|grep taospy
 pip3 uninstall taospy -y
 pip3 install taospy==2.7.6
 
+pip3 install taospyudf
+
 $TIMEOUT_CMD $cmd
 RET=$?
 echo "cmd exit code: $RET"

tests/script/sh/l2norm.c

@@ -16,7 +16,7 @@ DLL_EXPORT int32_t l2norm_destroy() {
 DLL_EXPORT int32_t l2norm_start(SUdfInterBuf *buf) {
   *(int64_t*)(buf->buf) = 0;
   buf->bufLen = sizeof(double);
-  buf->numOfResult = 0;
+  buf->numOfResult = 1;
   return 0;
 }
 
@@ -58,20 +58,11 @@ DLL_EXPORT int32_t l2norm(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInte
 
   *(double*)(newInterBuf->buf) = sumSquares;
   newInterBuf->bufLen = sizeof(double);
-
-  if (interBuf->numOfResult == 0 && numNotNull == 0) {
-    newInterBuf->numOfResult = 0;
-  } else {
   newInterBuf->numOfResult = 1;
-  }
   return 0;
 }
 
 DLL_EXPORT int32_t l2norm_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) {
-  if (buf->numOfResult == 0) {
-    resultData->numOfResult = 0;
-    return 0;
-  }
   double sumSquares = *(double*)(buf->buf);
   *(double*)(resultData->buf) = sqrt(sumSquares);
   resultData->bufLen = sizeof(double);

tests/script/sh/pybitand.py

@@ -0,0 +1,21 @@
+def init():
+    pass
+
+def process(block):
+    (rows, cols) = block.shape()
+    result = []
+    for i in range(rows):
+        r = 2 ** 32 - 1
+        for j in range(cols):
+            cell = block.data(i,j)
+            if cell is None:
+                result.append(None)
+                break
+            else:
+                r = r & cell
+        else:
+            result.append(r)
+    return result
+
+def destroy():
+    pass

tests/script/sh/pyl2norm.py

@@ -0,0 +1,27 @@
+import json
+import math
+
+def init():
+    pass
+
+def destroy():
+    pass
+
+def start():
+    return json.dumps(0.0).encode('utf-8')
+
+def finish(buf):
+    sum_squares = json.loads(buf)
+    result = math.sqrt(sum_squares)
+    return result
+
+def reduce(datablock, buf):
+    (rows, cols) = datablock.shape()
+    sum_squares = json.loads(buf)
+    
+    for i in range(rows):
+        for j in range(cols):
+            cell = datablock.data(i,j)
+            if cell is not None:
+                sum_squares += cell * cell
+    return json.dumps(sum_squares).encode('utf-8') 

tests/script/tsim/query/udfpy.sim

@@ -0,0 +1,278 @@
+system_content printf %OS%
+if $system_content == Windows_NT then
+  return 0;
+endi
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c udf -v 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print ======== step1 udf
+system sh/compile_udf.sh
+system mkdir -p /tmp/pyudf
+system cp sh/pybitand.py /tmp/pyudf/
+system cp sh/pyl2norm.py /tmp/pyudf/
+system ls /tmp/pyudf
+
+sql create database udf vgroups 3;
+sql use udf;
+sql select * from information_schema.ins_databases;
+
+sql create table t (ts timestamp, f int);
+sql insert into t values(now, 1)(now+1s, 2);
+
+system_content printf %OS%
+if $system_content == Windows_NT then
+  return 0;
+endi
+if $system_content == Windows_NT then
+  sql create function bit_and as 'C:\\Windows\\Temp\\bitand.dll' outputtype int bufSize 8;
+  sql create aggregate function l2norm as 'C:\\Windows\\Temp\\l2norm.dll' outputtype double bufSize 8;
+else
+  sql create function bit_and as '/tmp/udf/libbitand.so' outputtype int bufSize 8;
+  sql create aggregate function l2norm as '/tmp/udf/libl2norm.so' outputtype double bufSize 8;
+endi
+sql create function pybitand as '/tmp/pyudf/pybitand.py' outputtype int language 'python';
+sql create aggregate function pyl2norm as '/tmp/pyudf/pyl2norm.py' outputtype double bufSize 128 language 'python';
+
+sql show functions;
+if $rows != 4 then
+  return -1
+endi
+sql select bit_and(f, f) from t;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 1 then
+  return -1
+endi
+if $data10 != 2 then
+  return -1
+endi
+
+sql select pybitand(f, f) from t;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 1 then
+  return -1
+endi
+if $data10 != 2 then
+  return -1
+endi
+
+sql select l2norm(f) from t;
+if $rows != 1 then
+  print expect 1, actual $rows
+  return -1
+endi
+if $data00 != 2.236067977 then
+  return -1
+endi
+
+sql select pyl2norm(f) from t;
+if $rows != 1 then
+  print expect 1, actual $rows
+  return -1
+endi
+if $data00 != 2.236067977 then
+  return -1
+endi
+
+sql create table t2 (ts timestamp, f1 int, f2 int);
+sql insert into t2 values(now, 0, 0)(now+1s, 1, 1);
+sql select bit_and(f1, f2) from t2;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 0 then
+  return -1
+endi
+if $data10 != 1 then
+  return -1
+endi
+
+sql select l2norm(f1, f2) from t2;
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 1.414213562 then
+  return -1
+endi
+
+sql select pybitand(f1, f2) from t2;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 0 then
+  return -1
+endi
+if $data10 != 1 then
+  return -1
+endi
+
+sql select pyl2norm(f1, f2) from t2;
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 1.414213562 then
+  return -1
+endi
+
+sql insert into t2 values(now+2s, 1, null)(now+3s, null, 2);
+sql select bit_and(f1, f2) from t2;
+print $rows , $data00 , $data10 , $data20 , $data30
+if $rows != 4 then
+  return -1
+endi
+if $data00 != 0 then
+  return -1
+endi
+if $data10 != 1 then
+  return -1
+endi
+
+if $data20 != NULL then
+  return -1
+endi
+
+if $data30 != NULL then
+  return -1
+endi
+
+sql select l2norm(f1, f2) from t2;
+print $rows, $data00
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 2.645751311 then
+  return -1
+endi
+
+sql select pybitand(f1, f2) from t2;
+print $rows , $data00 , $data10 , $data20 , $data30
+if $rows != 4 then
+  return -1
+endi
+if $data00 != 0 then
+  return -1
+endi
+if $data10 != 1 then
+  return -1
+endi
+
+if $data20 != NULL then
+  return -1
+endi
+
+if $data30 != NULL then
+  return -1
+endi
+
+sql select pyl2norm(f1, f2) from t2;
+print $rows, $data00
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 2.645751311 then
+  return -1
+endi
+
+
+sql insert into t2 values(now+4s, 4, 8)(now+5s, 5, 9);
+sql select l2norm(f1-f2), l2norm(f1+f2) from t2;
+print $rows , $data00 , $data01
+if $rows != 1 then
+  return -1;
+endi
+if $data00 != 5.656854249 then
+  return -1
+endi
+if $data01 != 18.547236991 then
+  return -1
+endi
+
+sql select l2norm(bit_and(f2, f1)), l2norm(bit_and(f1, f2)) from t2;
+print $rows , $data00 , $data01
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 1.414213562 then
+  return -1
+endi
+if $data01 != 1.414213562 then
+  return -1
+endi
+
+sql select l2norm(f2) from udf.t2 group by 1-bit_and(f1, f2) order by 1-bit_and(f1,f2);
+print $rows , $data00 , $data10 , $data20
+if $rows != 3 then
+  return -1
+endi
+if $data00 != 2.000000000 then
+  return -1
+endi
+if $data10 != 9.055385138 then
+  return -1
+endi
+if $data20 != 8.000000000 then
+  return -1
+endi
+
+sql select pyl2norm(f1-f2), pyl2norm(f1+f2) from t2;
+print $rows , $data00 , $data01
+if $rows != 1 then
+  return -1;
+endi
+if $data00 != 5.656854249 then
+  return -1
+endi
+if $data01 != 18.547236991 then
+  return -1
+endi
+
+sql select pyl2norm(pybitand(f2, f1)), pyl2norm(pybitand(f1, f2)) from t2;
+print $rows , $data00 , $data01
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 1.414213562 then
+  return -1
+endi
+if $data01 != 1.414213562 then
+  return -1
+endi
+
+sql select pyl2norm(f2) from udf.t2 group by 1-pybitand(f1, f2) order by 1-pybitand(f1,f2);
+print $rows , $data00 , $data10 , $data20
+if $rows != 3 then
+  return -1
+endi
+if $data00 != 2.000000000 then
+  return -1
+endi
+if $data10 != 9.055385138 then
+  return -1
+endi
+if $data20 != 8.000000000 then
+  return -1
+endi
+
+
+#sql drop function bit_and;
+#sql show functions;
+#if $rows != 1 then
+#  return -1
+#endi
+#if $data00 != @l2norm@ then
+#  return -1
+#  endi
+#sql drop function l2norm;
+#sql show functions;
+#if $rows != 0 then
+#  return -1
+#endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT