enhance: add taospyudf test and run it in CI

This commit is contained in:
slzhou 2023-03-16 19:06:18 +08:00
parent 40b9cd21eb
commit e71df1aca6
6 changed files with 337 additions and 1115 deletions

File diff suppressed because it is too large Load Diff

View File

@ -51,6 +51,9 @@ if [ $ent -eq 0 ]; then
ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null
ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null
ln -s /home/TDengine/include/common/taosdef.h /usr/include/taosdef.h 2>/dev/null
ln -s /home/TDengine/include/util/taoserror.h /usr/include/taoserror.h 2>/dev/null
ln -s /home/TDengine/include/libs/function/taosudf.h /usr/include/taosudf.h 2>/dev/null
CONTAINER_TESTDIR=/home/TDengine
else
export PATH=$PATH:/home/TDinternal/debug/build/bin
@ -58,6 +61,9 @@ else
ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null
ln -s /home/TDinternal/community/include/client/taos.h /usr/include/taos.h 2>/dev/null
ln -s /home/TDinternal/community/include/common/taosdef.h /usr/include/taosdef.h 2>/dev/null
ln -s /home/TDinternal/community/include/util/taoserror.h /usr/include/taoserror.h 2>/dev/null
ln -s /home/TDinternal/community/include/libs/function/taosudf.h /usr/include/taosudf.h 2>/dev/null
CONTAINER_TESTDIR=/home/TDinternal/community
fi
mkdir -p /var/lib/taos/subscribe
@ -75,6 +81,8 @@ pip3 list|grep taospy
pip3 uninstall taospy -y
pip3 install taospy==2.7.6
pip3 install taospyudf
$TIMEOUT_CMD $cmd
RET=$?
echo "cmd exit code: $RET"

View File

@ -16,7 +16,7 @@ DLL_EXPORT int32_t l2norm_destroy() {
DLL_EXPORT int32_t l2norm_start(SUdfInterBuf *buf) {
*(int64_t*)(buf->buf) = 0;
buf->bufLen = sizeof(double);
buf->numOfResult = 0;
buf->numOfResult = 1;
return 0;
}
@ -58,20 +58,11 @@ DLL_EXPORT int32_t l2norm(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInte
*(double*)(newInterBuf->buf) = sumSquares;
newInterBuf->bufLen = sizeof(double);
if (interBuf->numOfResult == 0 && numNotNull == 0) {
newInterBuf->numOfResult = 0;
} else {
newInterBuf->numOfResult = 1;
}
newInterBuf->numOfResult = 1;
return 0;
}
DLL_EXPORT int32_t l2norm_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) {
if (buf->numOfResult == 0) {
resultData->numOfResult = 0;
return 0;
}
double sumSquares = *(double*)(buf->buf);
*(double*)(resultData->buf) = sqrt(sumSquares);
resultData->bufLen = sizeof(double);

View File

@ -0,0 +1,21 @@
def init():
pass
def process(block):
(rows, cols) = block.shape()
result = []
for i in range(rows):
r = 2 ** 32 - 1
for j in range(cols):
cell = block.data(i,j)
if cell is None:
result.append(None)
break
else:
r = r & cell
else:
result.append(r)
return result
def destroy():
pass

View File

@ -0,0 +1,27 @@
import json
import math
def init():
pass
def destroy():
pass
def start():
return json.dumps(0.0).encode('utf-8')
def finish(buf):
sum_squares = json.loads(buf)
result = math.sqrt(sum_squares)
return result
def reduce(datablock, buf):
(rows, cols) = datablock.shape()
sum_squares = json.loads(buf)
for i in range(rows):
for j in range(cols):
cell = datablock.data(i,j)
if cell is not None:
sum_squares += cell * cell
return json.dumps(sum_squares).encode('utf-8')

View File

@ -0,0 +1,278 @@
system_content printf %OS%
if $system_content == Windows_NT then
return 0;
endi
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c udf -v 1
system sh/exec.sh -n dnode1 -s start
sql connect
print ======== step1 udf
system sh/compile_udf.sh
system mkdir -p /tmp/pyudf
system cp sh/pybitand.py /tmp/pyudf/
system cp sh/pyl2norm.py /tmp/pyudf/
system ls /tmp/pyudf
sql create database udf vgroups 3;
sql use udf;
sql select * from information_schema.ins_databases;
sql create table t (ts timestamp, f int);
sql insert into t values(now, 1)(now+1s, 2);
system_content printf %OS%
if $system_content == Windows_NT then
return 0;
endi
if $system_content == Windows_NT then
sql create function bit_and as 'C:\\Windows\\Temp\\bitand.dll' outputtype int bufSize 8;
sql create aggregate function l2norm as 'C:\\Windows\\Temp\\l2norm.dll' outputtype double bufSize 8;
else
sql create function bit_and as '/tmp/udf/libbitand.so' outputtype int bufSize 8;
sql create aggregate function l2norm as '/tmp/udf/libl2norm.so' outputtype double bufSize 8;
endi
sql create function pybitand as '/tmp/pyudf/pybitand.py' outputtype int language 'python';
sql create aggregate function pyl2norm as '/tmp/pyudf/pyl2norm.py' outputtype double bufSize 128 language 'python';
sql show functions;
if $rows != 4 then
return -1
endi
sql select bit_and(f, f) from t;
if $rows != 2 then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data10 != 2 then
return -1
endi
sql select pybitand(f, f) from t;
if $rows != 2 then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data10 != 2 then
return -1
endi
sql select l2norm(f) from t;
if $rows != 1 then
print expect 1, actual $rows
return -1
endi
if $data00 != 2.236067977 then
return -1
endi
sql select pyl2norm(f) from t;
if $rows != 1 then
print expect 1, actual $rows
return -1
endi
if $data00 != 2.236067977 then
return -1
endi
sql create table t2 (ts timestamp, f1 int, f2 int);
sql insert into t2 values(now, 0, 0)(now+1s, 1, 1);
sql select bit_and(f1, f2) from t2;
if $rows != 2 then
return -1
endi
if $data00 != 0 then
return -1
endi
if $data10 != 1 then
return -1
endi
sql select l2norm(f1, f2) from t2;
if $rows != 1 then
return -1
endi
if $data00 != 1.414213562 then
return -1
endi
sql select pybitand(f1, f2) from t2;
if $rows != 2 then
return -1
endi
if $data00 != 0 then
return -1
endi
if $data10 != 1 then
return -1
endi
sql select pyl2norm(f1, f2) from t2;
if $rows != 1 then
return -1
endi
if $data00 != 1.414213562 then
return -1
endi
sql insert into t2 values(now+2s, 1, null)(now+3s, null, 2);
sql select bit_and(f1, f2) from t2;
print $rows , $data00 , $data10 , $data20 , $data30
if $rows != 4 then
return -1
endi
if $data00 != 0 then
return -1
endi
if $data10 != 1 then
return -1
endi
if $data20 != NULL then
return -1
endi
if $data30 != NULL then
return -1
endi
sql select l2norm(f1, f2) from t2;
print $rows, $data00
if $rows != 1 then
return -1
endi
if $data00 != 2.645751311 then
return -1
endi
sql select pybitand(f1, f2) from t2;
print $rows , $data00 , $data10 , $data20 , $data30
if $rows != 4 then
return -1
endi
if $data00 != 0 then
return -1
endi
if $data10 != 1 then
return -1
endi
if $data20 != NULL then
return -1
endi
if $data30 != NULL then
return -1
endi
sql select pyl2norm(f1, f2) from t2;
print $rows, $data00
if $rows != 1 then
return -1
endi
if $data00 != 2.645751311 then
return -1
endi
sql insert into t2 values(now+4s, 4, 8)(now+5s, 5, 9);
sql select l2norm(f1-f2), l2norm(f1+f2) from t2;
print $rows , $data00 , $data01
if $rows != 1 then
return -1;
endi
if $data00 != 5.656854249 then
return -1
endi
if $data01 != 18.547236991 then
return -1
endi
sql select l2norm(bit_and(f2, f1)), l2norm(bit_and(f1, f2)) from t2;
print $rows , $data00 , $data01
if $rows != 1 then
return -1
endi
if $data00 != 1.414213562 then
return -1
endi
if $data01 != 1.414213562 then
return -1
endi
sql select l2norm(f2) from udf.t2 group by 1-bit_and(f1, f2) order by 1-bit_and(f1,f2);
print $rows , $data00 , $data10 , $data20
if $rows != 3 then
return -1
endi
if $data00 != 2.000000000 then
return -1
endi
if $data10 != 9.055385138 then
return -1
endi
if $data20 != 8.000000000 then
return -1
endi
sql select pyl2norm(f1-f2), pyl2norm(f1+f2) from t2;
print $rows , $data00 , $data01
if $rows != 1 then
return -1;
endi
if $data00 != 5.656854249 then
return -1
endi
if $data01 != 18.547236991 then
return -1
endi
sql select pyl2norm(pybitand(f2, f1)), pyl2norm(pybitand(f1, f2)) from t2;
print $rows , $data00 , $data01
if $rows != 1 then
return -1
endi
if $data00 != 1.414213562 then
return -1
endi
if $data01 != 1.414213562 then
return -1
endi
sql select pyl2norm(f2) from udf.t2 group by 1-pybitand(f1, f2) order by 1-pybitand(f1,f2);
print $rows , $data00 , $data10 , $data20
if $rows != 3 then
return -1
endi
if $data00 != 2.000000000 then
return -1
endi
if $data10 != 9.055385138 then
return -1
endi
if $data20 != 8.000000000 then
return -1
endi
#sql drop function bit_and;
#sql show functions;
#if $rows != 1 then
# return -1
#endi
#if $data00 != @l2norm@ then
# return -1
# endi
#sql drop function l2norm;
#sql show functions;
#if $rows != 0 then
# return -1
#endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT