xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #11572 from taosdata/3.0_udfd 

feat(udf): UDF service refactoring with new interface
This commit is contained in:
shenglian-zhou 2022-04-19 17:29:57 +08:00 committed by GitHub
parent e5bf165e24 adf9957533
commit b62b17865a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1024 additions and 575 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
source/libs/function/CMakeLists.txt
View File

@ -5,6 +5,9 @@ target_include_directories(
function function
PUBLIC PUBLIC
"${TD_SOURCE_DIR}/include/libs/function" "${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/contrib/libuv/include" "${TD_SOURCE_DIR}/contrib/libuv/include"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
@ -21,6 +24,9 @@ target_include_directories(
PUBLIC PUBLIC
"${TD_SOURCE_DIR}/include/libs/function" "${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/contrib/libuv/include" "${TD_SOURCE_DIR}/contrib/libuv/include"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/include/os" "${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
@ -35,6 +41,9 @@ target_include_directories(
udf1 udf1
PUBLIC PUBLIC
"${TD_SOURCE_DIR}/include/libs/function" "${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/client"
"${TD_SOURCE_DIR}/include/os" "${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
@ -46,6 +55,10 @@ target_include_directories(
PUBLIC PUBLIC
"${TD_SOURCE_DIR}/include/libs/function" "${TD_SOURCE_DIR}/include/libs/function"
"${TD_SOURCE_DIR}/contrib/libuv/include" "${TD_SOURCE_DIR}/contrib/libuv/include"
"${TD_SOURCE_DIR}/include/util"
"${TD_SOURCE_DIR}/include/common"
"${TD_SOURCE_DIR}/include/libs/transport"
"${TD_SOURCE_DIR}/include/client"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )

144
source/libs/function/inc/tudf.h
View File

@ -16,12 +16,25 @@
#ifndef TDENGINE_TUDF_H #ifndef TDENGINE_TUDF_H
#define TDENGINE_TUDF_H #define TDENGINE_TUDF_H
#include <stdint.h>
#include <stdbool.h>
#include "tmsg.h"
#include "tcommon.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
//====================================================================================== //======================================================================================
//begin API to taosd and qworker //begin API to taosd and qworker
enum {
UDFC_CODE_STOPPING = -1,
UDFC_CODE_RESTARTING = -2,
UDFC_CODE_PIPE_READ_ERR = -3,
};
/** /**
* start udf dameon service * start udf dameon service
* @return error code * @return error code
@ -34,28 +47,6 @@ int32_t startUdfService();
*/ */
int32_t stopUdfService(); int32_t stopUdfService();
enum {
TSDB_UDF_TYPE_SCALAR = 0,
TSDB_UDF_TYPE_AGGREGATE = 1
};
enum {
TSDB_UDF_SCRIPT_BIN_LIB = 0,
TSDB_UDF_SCRIPT_LUA = 1,
};
typedef struct SUdfInfo {
char *udfName; // function name
int32_t udfType; // scalar function or aggregate function
int8_t scriptType;
char *path;
int8_t resType; // result type
int16_t resBytes; // result byte
int32_t bufSize; //interbuf size
} SUdfInfo;
typedef void *UdfHandle; typedef void *UdfHandle;
/** /**
@ -64,36 +55,67 @@ typedef void *UdfHandle;
* @param handle, out * @param handle, out
* @return error code * @return error code
*/ */
int32_t setupUdf(SUdfInfo* udf, UdfHandle *handle); int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle);
typedef struct SUdfColumnMeta {
int16_t type;
int32_t bytes; // <0 var length, others fixed length bytes
uint8_t precision;
uint8_t scale;
} SUdfColumnMeta;
typedef struct SUdfColumnData {
int32_t numOfRows;
bool varLengthColumn;
union {
struct {
int32_t nullBitmapLen;
char *nullBitmap;
int32_t dataLen;
char *data;
} fixLenCol;
struct {
int32_t varOffsetsLen;
char *varOffsets;
int32_t payloadLen;
char *payload;
} varLenCol;
};
} SUdfColumnData;
enum { typedef struct SUdfColumn {
TSDB_UDF_STEP_NORMAL = 0, SUdfColumnMeta colMeta;
TSDB_UDF_STEP_MERGE, SUdfColumnData colData;
TSDb_UDF_STEP_FINALIZE, } SUdfColumn;
TSDB_UDF_STEP_MAX_NUM
};
/**
* call udf
* @param handle udf handle
* @param step
* @param state
* @param stateSize
* @param input
* @param newstate
* @param newStateSize
* @param output
* @return error code
*/
//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined
typedef struct SUdfDataBlock { typedef struct SUdfDataBlock {
char* data; int32_t numOfRows;
int32_t size; int32_t numOfCols;
SUdfColumn **udfCols;
} SUdfDataBlock; } SUdfDataBlock;
int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newstate, typedef struct SUdfInterBuf {
int32_t *newStateSize, SUdfDataBlock *output); int32_t bufLen;
char* buf;
} SUdfInterBuf;
//TODO: translate these calls to callUdf
// output: interBuf
int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf);
// input: block, state
// output: newState
int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
// input: interBuf
// output: resultData
int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
// input: interbuf1, interbuf2
// output: resultBuf
int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
// input: block
// output: resultData
int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData);
/** /**
* tearn down udf * tearn down udf
@ -104,29 +126,25 @@ int32_t teardownUdf(UdfHandle handle);
// end API to taosd and qworker // end API to taosd and qworker
//============================================================================================================================= //=============================================================================================================================
// TODO: Must change
// begin API to UDF writer. // begin API to UDF writer.
// script // dynamic lib init and destroy
typedef int32_t (*TUdfSetupFunc)();
typedef int32_t (*TUdfTeardownFunc)();
//typedef int32_t (*scriptInitFunc)(void* pCtx); //TODO: add API to check function arguments type, number etc.
//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, //TODO: another way to manage memory is provide api for UDF to add data to SUdfColumnData and UDF framework will allocate memory.
// int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, // then UDF framework will free the memory
// int16_t oType, int16_t oBytes); //typedef int32_t addFixedLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t colBytes, char* data);
//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); //typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data);
//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
//typedef void (*scriptDestroyFunc)(void* pCtx);
// dynamic lib typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
typedef int32_t (*TUdfInitFunc)();
typedef void (*TUdfDestroyFunc)();
typedef void (*TUdfFunc)(int8_t step, typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock block, SUdfColumn *resultCol);
char *state, int32_t stateSize, SUdfDataBlock input, typedef int32_t (*TUdfAggInitFunc)(SUdfInterBuf *buf);
char **newstate, int32_t *newStateSize, SUdfDataBlock *output); typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock block, SUdfInterBuf *interBuf);
typedef int32_t (*TUdfAggFinalizeFunc)(SUdfInterBuf buf, SUdfInterBuf *resultData);
//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput);
//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output);
// end API to UDF writer // end API to UDF writer
//======================================================================================================================= //=======================================================================================================================

64
source/libs/function/inc/tudfInt.h
View File

@ -15,7 +15,6 @@
#ifndef TDENGINE_TUDF_INT_H #ifndef TDENGINE_TUDF_INT_H
#define TDENGINE_TUDF_INT_H #define TDENGINE_TUDF_INT_H
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -30,36 +29,37 @@ enum {
}; };
enum {
TSDB_UDF_CALL_AGG_INIT = 0,
TSDB_UDF_CALL_AGG_PROC,
TSDB_UDF_CALL_AGG_MERGE,
TSDB_UDF_CALL_AGG_FIN,
TSDB_UDF_CALL_SCALA_PROC,
};
typedef struct SUdfSetupRequest { typedef struct SUdfSetupRequest {
char udfName[16]; // char udfName[TSDB_FUNC_NAME_LEN];
int8_t scriptType; // 0:c, 1: lua, 2:js SEpSet epSet;
int8_t udfType; //udaf, udf
int16_t pathSize;
char *path;
} SUdfSetupRequest; } SUdfSetupRequest;
typedef struct SUdfSetupResponse { typedef struct SUdfSetupResponse {
int64_t udfHandle; int64_t udfHandle;
} SUdfSetupResponse; } SUdfSetupResponse;
typedef struct SUdfCallRequest { typedef struct SUdfCallRequest {
int64_t udfHandle; int64_t udfHandle;
int8_t step; int8_t callType;
int32_t inputBytes; SSDataBlock block;
char *input; SUdfInterBuf interBuf;
SUdfInterBuf interBuf2;
int32_t stateBytes; int8_t initFirst;
char *state;
} SUdfCallRequest; } SUdfCallRequest;
typedef struct SUdfCallResponse { typedef struct SUdfCallResponse {
int32_t outputBytes; int8_t callType;
char *output; SSDataBlock resultData;
int32_t newStateBytes; SUdfInterBuf resultBuf;
char *newState;
} SUdfCallResponse; } SUdfCallResponse;
@ -76,7 +76,11 @@ typedef struct SUdfRequest {
int64_t seqNum; int64_t seqNum;
int8_t type; int8_t type;
void *subReq; union {
SUdfSetupRequest setup;
SUdfCallRequest call;
SUdfTeardownRequest teardown;
};
} SUdfRequest; } SUdfRequest;
typedef struct SUdfResponse { typedef struct SUdfResponse {
@ -85,13 +89,25 @@ typedef struct SUdfResponse {
int8_t type; int8_t type;
int32_t code; int32_t code;
void *subRsp; union {
SUdfSetupResponse setupRsp;
SUdfCallResponse callRsp;
SUdfTeardownResponse teardownRsp;
};
} SUdfResponse; } SUdfResponse;
int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); int32_t encodeUdfRequest(void **buf, const SUdfRequest* request);
int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); void* decodeUdfRequest(const void *buf, SUdfRequest* request);
int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request);
int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); int32_t encodeUdfResponse(void **buf, const SUdfResponse *response);
void* decodeUdfResponse(const void* buf, SUdfResponse *response);
void freeUdfColumnData(SUdfColumnData *data);
void freeUdfColumn(SUdfColumn* col);
void freeUdfDataDataBlock(SUdfDataBlock *block);
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock);
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block);
#ifdef __cplusplus #ifdef __cplusplus
} }

1040
source/libs/function/src/tudf.c
View File

File diff suppressed because it is too large Load Diff

217
source/libs/function/src/udfd.c
View File

@ -20,6 +20,10 @@
#include "tudf.h" #include "tudf.h"
#include "tudfInt.h" #include "tudfInt.h"
#include "tdataformat.h"
#include "tglobal.h"
#include "tmsg.h"
#include "trpc.h"
static uv_loop_t *loop; static uv_loop_t *loop;
@ -44,7 +48,8 @@ typedef struct SUdf {
int8_t type; int8_t type;
uv_lib_t lib; uv_lib_t lib;
TUdfFunc normalFunc; TUdfScalarProcFunc scalarProcFunc;
TUdfFreeUdfColumnFunc freeUdfColumn;
} SUdf; } SUdf;
//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix //TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
@ -56,119 +61,120 @@ typedef struct SUdfHandle {
void udfdProcessRequest(uv_work_t *req) { void udfdProcessRequest(uv_work_t *req) {
SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data); SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data);
SUdfRequest *request = NULL; SUdfRequest request = {0};
decodeRequest(uvUdf->input.base, uvUdf->input.len, &request); decodeUdfRequest(uvUdf->input.base, &request);
switch (request->type) { switch (request.type) {
case UDF_TASK_SETUP: { case UDF_TASK_SETUP: {
debugPrint("%s", "process setup request"); debugPrint("%s", "process setup request");
SUdf *udf = taosMemoryMalloc(sizeof(SUdf)); SUdf *udf = taosMemoryMalloc(sizeof(SUdf));
udf->refCount = 0; udf->refCount = 0;
SUdfSetupRequest *setup = request->subReq; SUdfSetupRequest *setup = &request.setup;
strcpy(udf->name, setup->udfName); strcpy(udf->name, setup->udfName);
int err = uv_dlopen(setup->path, &udf->lib); //TODO: retrive udf info from mnode
char* path = "libudf1.so";
int err = uv_dlopen(path, &udf->lib);
if (err != 0) { if (err != 0) {
debugPrint("can not load library %s. error: %s", setup->path, uv_strerror(err)); debugPrint("can not load library %s. error: %s", path, uv_strerror(err));
//TODO set error //TODO set error
} }
char normalFuncName[32] = {0}; char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
strcpy(normalFuncName, setup->udfName); strcpy(normalFuncName, setup->udfName);
//TODO error, //TODO error, multi-thread, same udf, lock it
//TODO find all functions normal, init, destroy, normal, merge, finalize //TODO find all functions normal, init, destroy, normal, merge, finalize
uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc)); uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->scalarProcFunc));
char freeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
char *freeSuffix = "_free";
strncpy(freeFuncName, normalFuncName, strlen(normalFuncName));
strncat(freeFuncName, freeSuffix, strlen(freeSuffix));
uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn));
SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle));
handle->udf = udf; handle->udf = udf;
udf->refCount++; udf->refCount++;
//TODO: allocate private structure and call init function and set it to handle //TODO: allocate private structure and call init function and set it to handle
SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); SUdfResponse rsp;
rsp->seqNum = request->seqNum; rsp.seqNum = request.seqNum;
rsp->type = request->type; rsp.type = request.type;
rsp->code = 0; rsp.code = 0;
SUdfSetupResponse *subRsp = taosMemoryMalloc(sizeof(SUdfSetupResponse)); rsp.setupRsp.udfHandle = (int64_t) (handle);
subRsp->udfHandle = (int64_t) (handle); int32_t len = encodeUdfResponse(NULL, &rsp);
rsp->subRsp = subRsp; rsp.msgLen = len;
char *buf; void *bufBegin = taosMemoryMalloc(len);
int32_t len; void *buf = bufBegin;
encodeResponse(&buf, &len, rsp); encodeUdfResponse(&buf, &rsp);
uvUdf->output = uv_buf_init(buf, len); uvUdf->output = uv_buf_init(bufBegin, len);
taosMemoryFree(rsp->subRsp);
taosMemoryFree(rsp);
taosMemoryFree(request->subReq);
taosMemoryFree(request);
taosMemoryFree(uvUdf->input.base); taosMemoryFree(uvUdf->input.base);
break; break;
} }
case UDF_TASK_CALL: { case UDF_TASK_CALL: {
debugPrint("%s", "process call request"); debugPrint("%s", "process call request");
SUdfCallRequest *call = request->subReq; SUdfCallRequest *call = &request.call;
SUdfHandle *handle = (SUdfHandle *) (call->udfHandle); SUdfHandle *handle = (SUdfHandle *) (call->udfHandle);
SUdf *udf = handle->udf; SUdf *udf = handle->udf;
char *newState;
int32_t newStateSize;
SUdfDataBlock input = {.data = call->input, .size= call->inputBytes};
SUdfDataBlock output;
//TODO: call different functions according to the step
udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output);
SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); SUdfDataBlock input = {0};
rsp->seqNum = request->seqNum; convertDataBlockToUdfDataBlock(&call->block, &input);
rsp->type = request->type; SUdfColumn output = {0};
rsp->code = 0; //TODO: call different functions according to call type, for now just calar
SUdfCallResponse *subRsp = taosMemoryMalloc(sizeof(SUdfCallResponse)); if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
subRsp->outputBytes = output.size; udf->scalarProcFunc(input, &output);
subRsp->output = output.data; }
subRsp->newStateBytes = newStateSize;
subRsp->newState = newState;
rsp->subRsp = subRsp;
char *buf; SUdfResponse response = {0};
int32_t len; SUdfResponse *rsp = &response;
encodeResponse(&buf, &len, rsp); if (call->callType == TSDB_UDF_CALL_SCALA_PROC) {
uvUdf->output = uv_buf_init(buf, len); rsp->seqNum = request.seqNum;
rsp->type = request.type;
rsp->code = 0;
SUdfCallResponse *subRsp = &rsp->callRsp;
subRsp->callType = call->callType;
convertUdfColumnToDataBlock(&output, &subRsp->resultData);
}
int32_t len = encodeUdfResponse(NULL, rsp);
rsp->msgLen = len;
void *bufBegin = taosMemoryMalloc(len);
void *buf = bufBegin;
encodeUdfResponse(&buf, rsp);
uvUdf->output = uv_buf_init(bufBegin, len);
//TODO: free
udf->freeUdfColumn(&output);
taosMemoryFree(rsp->subRsp);
taosMemoryFree(rsp);
taosMemoryFree(newState);
taosMemoryFree(output.data);
taosMemoryFree(request->subReq);
taosMemoryFree(request);
taosMemoryFree(uvUdf->input.base); taosMemoryFree(uvUdf->input.base);
break; break;
} }
case UDF_TASK_TEARDOWN: { case UDF_TASK_TEARDOWN: {
debugPrint("%s", "process teardown request"); debugPrint("%s", "process teardown request");
SUdfTeardownRequest *teardown = request->subReq; SUdfTeardownRequest *teardown = &request.teardown;
SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle); SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle);
SUdf *udf = handle->udf; SUdf *udf = handle->udf;
udf->refCount--; udf->refCount--;
if (udf->refCount == 0) { if (udf->refCount == 0) {
uv_dlclose(&udf->lib); uv_dlclose(&udf->lib);
taosMemoryFree(udf);
} }
taosMemoryFree(udf); //TODO: call destroy and free udf private
//TODO: call destroy and free udf private
taosMemoryFree(handle); taosMemoryFree(handle);
SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); SUdfResponse response;
rsp->seqNum = request->seqNum; SUdfResponse *rsp = &response;
rsp->type = request->type; rsp->seqNum = request.seqNum;
rsp->type = request.type;
rsp->code = 0; rsp->code = 0;
SUdfTeardownResponse *subRsp = taosMemoryMalloc(sizeof(SUdfTeardownResponse)); int32_t len = encodeUdfResponse(NULL, rsp);
rsp->subRsp = subRsp; rsp->msgLen = len;
char *buf; void *bufBegin = taosMemoryMalloc(len);
int32_t len; void *buf = bufBegin;
encodeResponse(&buf, &len, rsp); encodeUdfResponse(&buf, rsp);
uvUdf->output = uv_buf_init(buf, len); uvUdf->output = uv_buf_init(bufBegin, len);
taosMemoryFree(rsp->subRsp);
taosMemoryFree(rsp);
taosMemoryFree(request->subReq);
taosMemoryFree(request);
taosMemoryFree(uvUdf->input.base); taosMemoryFree(uvUdf->input.base);
break; break;
} }
@ -181,7 +187,7 @@ void udfdProcessRequest(uv_work_t *req) {
} }
void udfdOnWrite(uv_write_t *req, int status) { void udfdOnWrite(uv_write_t *req, int status) {
debugPrint("%s", "after writing to pipe"); debugPrint("%s", "server after writing to pipe");
if (status < 0) { if (status < 0) {
debugPrint("Write error %s", uv_err_name(status)); debugPrint("Write error %s", uv_err_name(status));
} }
@ -205,7 +211,7 @@ void udfdSendResponse(uv_work_t *work, int status) {
} }
void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
debugPrint("%s", "allocate buffer for read"); debugPrint("%s", "server allocate buffer for read");
SUdfdUvConn *ctx = handle->data; SUdfdUvConn *ctx = handle->data;
int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
if (ctx->inputCap == 0) { if (ctx->inputCap == 0) {
@ -329,6 +335,77 @@ void removeListeningPipe(int sig) {
exit(0); exit(0);
} }
typedef struct SServerContext {
void *clientRpc;
} SUdfdContext;
void udfdProcessRpcRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
return;
}
int32_t fetchUdfFuncInfo(void *clientRpc, SEpSet* pEpSet, char* udfNames[], int32_t numOfUdfs) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
for (int32_t i = 0; i < numOfUdfs; ++i) {
taosArrayPush(retrieveReq.pFuncNames, udfNames[i]);
}
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
taosArrayDestroy(retrieveReq.pFuncNames);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = contLen;
rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
SRpcMsg rpcRsp = {0};
rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0);
taosArrayDestroy(retrieveRsp.pFuncInfos);
rpcFreeCont(rpcRsp.pCont);
return 0;
}
int32_t openUdfdClientRpc(SUdfdContext *ctx) {
char *pass = "taosdata";
char *user = "root";
char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt);
SRpcInit rpcInit = {0};
rpcInit.label = (char*)"UDFD";
rpcInit.numOfThreads = 1;
rpcInit.cfp = udfdProcessRpcRsp;
rpcInit.sessions = 1024;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = 30 * 1000;
rpcInit.parent = ctx;
rpcInit.user = (char*)user;
rpcInit.ckey = (char*)"key";
rpcInit.secret = (char*)secretEncrypt;
rpcInit.spi = 1;
ctx->clientRpc = rpcOpen(&rpcInit);
return 0;
}
int32_t closeUdfdClientRpc(SUdfdContext *ctx) {
rpcClose(ctx->clientRpc);
return 0;
}
int main() { int main() {
debugPrint("libuv version: %x", UV_VERSION_HEX); debugPrint("libuv version: %x", UV_VERSION_HEX);

47
source/libs/function/test/runUdf.c
View File

@ -5,6 +5,7 @@
#include "uv.h" #include "uv.h"
#include "os.h" #include "os.h"
#include "tudf.h" #include "tudf.h"
#include "tdatablock.h"
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
startUdfService(); startUdfService();
@ -18,28 +19,38 @@ int main(int argc, char *argv[]) {
} }
fprintf(stdout, "current working directory:%s\n", path); fprintf(stdout, "current working directory:%s\n", path);
strcat(path, "/libudf1.so"); strcat(path, "/libudf1.so");
SUdfInfo udfInfo = {.udfName="udf1", .path=path};
UdfHandle handle; UdfHandle handle;
setupUdf(&udfInfo, &handle); SEpSet epSet;
setupUdf("udf1", &epSet, &handle);
//char state[5000000] = "state"; SSDataBlock block = {0};
//char input[5000000] = "input"; SSDataBlock* pBlock = &block;
int dataSize = 500; pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
int callCount = 2; pBlock->info.numOfCols = 1;
if (argc > 1) dataSize = atoi(argv[1]); pBlock->info.rows = 4;
if (argc > 2) callCount = atoi(argv[2]); char data[16] = {0};
char *state = taosMemoryMalloc(dataSize); char bitmap[4] = {0};
char *input = taosMemoryMalloc(dataSize); for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SUdfDataBlock blockInput = {.data = input, .size = dataSize}; SColumnInfoData colInfo = {0};
SUdfDataBlock blockOutput; colInfo.info.type = TSDB_DATA_TYPE_INT;
char* newState; colInfo.info.bytes = sizeof(int32_t);
int32_t newStateSize; colInfo.info.colId = 1;
for (int l = 0; l < callCount; ++l) { colInfo.pData = data;
callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput); colInfo.nullbitmap = bitmap;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
}
SSDataBlock output = {0};
callUdfScalaProcess(handle, pBlock, &output);
SColumnInfoData *col = taosArrayGet(output.pDataBlock, 0);
for (int32_t i = 0; i < output.info.rows; ++i) {
fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t)));
} }
taosMemoryFree(state);
taosMemoryFree(input);
teardownUdf(handle); teardownUdf(handle);
stopUdfService(); stopUdfService();

74
source/libs/function/test/udf1.c
View File

@ -2,20 +2,68 @@
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include "os.h"
#include "tudf.h" #include "tudf.h"
void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, #undef malloc
char **newState, int32_t *newStateSize, SUdfDataBlock *output) { #define malloc malloc
fprintf(stdout, "%s, step:%d\n", "udf function called", step); #undef free
char *newStateBuf = taosMemoryMalloc(stateSize); #define free free
memcpy(newStateBuf, state, stateSize);
*newState = newStateBuf;
*newStateSize = stateSize;
char *outputBuf = taosMemoryMalloc(input.size); int32_t udf1_setup() {
memcpy(outputBuf, input.data, input.size); return 0;
output->data = outputBuf;
output->size = input.size;
return;
} }
int32_t udf1_teardown() {
return 0;
}
int32_t udf1(SUdfDataBlock block, SUdfColumn *resultCol) {
SUdfColumnData *resultData = &resultCol->colData;
resultData->numOfRows = block.numOfRows;
SUdfColumnData *srcData = &block.udfCols[0]->colData;
resultData->varLengthColumn = srcData->varLengthColumn;
if (resultData->varLengthColumn) {
resultData->varLenCol.varOffsetsLen = srcData->varLenCol.varOffsetsLen;
resultData->varLenCol.varOffsets = malloc(resultData->varLenCol.varOffsetsLen);
memcpy(resultData->varLenCol.varOffsets, srcData->varLenCol.varOffsets, srcData->varLenCol.varOffsetsLen);
resultData->varLenCol.payloadLen = srcData->varLenCol.payloadLen;
resultData->varLenCol.payload = malloc(resultData->varLenCol.payloadLen);
memcpy(resultData->varLenCol.payload, srcData->varLenCol.payload, srcData->varLenCol.payloadLen);
} else {
resultData->fixLenCol.nullBitmapLen = srcData->fixLenCol.nullBitmapLen;
resultData->fixLenCol.nullBitmap = malloc(resultData->fixLenCol.nullBitmapLen);
memcpy(resultData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmapLen);
resultData->fixLenCol.dataLen = srcData->fixLenCol.dataLen;
resultData->fixLenCol.data = malloc(resultData->fixLenCol.dataLen);
memcpy(resultData->fixLenCol.data, srcData->fixLenCol.data, srcData->fixLenCol.dataLen);
for (int32_t i = 0; i < resultData->numOfRows; ++i) {
*(resultData->fixLenCol.data + i * sizeof(int32_t)) = 88;
}
}
SUdfColumnMeta *meta = &resultCol->colMeta;
meta->bytes = 4;
meta->type = TSDB_DATA_TYPE_INT;
meta->scale = 0;
meta->precision = 0;
return 0;
}
int32_t udf1_free(SUdfColumn *col) {
SUdfColumnData *data = &col->colData;
if (data->varLengthColumn) {
free(data->varLenCol.varOffsets);
data->varLenCol.varOffsets = NULL;
free(data->varLenCol.payload);
data->varLenCol.payload = NULL;
} else {
free(data->fixLenCol.nullBitmap);
data->fixLenCol.nullBitmap = NULL;
free(data->fixLenCol.data);
data->fixLenCol.data = NULL;
}
return 0;
}