From 8e3d10252eb4d72ce7ac541e786cc7142941b4ff Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 23 Mar 2022 15:33:28 +0800 Subject: [PATCH 01/15] move from private repo to private brach of public repo --- source/libs/function/inc/tudf.h | 151 ++++-- source/libs/function/inc/udfc.h | 110 ++++ source/libs/function/src/udfc.c | 914 ++++++++++++++++++++++++++++++++ source/libs/function/src/udfd.c | 350 ++++++++++++ 4 files changed, 1474 insertions(+), 51 deletions(-) create mode 100644 source/libs/function/inc/udfc.h create mode 100644 source/libs/function/src/udfc.c create mode 100644 source/libs/function/src/udfd.c diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 163fbdf4bb..9faf8d6d39 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -22,7 +22,107 @@ extern "C" { #include "os.h" #include "taoserror.h" +#include "tcommon.h" +//====================================================================================== +//begin API to taosd and qworker +/** + * start udf dameon service + * @return error code + */ +int32_t startUdfService(); + +/** + * stop udf dameon service + * @return error code + */ +int32_t stopUdfService(); + +typedef struct SUdfInfo { + char *udfName; // function name + int32_t funcType; // scalar function or aggregate function + int8_t isScript; + char *path; + + int8_t resType; // result type + int16_t resBytes; // result byte + int32_t bufSize; //interbuf size + +} SUdfInfo; + +typedef void *UdfHandle; + +/** + * setup udf + * @param udf, in + * @param handle, out + * @return error code + */ +int32_t setupUdf(SUdfInfo* udf, UdfHandle *handle); + + +enum { + TSDB_UDF_STEP_NORMAL = 0, + TSDB_UDF_STEP_MERGE, + TSDb_UDF_STEP_FINALIZE, + TSDB_UDF_STEP_MAX_NUM +}; +/** + * call udf + * @param handle udf handle + * @param step + * @param state + * @param stateSize + * @param input + * @param newstate + * @param newStateSize + * @param output + * @return error code + */ + +typedef struct SUdfDataBlock { + int16_t numOfCols; + struct { + char* data; + int32_t length; + } *colsData; +} SUdfDataBlock; + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock *input, char **newstate, + int32_t *newStateSize, SUdfDataBlock **output); + +/** + * tearn down udf + * @param handle + * @return + */ +int32_t teardownUdf(UdfHandle handle); + +// end API to taosd and qworker +//============================================================================================================================= +// begin API to UDF writer + +// script + +//typedef int32_t (*scriptInitFunc)(void* pCtx); +//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, +// int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, +// int16_t oType, int16_t oBytes); +//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); +//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); +//typedef void (*scriptDestroyFunc)(void* pCtx); + +// dynamic lib +typedef int32_t (*udfInitFunc)(); +typedef void (*udfDestroyFunc)(); + +typedef void (*udfNormalFunc)(char *state, int32_t stateSize, SUdfDataBlock input, char **newstate, + int32_t *newStateSize, SUdfDataBlock *output); +typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); +typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output); + +// end API to UDF writer +//======================================================================================================================= enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, @@ -32,57 +132,6 @@ enum { TSDB_UDF_FUNC_MAX_NUM }; -typedef struct SUdfInit { - int32_t maybe_null; /* 1 if function can return NULL */ - uint32_t decimals; /* for real functions */ - uint64_t length; /* For string functions */ - char* ptr; /* free pointer for function data */ - int32_t const_item; /* 0 if result is independent of arguments */ - - // script like lua/javascript - void* script_ctx; - void (*destroyCtxFunc)(void* script_ctx); -} SUdfInit; - -typedef struct SUdfInfo { - int32_t functionId; // system assigned function id - int32_t funcType; // scalar function or aggregate function - int8_t resType; // result type - int16_t resBytes; // result byte - int32_t contLen; // content length - int32_t bufSize; // interbuf size - char* name; // function name - void* handle; // handle loaded in mem - void* funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr - - // for script like lua/javascript only - int isScript; - void* pScriptCtx; - - SUdfInit init; - char* content; - char* path; -} SUdfInfo; - -// script - -typedef int32_t (*scriptInitFunc)(void* pCtx); -typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows, - int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, - int16_t oType, int16_t oBytes); -typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); -typedef void (*scriptDestroyFunc)(void* pCtx); - -// dynamic lib -typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, - char* dataOutput, char* interBuf, char* tsOutput, int32_t* numOfOutput, int16_t oType, - int16_t oBytes, SUdfInit* buf); -typedef int32_t (*udfInitFunc)(SUdfInit* data); -typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf); -typedef void (*udfDestroyFunc)(SUdfInit* buf); - #ifdef __cplusplus } #endif diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h new file mode 100644 index 0000000000..4d2aeb7049 --- /dev/null +++ b/source/libs/function/inc/udfc.h @@ -0,0 +1,110 @@ +// +// Created by shenglian on 28/02/22. +// + +#ifndef UDF_UDF_H +#define UDF_UDF_H + +#include +#define DEBUG +#ifdef DEBUG +#define debugPrint(...) fprintf(__VA_ARGS__) +#else +#define debugPrint(...) /**/ +#endif + +enum { + UDF_TASK_SETUP = 0, + UDF_TASK_CALL = 1, + UDF_TASK_TEARDOWN = 2 + +}; + +typedef struct SSDataBlock{ + char *data; + int32_t size; +} SSDataBlock; + +typedef struct SUdfInfo { + char *udfName; + char *path; +} SUdfInfo; + +typedef void *UdfHandle; + +int32_t startUdfService(); + +int32_t stopUdfService(); + +//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles); + +int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle); + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, + int32_t *newStateSize, SSDataBlock *output); + +int32_t teardownUdf(UdfHandle handle); + +typedef struct SUdfSetupRequest { + char udfName[16]; // + int8_t scriptType; // 0:c, 1: lua, 2:js + int8_t udfType; //udaf, udf, udtf + int16_t pathSize; + char *path; +} SUdfSetupRequest; + +typedef struct SUdfSetupResponse { + int64_t udfHandle; +} SUdfSetupResponse; + + +typedef struct SUdfCallRequest { + int64_t udfHandle; + int8_t step; + + int32_t inputBytes; + char *input; + + int32_t stateBytes; + char *state; +} SUdfCallRequest; + + +typedef struct SUdfCallResponse { + int32_t outputBytes; + char *output; + int32_t newStateBytes; + char *newState; +} SUdfCallResponse; + + +typedef struct SUdfTeardownRequest { + int64_t udfHandle; +} SUdfTeardownRequest; + + +typedef struct SUdfTeardownResponse { +} SUdfTeardownResponse; + +typedef struct SUdfRequest { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + void *subReq; +} SUdfRequest; + +typedef struct SUdfResponse { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + int32_t code; + void *subRsp; +} SUdfResponse; + +int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); +int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); +int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request); +int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); +#endif //UDF_UDF_H diff --git a/source/libs/function/src/udfc.c b/source/libs/function/src/udfc.c new file mode 100644 index 0000000000..b7fd1bcba1 --- /dev/null +++ b/source/libs/function/src/udfc.c @@ -0,0 +1,914 @@ +// +// Created by shenglian on 28/02/22. +// +#include +#include +#include +#include +#include +#include +#include + +//TODO: when startup, set thread poll size. add it to cfg +//TODO: udfd restart when exist or aborts +//TODO: network error processing. +//TODO: add unit test +//TODO: add lua support +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); + +enum { + UV_TASK_CONNECT = 0, + UV_TASK_REQ_RSP = 1, + UV_TASK_DISCONNECT = 2 +}; + +typedef struct SUdfUvSession { + int64_t severHandle; + uv_pipe_t *udfSvcPipe; +} SUdfUvSession; + +typedef struct SClientUvTaskNode { + int8_t type; + int errCode; + + uv_pipe_t *pipe; + + int64_t seqNum; + uv_buf_t reqBuf; + + uv_sem_t taskSem; + uv_buf_t rspBuf; + + struct SClientUvTaskNode *prev; + struct SClientUvTaskNode *next; +} SClientUvTaskNode; + +typedef struct SClientUdfTask { + int8_t type; + + SUdfUvSession *session; + + int32_t errCode; + + union { + struct { + SUdfSetupRequest req; + SUdfSetupResponse rsp; + } _setup; + struct { + SUdfCallRequest req; + SUdfCallResponse rsp; + } _call; + struct { + SUdfTeardownRequest req; + SUdfTeardownResponse rsp; + } _teardown; + }; + + +} SClientUdfTask; + +typedef struct SClientConnBuf { + char *buf; + int32_t len; + int32_t cap; + int32_t total; +} SClientConnBuf; + +typedef struct SClientUvConn { + uv_pipe_t *pipe; + SClientUvTaskNode taskQueue; + SClientConnBuf readBuf; +} SClientUvConn; + +uv_process_t gUdfdProcess; + +uv_barrier_t gUdfInitBarrier; + +uv_loop_t gUdfdLoop; +uv_thread_t gUdfLoopThread; +uv_async_t gUdfLoopTaskAync; + +uv_async_t gUdfLoopStopAsync; + +uv_mutex_t gUdfTaskQueueMutex; +int64_t gUdfTaskSeqNum = 0; + +//double circular linked list +typedef SClientUvTaskNode *SClientUvTaskQueue; +SClientUvTaskNode gUdfQueueNode; +SClientUvTaskQueue gUdfTaskQueue = &gUdfQueueNode; + +//add SClientUvTaskNode task that close conn + + + +void udfTaskQueueInit(SClientUvTaskQueue q) { + q->next = q; + q->prev = q; +} + +bool udfTaskQueueIsEmpty(SClientUvTaskQueue q) { + return q == q->next; +} + +void udfTaskQueueInsertTail(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q; + e->prev = q->prev; + e->prev->next = e; + q->prev = e; +} + +void udfTaskQueueInsertTaskAtHead(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q->next; + e->prev = q; + q->next->prev = e; + q->next = e; +} + +void udfTaskQueueRemoveTask(SClientUvTaskNode *e) { + e->prev->next = e->next; + e->next->prev = e->prev; +} + +void udfTaskQueueSplit(SClientUvTaskQueue q, SClientUvTaskNode *from, SClientUvTaskQueue n) { + n->prev = q->prev; + n->prev->next = n; + n->next = from; + q->prev = from->prev; + q->prev->next = q; + from->prev = n; +} + +SClientUvTaskNode *udfTaskQueueHeadTask(SClientUvTaskQueue q) { + return q->next; +} + +SClientUvTaskNode *udfTaskQueueTailTask(SClientUvTaskQueue q) { + return q->prev; +} + +SClientUvTaskNode *udfTaskQueueNext(SClientUvTaskNode *e) { + return e->next; +} + +void udfTaskQueueMove(SClientUvTaskQueue q, SClientUvTaskQueue n) { + if (udfTaskQueueIsEmpty(q)) { + udfTaskQueueInit(n); + } else { + SClientUvTaskNode *h = udfTaskQueueHeadTask(q); + udfTaskQueueSplit(q, h, n); + } +} + + +int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { + debugPrint(stdout, "%s\n", "encoding request"); + + int len = sizeof(SUdfRequest) - sizeof(void *); + switch (request->type) { + case 0: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + len += sizeof(SUdfSetupRequest) - 1 * sizeof(char *) + setup->pathSize; + break; + } + case 1: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + len += sizeof(SUdfCallRequest) - 2 * sizeof(char *) + call->inputBytes + call->stateBytes; + break; + } + case 2: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + len += sizeof(SUdfTeardownRequest); + break; + } + default: + break; + } + + char *bufBegin = malloc(len); + char *buf = bufBegin; + + //skip msgLen first + buf += sizeof(int32_t); + + *(int64_t *) buf = request->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = request->type; + buf += sizeof(int8_t); + + switch (request->type) { + case 0: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + memcpy(buf, setup->udfName, 16); + buf += 16; + *(int8_t *) buf = setup->scriptType; + buf += sizeof(int8_t); + *(int8_t *) buf = setup->udfType; + buf += sizeof(int8_t); + *(int16_t *) buf = setup->pathSize; + buf += sizeof(int16_t); + memcpy(buf, setup->path, setup->pathSize); + buf += setup->pathSize; + break; + } + + case 1: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + *(int64_t *) buf = call->udfHandle; + buf += sizeof(int64_t); + *(int8_t *) buf = call->step; + buf += sizeof(int8_t); + *(int32_t *) buf = call->inputBytes; + buf += sizeof(int32_t); + memcpy(buf, call->input, call->inputBytes); + buf += call->inputBytes; + *(int32_t *) buf = call->stateBytes; + buf += sizeof(int32_t); + memcpy(buf, call->state, call->stateBytes); + buf += call->stateBytes; + break; + } + + case 2: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + *(int64_t *) buf = teardown->udfHandle; + buf += sizeof(int64_t); + break; + } + default: + break; + } + + request->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = request->msgLen; + *pBuf = bufBegin; + *pBufLen = request->msgLen; + return 0; +} + +int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { + debugPrint(stdout, "%s\n", "decoding request"); + if (*(int32_t *) bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "dequest request error"); + return -1; + } + char *buf = bufMsg; + SUdfRequest *request = malloc(sizeof(SUdfRequest)); + request->subReq = NULL; + request->msgLen = *(int32_t *) (buf); + buf += sizeof(int32_t); + request->seqNum = *(int64_t *) (buf); + buf += sizeof(int64_t); + request->type = *(int8_t *) (buf); + buf += sizeof(int8_t); + + switch (request->type) { + case 0: { + SUdfSetupRequest *setup = malloc(sizeof(SUdfSetupRequest)); + + memcpy(setup->udfName, buf, 16); + buf += 16; + setup->scriptType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->udfType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->pathSize = *(int16_t *) buf; + buf += sizeof(int16_t); + setup->path = buf; + buf += setup->pathSize; + + request->subReq = setup; + break; + } + case 1: { + SUdfCallRequest *call = malloc(sizeof(SUdfCallRequest)); + + call->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + call->step = *(int8_t *) buf; + buf += sizeof(int8_t); + call->inputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->input = buf; + buf += call->inputBytes; + call->stateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->state = buf; + buf += call->stateBytes; + + request->subReq = call; + break; + } + + case 2: { + SUdfTeardownRequest *teardown = malloc(sizeof(SUdfTeardownRequest)); + + teardown->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + + request->subReq = teardown; + } + + } + if (buf - bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "decode request error"); + free(request->subReq); + free(request); + return -1; + } + *pRequest = request; + return 0; +} + +int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { + debugPrint(stdout, "%s\n", "encoding response"); + + int32_t len = sizeof(SUdfResponse) - sizeof(void *); + + switch (response->type) { + case 0: { + len += sizeof(SUdfSetupResponse); + break; + } + case 1: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + len += sizeof(SUdfCallResponse) - 2 * sizeof(char *) + + callResp->outputBytes + callResp->newStateBytes; + break; + } + case 2: { + len += sizeof(SUdfTeardownResponse); + break; + } + } + + char *bufBegin = malloc(len); + char *buf = bufBegin; + + //skip msgLen + buf += sizeof(int32_t); + + *(int64_t *) buf = response->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = response->type; + buf += sizeof(int8_t); + *(int32_t *) buf = response->code; + buf += sizeof(int32_t); + + + switch (response->type) { + case 0: { + SUdfSetupResponse *setupResp = (SUdfSetupResponse *) (response->subRsp); + *(int64_t *) buf = setupResp->udfHandle; + buf += sizeof(int64_t); + break; + } + case 1: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + *(int32_t *) buf = callResp->outputBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->output, callResp->outputBytes); + buf += callResp->outputBytes; + + *(int32_t *) buf = callResp->newStateBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->newState, callResp->newStateBytes); + buf += callResp->newStateBytes; + break; + } + case 2: { + SUdfTeardownResponse *teardownResp = (SUdfTeardownResponse *) (response->subRsp); + break; + } + default: + break; + } + response->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = response->msgLen; + *pBuf = bufBegin; + *pBufLen = response->msgLen; + return 0; +} + +int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { + debugPrint(stdout, "%s\n", "decoding response"); + + if (*(int32_t *) bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "can not decode response"); + return -1; + } + char *buf = bufMsg; + SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + rsp->msgLen = *(int32_t *) buf; + buf += sizeof(int32_t); + rsp->seqNum = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->type = *(int8_t *) buf; + buf += sizeof(int8_t); + rsp->code = *(int32_t *) buf; + buf += sizeof(int32_t); + + switch (rsp->type) { + case 0: { + SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) malloc(sizeof(SUdfSetupResponse)); + setupRsp->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->subRsp = (char *) setupRsp; + break; + } + case 1: { + SUdfCallResponse *callRsp = (SUdfCallResponse *) malloc(sizeof(SUdfCallResponse)); + callRsp->outputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + + callRsp->output = buf; + buf += callRsp->outputBytes; + + callRsp->newStateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + + callRsp->newState = buf; + buf += callRsp->newStateBytes; + + rsp->subRsp = callRsp; + break; + } + case 2: { + SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) malloc(sizeof(SUdfTeardownResponse)); + rsp->subRsp = teardownRsp; + break; + } + default: + break; + } + if (buf - bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "can not decode response"); + free(rsp->subRsp); + free(rsp); + return -1; + } + *pResponse = rsp; + return 0; +} + +void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { + debugPrint(stderr, "Process exited with status %" PRId64 ", signal %d\n", exit_status, term_signal); + uv_close((uv_handle_t *) req, NULL); +} + +void onUdfcPipeClose(uv_handle_t *handle) { + SClientUvConn *conn = handle->data; + if (!udfTaskQueueIsEmpty(&conn->taskQueue)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(&conn->taskQueue); + task->errCode = 0; + uv_sem_post(&task->taskSem); + } + + free(conn->readBuf.buf); + free(conn); + free((uv_pipe_t *) handle); + +} + +int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { + debugPrint(stdout, "%s\n", "get uv task result"); + if (uvTask->type == UV_TASK_REQ_RSP) { + if (uvTask->rspBuf.base != NULL) { + SUdfResponse *rsp; + decodeResponse(uvTask->rspBuf.base, uvTask->rspBuf.len, &rsp); + task->errCode = rsp->code; + + switch (task->type) { + case UDF_TASK_SETUP: { + //TODO: copy + task->_setup.rsp = *(SUdfSetupResponse *) (rsp->subRsp); + break; + } + case UDF_TASK_CALL: { + task->_call.rsp = *(SUdfCallResponse *) (rsp->subRsp); + //TODO: copy + break; + } + case UDF_TASK_TEARDOWN: { + task->_teardown.rsp = *(SUdfTeardownResponse *) (rsp->subRsp); + //TODO: copy or not? + break; + } + default: { + break; + } + } + + // TODO: the call buffer is setup and freed by udf invocation + free(uvTask->rspBuf.base); + free(rsp->subRsp); + free(rsp); + } else { + task->errCode = uvTask->errCode; + } + } else if (uvTask->type == UV_TASK_CONNECT) { + task->errCode = uvTask->errCode; + } else if (uvTask->type == UV_TASK_DISCONNECT) { + task->errCode = uvTask->errCode; + } + return 0; +} + +void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + debugPrint(stdout, "%s\n", "client allocate buffer to receive from pipe"); + SClientUvConn *conn = handle->data; + SClientConnBuf *connBuf = &conn->readBuf; + + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (connBuf->cap == 0) { + connBuf->buf = malloc(msgHeadSize); + if (connBuf->buf) { + connBuf->len = 0; + connBuf->cap = msgHeadSize; + connBuf->total = -1; + + buf->base = connBuf->buf; + buf->len = connBuf->cap; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } else { + connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; + void *resultBuf = realloc(connBuf->buf, connBuf->cap); + if (resultBuf) { + connBuf->buf = resultBuf; + buf->base = connBuf->buf + connBuf->len; + buf->len = connBuf->cap - connBuf->len; + } else { + //TODO: log error free connBuf->buf + buf->base = NULL; + buf->len = 0; + } + } + + debugPrint(stdout, "\tconn buf cap - len - total : %d - %d - %d\n", connBuf->cap, connBuf->len, connBuf->total); + +} + +bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(int32_t)) { + connBuf->total = *(int32_t *) (connBuf->buf); + } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + return true; + } + return false; +} + +void udfcUvHandleRsp(SClientUvConn *conn) { + SClientConnBuf *connBuf = &conn->readBuf; + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum + + if (udfTaskQueueIsEmpty(&conn->taskQueue)) { + //LOG error + return; + } + bool found = false; + SClientUvTaskNode *taskFound = NULL; + SClientUvTaskNode *task = udfTaskQueueNext(&conn->taskQueue); + while (task != &conn->taskQueue) { + if (task->seqNum == seqNum) { + if (found == false) { + found = true; + taskFound = task; + } else { + //LOG error; + continue; + } + } + task = udfTaskQueueNext(task); + } + + if (taskFound) { + taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); + udfTaskQueueRemoveTask(taskFound); + uv_sem_post(&taskFound->taskSem); + } else { + //LOG error + } + connBuf->buf = NULL; + connBuf->total = -1; + connBuf->len = 0; + connBuf->cap = 0; +} + +void udfcUvHandleError(SClientUvConn *conn) { + uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); +} + +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + debugPrint(stdout, "%s, nread: %zd\n", "client read from pipe", nread); + if (nread == 0) return; + + SClientUvConn *conn = client->data; + SClientConnBuf *connBuf = &conn->readBuf; + if (nread > 0) { + connBuf->len += nread; + if (isUdfcUvMsgComplete(connBuf)) { + udfcUvHandleRsp(conn); + } + + } + if (nread < 0) { + debugPrint(stderr, "\tclient read error: %s\n", uv_strerror(nread)); + if (nread == UV_EOF) { + //TODO: + } + udfcUvHandleError(conn); + } + +} + +void onUdfClientWrite(uv_write_t *write, int status) { + debugPrint(stderr, "%s\n", "after writing to pipe"); + SClientUvTaskNode *uvTask = write->data; + if (status == 0) { + uv_pipe_t *pipe = uvTask->pipe; + SClientUvConn *conn = pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + } else { + //TODO Log error; + } + debugPrint(stdout, "\tlength:%zu\n", uvTask->reqBuf.len); + free(write); + free(uvTask->reqBuf.base); +} + +void onUdfClientConnect(uv_connect_t *connect, int status) { + SClientUvTaskNode *uvTask = connect->data; + uvTask->errCode = status; + if (status != 0) { + //TODO: LOG error + } + uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); + free(connect); + uv_sem_post(&uvTask->taskSem); +} + +int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { + SClientUvTaskNode *uvTask = calloc(1, sizeof(SClientUvTaskNode)); + uvTask->type = uvTaskType; + + if (uvTaskType == UV_TASK_CONNECT) { + } else if (uvTaskType == UV_TASK_REQ_RSP) { + uvTask->pipe = task->session->udfSvcPipe; + SUdfRequest request; + request.type = task->type; + request.seqNum = gUdfTaskSeqNum++; + + if (task->type == UDF_TASK_SETUP) { + request.subReq = &task->_setup.req; + request.type = UDF_TASK_SETUP; + } else if (task->type == UDF_TASK_CALL) { + request.subReq = &task->_call.req; + request.type = UDF_TASK_CALL; + } else if (task->type == UDF_TASK_TEARDOWN) { + request.subReq = &task->_teardown.req; + request.type = UDF_TASK_TEARDOWN; + } else { + //TODO log and return error + } + char *buf = NULL; + int32_t bufLen = 0; + encodeRequest(&buf, &bufLen, &request); + uvTask->reqBuf = uv_buf_init(buf, bufLen); + uvTask->seqNum = request.seqNum; + } else if (uvTaskType == UV_TASK_DISCONNECT) { + uvTask->pipe = task->session->udfSvcPipe; + } + uv_sem_init(&uvTask->taskSem, 0); + + *pUvTask = uvTask; + return 0; +} + +int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint(stdout, "%s, %d\n", "queue uv task", uvTask->type); + + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueInsertTail(gUdfTaskQueue, uvTask); + uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_async_send(&gUdfLoopTaskAync); + + uv_sem_wait(&uvTask->taskSem); + uv_sem_destroy(&uvTask->taskSem); + + return 0; +} + +int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint(stdout, "%s, type %d\n", "start uv task ", uvTask->type); + switch (uvTask->type) { + case UV_TASK_CONNECT: { + uv_pipe_t *pipe = malloc(sizeof(uv_pipe_t)); + uv_pipe_init(&gUdfdLoop, pipe, 0); + uvTask->pipe = pipe; + + SClientUvConn *conn = malloc(sizeof(SClientUvConn)); + conn->pipe = pipe; + conn->readBuf.len = 0; + conn->readBuf.cap = 0; + conn->readBuf.buf = 0; + conn->readBuf.total = -1; + udfTaskQueueInit(&conn->taskQueue); + + pipe->data = conn; + + uv_connect_t *connReq = malloc(sizeof(uv_connect_t)); + connReq->data = uvTask; + + uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); + break; + } + case UV_TASK_REQ_RSP: { + uv_pipe_t *pipe = uvTask->pipe; + uv_write_t *write = malloc(sizeof(uv_write_t)); + write->data = uvTask; + uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); + break; + } + case UV_TASK_DISCONNECT: { + SClientUvConn *conn = uvTask->pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); + break; + } + default: { + break; + } + } + + return 0; +} + +void udfClientAsyncCb(uv_async_t *async) { + SClientUvTaskNode node; + SClientUvTaskQueue q = &node; + udfTaskQueueInit(q); + + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueMove(gUdfTaskQueue, q); + uv_mutex_unlock(&gUdfTaskQueueMutex); + + while (!udfTaskQueueIsEmpty(q)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(q); + udfTaskQueueRemoveTask(task); + startUvUdfTask(task); + } + +} + +void udfStopAsyncCb(uv_async_t *async) { + uv_stop(&gUdfdLoop); + uv_loop_close(&gUdfdLoop); +} + +void startUdfd(void *argsThread) { + uv_loop_init(&gUdfdLoop); + +// uv_process_options_t options; +// static char path[256] = {0}; +// size_t cwdSize; +// uv_cwd(path, &cwdSize); +// strcat(path, "/udfd"); +// char* args[2] = {path, NULL}; +// options.args = args; +// options.file = path; +// options.exit_cb = onUdfdExit; +// +// int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); +// if (err != 0) { +// debugPrint(stderr, "can not spawn udfd. error: %s", uv_strerror(err)); +// } + + uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); + uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); + uv_mutex_init(&gUdfTaskQueueMutex); + udfTaskQueueInit(gUdfTaskQueue); + uv_barrier_wait(&gUdfInitBarrier); + uv_run(&gUdfdLoop, UV_RUN_DEFAULT); +} + +int32_t startUdfService() { + uv_barrier_init(&gUdfInitBarrier, 2); + uv_thread_create(&gUdfLoopThread, startUdfd, 0); + uv_barrier_wait(&gUdfInitBarrier); + return 0; +} + +int32_t stopUdfService() { + uv_barrier_destroy(&gUdfInitBarrier); + //uv_process_kill(&gUdfdProcess, SIGINT); + uv_async_send(&gUdfLoopStopAsync); + uv_mutex_destroy(&gUdfTaskQueueMutex); + uv_thread_join(&gUdfLoopThread); + return 0; +} + +int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { + SClientUvTaskNode *uvTask = NULL; + + createUdfcUvTask(task, uvTaskType, &uvTask); + queueUvUdfTask(uvTask); + udfcGetUvTaskResponseResult(task, uvTask); + if (uvTaskType == UV_TASK_CONNECT) { + task->session->udfSvcPipe = uvTask->pipe; + } + free(uvTask); + uvTask = NULL; + return task->errCode; +} + +int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { + debugPrint(stdout, "%s\n", "client setup udf"); + SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = malloc(sizeof(SUdfUvSession)); + task->type = UDF_TASK_SETUP; + + SUdfSetupRequest *req = &task->_setup.req; + memcpy(req->udfName, udfInfo->udfName, 16); + req->path = udfInfo->path; + req->pathSize = strlen(req->path) + 1; + req->udfType = 0; + req->scriptType = 0; + + int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); + if (errCode != 0) { + //TODO: log error + return -1; + } + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfSetupResponse *rsp = &task->_setup.rsp; + task->session->severHandle = rsp->udfHandle; + *handle = task->session; + int32_t err = task->errCode; + free(task); + return err; +} + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newState, + int32_t *newStateSize, SSDataBlock *output) { + debugPrint(stdout, "%s\n", "client call udf"); + + SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_CALL; + + SUdfCallRequest *req = &task->_call.req; + + req->state = state; + req->stateBytes = stateSize; + req->inputBytes = input.size; + req->input = input.data; + req->udfHandle = task->session->severHandle; + req->step = step; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfCallResponse *rsp = &task->_call.rsp; + *newState = rsp->newState; + *newStateSize = rsp->newStateBytes; + output->size = rsp->outputBytes; + output->data = rsp->output; + int32_t err = task->errCode; + free(task); + return err; +} + +int32_t teardownUdf(UdfHandle handle) { + debugPrint(stdout, "%s\n", "client teardown udf"); + + SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_TEARDOWN; + + SUdfTeardownRequest *req = &task->_teardown.req; + req->udfHandle = task->session->severHandle; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + + SUdfTeardownResponse *rsp = &task->_teardown.rsp; + + int32_t err = task->errCode; + + udfcRunUvTask(task, UV_TASK_DISCONNECT); + + free(task->session); + free(task); + + return err; +} diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c new file mode 100644 index 0000000000..1442914888 --- /dev/null +++ b/source/libs/function/src/udfd.c @@ -0,0 +1,350 @@ +#include +#include +#include +#include +#include +#include + +uv_loop_t *loop; + +typedef struct SUdfdUvConn { + uv_stream_t *client; + char *inputBuf; + int32_t inputLen; + int32_t inputCap; + int32_t inputTotal; +} SUdfdUvConn; + +typedef struct SUvUdfWork { + uv_stream_t *client; + uv_buf_t input; + uv_buf_t output; +} SUvUdfWork; + +typedef void (*TUdfNormalFunc)(char *state, int32_t stateSize, char **newstate, int32_t *newStateSize, + SSDataBlock input, SSDataBlock *output); + + +typedef struct SUdf { + int32_t refCount; + + char name[16]; + int8_t type; + + uv_lib_t lib; + TUdfNormalFunc normalFunc; + +} SUdf; + +//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix +//TODO: add private udf structure. +typedef struct SUdfHandle { + SUdf *udf; +} SUdfHandle; + + +typedef void (*TUdfNormalFunc)(char *state, int32_t stateSize, char **newstate, int32_t *newStateSize, + SSDataBlock input, SSDataBlock *output); + + +void udfdProcessRequest(uv_work_t *req) { + SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data); + SUdfRequest *request = NULL; + decodeRequest(uvUdf->input.base, uvUdf->input.len, &request); + + switch (request->type) { + case UDF_TASK_SETUP: { + debugPrint(stdout, "%s\n", "process setup request"); + SUdf *udf = malloc(sizeof(SUdf)); + udf->refCount = 0; + SUdfSetupRequest *setup = request->subReq; + strcpy(udf->name, setup->udfName); + int err = uv_dlopen(setup->path, &udf->lib); + if (err != 0) { + debugPrint(stderr, "can not load library %s. error: %s", setup->path, uv_strerror(err)); + //TODO set error + } + + char normalFuncName[32] = {0}; + strcpy(normalFuncName, setup->udfName); + strcat(normalFuncName, "_normal"); + //TODO error, + //TODO find all functions normal, init, destroy, normal, merge, finalize + uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc)); + + SUdfHandle *handle = malloc(sizeof(SUdfHandle)); + handle->udf = udf; + udf->refCount++; + //TODO: allocate private structure and call init function and set it to handle + SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfSetupResponse *subRsp = malloc(sizeof(SUdfSetupResponse)); + subRsp->udfHandle = (int64_t) (handle); + rsp->subRsp = subRsp; + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + + uvUdf->output = uv_buf_init(buf, len); + + free(rsp->subRsp); + free(rsp); + free(request->subReq); + free(request); + free(uvUdf->input.base); + break; + } + + case UDF_TASK_CALL: { + debugPrint(stdout, "%s\n", "process call request"); + SUdfCallRequest *call = request->subReq; + SUdfHandle *handle = (SUdfHandle *) (call->udfHandle); + SUdf *udf = handle->udf; + char *newState; + int32_t newStateSize; + SSDataBlock input = {.data = call->input, .size= call->inputBytes}; + SSDataBlock output; + //TODO: call different functions according to the step + udf->normalFunc(call->state, call->stateBytes, &newState, &newStateSize, input, &output); + + SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfCallResponse *subRsp = malloc(sizeof(SUdfCallResponse)); + subRsp->outputBytes = output.size; + subRsp->output = output.data; + subRsp->newStateBytes = newStateSize; + subRsp->newState = newState; + rsp->subRsp = subRsp; + + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + uvUdf->output = uv_buf_init(buf, len); + + free(rsp->subRsp); + free(rsp); + free(newState); + free(output.data); + free(request->subReq); + free(request); + free(uvUdf->input.base); + break; + } + case UDF_TASK_TEARDOWN: { + debugPrint(stdout, "%s\n", "process teardown request"); + + SUdfTeardownRequest *teardown = request->subReq; + SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle); + SUdf *udf = handle->udf; + udf->refCount--; + if (udf->refCount == 0) { + uv_dlclose(&udf->lib); + } + free(udf); + //TODO: call destroy and free udf private + free(handle); + + SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = 0; + SUdfTeardownResponse *subRsp = malloc(sizeof(SUdfTeardownResponse)); + rsp->subRsp = subRsp; + char *buf; + int32_t len; + encodeResponse(&buf, &len, rsp); + uvUdf->output = uv_buf_init(buf, len); + + free(rsp->subRsp); + free(rsp); + free(request->subReq); + free(request); + free(uvUdf->input.base); + break; + } + default: { + break; + } + + } + +} + +void udfdOnWrite(uv_write_t *req, int status) { + debugPrint(stdout, "%s\n", "after writing to pipe"); + if (status < 0) { + debugPrint(stderr, "Write error %s\n", uv_err_name(status)); + } + SUvUdfWork *work = (SUvUdfWork *) req->data; + debugPrint(stdout, "\tlength: %zu\n", work->output.len); + free(work->output.base); + free(work); + free(req); +} + + +void udfdSendResponse(uv_work_t *work, int status) { + debugPrint(stdout, "%s\n", "send response"); + SUvUdfWork *udfWork = (SUvUdfWork *) (work->data); + + uv_write_t *write_req = malloc(sizeof(uv_write_t)); + write_req->data = udfWork; + uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); + + free(work); +} + +void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + debugPrint(stdout, "%s\n", "allocate buffer for read"); + SUdfdUvConn *ctx = handle->data; + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (ctx->inputCap == 0) { + ctx->inputBuf = malloc(msgHeadSize); + if (ctx->inputBuf) { + ctx->inputLen = 0; + ctx->inputCap = msgHeadSize; + ctx->inputTotal = -1; + + buf->base = ctx->inputBuf; + buf->len = ctx->inputCap; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } else { + ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; + void *inputBuf = realloc(ctx->inputBuf, ctx->inputCap); + if (inputBuf) { + ctx->inputBuf = inputBuf; + buf->base = ctx->inputBuf + ctx->inputLen; + buf->len = ctx->inputCap - ctx->inputLen; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } + debugPrint(stdout, "\tinput buf cap - len - total : %d - %d - %d\n", ctx->inputCap, ctx->inputLen, ctx->inputTotal); + +} + +bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) { + if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) { + pipe->inputTotal = *(int32_t *) (pipe->inputBuf); + } + if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) { + return true; + } + return false; +} + +void udfdHandleRequest(SUdfdUvConn *conn) { + uv_work_t *work = malloc(sizeof(uv_work_t)); + SUvUdfWork *udfWork = malloc(sizeof(SUvUdfWork)); + udfWork->client = conn->client; + udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); + conn->inputBuf = NULL; + conn->inputLen = 0; + conn->inputCap = 0; + conn->inputTotal = -1; + work->data = udfWork; + uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse); +} + +void udfdPipeCloseCb(uv_handle_t *pipe) { + SUdfdUvConn *conn = pipe->data; + free(conn->client); + free(conn->inputBuf); + free(conn); +} + +void udfdUvHandleError(SUdfdUvConn *conn) { + uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb); +} + +void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + debugPrint(stdout, "%s, nread: %zd\n", "read from pipe", nread); + + if (nread == 0) return; + + SUdfdUvConn *conn = client->data; + + if (nread > 0) { + conn->inputLen += nread; + if (isUdfdUvMsgComplete(conn)) { + udfdHandleRequest(conn); + } else { + //log error or continue; + } + return; + } + + if (nread < 0) { + debugPrint(stderr, "Read error %s\n", uv_err_name(nread)); + if (nread == UV_EOF) { + //TODO check more when close + } else { + } + udfdUvHandleError(conn); + } +} + +void udfdOnNewConnection(uv_stream_t *server, int status) { + debugPrint(stdout, "%s\n", "on new connection"); + if (status < 0) { + // TODO + return; + } + + uv_pipe_t *client = (uv_pipe_t *) malloc(sizeof(uv_pipe_t)); + uv_pipe_init(loop, client, 0); + if (uv_accept(server, (uv_stream_t *) client) == 0) { + SUdfdUvConn *ctx = malloc(sizeof(SUdfdUvConn)); + ctx->client = (uv_stream_t *) client; + ctx->inputBuf = 0; + ctx->inputLen = 0; + ctx->inputCap = 0; + client->data = ctx; + ctx->client = (uv_stream_t *) client; + uv_read_start((uv_stream_t *) client, udfdAllocBuffer, udfdPipeRead); + } else { + uv_close((uv_handle_t *) client, NULL); + } +} + +void removeListeningPipe(int sig) { + uv_fs_t req; + uv_fs_unlink(loop, &req, "udf.sock", NULL); + exit(0); +} + +int main() { + debugPrint(stderr, "libuv version: %x\n", UV_VERSION_HEX); + + loop = uv_default_loop(); + uv_fs_t req; + uv_fs_unlink(loop, &req, "udf.sock", NULL); + + uv_pipe_t server; + uv_pipe_init(loop, &server, 0); + + signal(SIGINT, removeListeningPipe); + + int r; + if ((r = uv_pipe_bind(&server, "udf.sock"))) { + debugPrint(stderr, "Bind error %s\n", uv_err_name(r)); + removeListeningPipe(0); + return 1; + } + if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) { + debugPrint(stderr, "Listen error %s\n", uv_err_name(r)); + return 2; + } + uv_run(loop, UV_RUN_DEFAULT); + uv_loop_close(loop); +} From b05706f9cf6727ad99fd1ce1515d1293a1f1404f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 23 Mar 2022 18:53:58 +0800 Subject: [PATCH 02/15] sync refactor --- source/libs/sync/inc/syncRaftEntry.h | 8 ++++++++ source/libs/sync/src/syncRaftEntry.c | 9 +++++++++ source/libs/sync/test/syncEntryTest.cpp | 20 ++++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 037a49f45c..876859216f 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -27,6 +27,12 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" +typedef enum EntryType { + SYNC_RAFT_ENTRY_NULL = 0, + SYNC_RAFT_ENTRY_DATA = 1, + SYNC_RAFT_ENTRY_CONFIG = 2, +} EntryType; + typedef struct SSyncRaftEntry { uint32_t bytes; uint32_t msgType; @@ -35,12 +41,14 @@ typedef struct SSyncRaftEntry { bool isWeak; SyncTerm term; SyncIndex index; + EntryType entryType; uint32_t dataLen; char data[]; } SSyncRaftEntry; SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType); void syncEntryDestory(SSyncRaftEntry* pEntry); char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index e4c5882e98..30ffebbc32 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -28,6 +28,13 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { // step 4. SyncClientRequest => SSyncRaftEntry, add term, index SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index, SYNC_RAFT_ENTRY_DATA); + assert(pEntry != NULL); + + return pEntry; +} + +SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); assert(pEntry != NULL); @@ -37,6 +44,7 @@ SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncInde pEntry->isWeak = pMsg->isWeak; pEntry->term = term; pEntry->index = index; + pEntry->entryType = entryType; pEntry->dataLen = pMsg->dataLen; memcpy(pEntry->data, pMsg->data, pMsg->dataLen); @@ -85,6 +93,7 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); cJSON_AddStringToObject(pRoot, "index", u64buf); + cJSON_AddNumberToObject(pRoot, "entryType", pEntry->entryType); cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); char* s; diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index e54daaa79a..7f211276f8 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -46,6 +46,20 @@ void test2() { } void test3() { + SyncClientRequest* pSyncMsg = syncClientRequestBuild(10); + pSyncMsg->originalRpcType = 33; + pSyncMsg->seqNum = 11; + pSyncMsg->isWeak = 1; + strcpy(pSyncMsg->data, "test3"); + + SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200, SYNC_RAFT_ENTRY_NULL); + syncEntryPrint(pEntry); + + syncClientRequestDestroy(pSyncMsg); + syncEntryDestory(pEntry); +} + +void test4() { SSyncRaftEntry* pEntry = syncEntryBuild(10); assert(pEntry != NULL); pEntry->msgType = 11; @@ -54,7 +68,8 @@ void test3() { pEntry->isWeak = true; pEntry->term = 44; pEntry->index = 55; - strcpy(pEntry->data, "test3"); + pEntry->entryType = SYNC_RAFT_ENTRY_CONFIG; + strcpy(pEntry->data, "test4"); syncEntryPrint(pEntry); uint32_t len; @@ -75,7 +90,8 @@ int main(int argc, char** argv) { test1(); test2(); - test3(); + test3(); + test4(); return 0; } From e58812aab5df79c724ba647d587096c0bfcf9211 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Mar 2022 11:40:36 +0800 Subject: [PATCH 03/15] sync refactor --- source/libs/sync/inc/syncRaftEntry.h | 3 ++- source/libs/sync/src/syncAppendEntries.c | 7 ++++--- source/libs/sync/src/syncCommit.c | 2 +- source/libs/sync/src/syncMain.c | 26 +++++++++++++++++++++--- source/libs/sync/src/syncRaftEntry.c | 10 +++++++++ source/libs/sync/src/syncRaftLog.c | 6 +++--- source/libs/sync/test/syncEntryTest.cpp | 2 +- 7 files changed, 44 insertions(+), 12 deletions(-) diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 876859216f..d4e63e9676 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -28,7 +28,7 @@ extern "C" { #include "taosdef.h" typedef enum EntryType { - SYNC_RAFT_ENTRY_NULL = 0, + SYNC_RAFT_ENTRY_NOOP = 0, SYNC_RAFT_ENTRY_DATA = 1, SYNC_RAFT_ENTRY_CONFIG = 2, } EntryType; @@ -49,6 +49,7 @@ typedef struct SSyncRaftEntry { SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType); +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index); void syncEntryDestory(SSyncRaftEntry* pEntry); char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 1116c1a905..0a83e09605 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -198,6 +198,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { if (ths->pFsm->FpRollBackCb != NULL) { SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); assert(pRollBackEntry != NULL); + assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA); SRpcMsg rpcMsg; syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); @@ -217,7 +218,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state); } } @@ -242,7 +243,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) { ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state); } } @@ -298,7 +299,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (ths->pFsm->FpCommitCb != NULL) { + if (ths->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index f581b31c4b..6023c00afa 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -97,7 +97,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); - if (pSyncNode->pFsm->FpCommitCb != NULL) { + if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f2341ef521..3080cbcd0d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -41,10 +41,11 @@ static void syncNodeEqPingTimer(void* param, void* tmrId); static void syncNodeEqElectTimer(void* param, void* tmrId); static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); -// on message ---- +// process message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); +static int32_t syncNodeAppendNoop(SSyncNode* ths); // life cycle static void syncFreeNode(void* param); @@ -669,6 +670,9 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); assert(voteGrantedMajority(pSyncNode->pVotesGranted)); syncNodeBecomeLeader(pSyncNode); + + // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { @@ -851,7 +855,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state); } } @@ -866,7 +870,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - if (ths->pFsm->FpPreCommitCb != NULL) { + if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) { ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 1, ths->state); } } @@ -877,6 +881,22 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg return ret; } +static int32_t syncNodeAppendNoop(SSyncNode* ths) { + int32_t ret = 0; + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + assert(pEntry != NULL); + + if (ths->state == TAOS_SYNC_STATE_LEADER) { + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + syncNodeReplicate(ths); + } + + return ret; +} + static void syncFreeNode(void* param) { SSyncNode* pNode = param; syncNodePrint2((char*)"==syncFreeNode==", pNode); diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 30ffebbc32..9b9842b81d 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -51,6 +51,16 @@ SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncInde return pEntry; } +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild(0); + assert(pEntry != NULL); + pEntry->term = term; + pEntry->index = index; + pEntry->entryType = SYNC_RAFT_ENTRY_NOOP; + + return pEntry; +} + void syncEntryDestory(SSyncRaftEntry* pEntry) { if (pEntry != NULL) { free(pEntry); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index f569c5d621..cad2d5da41 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -34,7 +34,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->getLastTerm = logStoreLastTerm; pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; - return pLogStore; // to avoid compiler error + return pLogStore; } void logStoreDestory(SSyncLogStore* pLogStore) { @@ -54,12 +54,12 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { assert(serialized != NULL); int code; - code = walWrite(pWal, pEntry->index, pEntry->msgType, serialized, len); + code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len); assert(code == 0); walFsync(pWal, true); free(serialized); - return code; // to avoid compiler error + return code; } SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index 7f211276f8..4acdb6052e 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -52,7 +52,7 @@ void test3() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test3"); - SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200, SYNC_RAFT_ENTRY_NULL); + SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200, SYNC_RAFT_ENTRY_NOOP); syncEntryPrint(pEntry); syncClientRequestDestroy(pSyncMsg); From 8c12733e193b385f0676efdb7ef5de8fd82fcb2e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 24 Mar 2022 14:09:29 +0800 Subject: [PATCH 04/15] pass compilation before change log --- source/libs/function/CMakeLists.txt | 13 + source/libs/function/inc/tudf.h | 55 +- source/libs/function/inc/tudfInt.h | 97 +++ source/libs/function/src/tudf.c | 1043 +++++++++++++++++++++++---- source/libs/function/src/udfc.c | 914 ----------------------- source/libs/function/src/udfd.c | 55 +- 6 files changed, 1059 insertions(+), 1118 deletions(-) create mode 100644 source/libs/function/inc/tudfInt.h delete mode 100644 source/libs/function/src/udfc.c diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index a10a542b6b..f44cbe6d63 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -1,4 +1,5 @@ aux_source_directory(src FUNCTION_SRC) +list(REMOVE_ITEM FUNCTION_SRC src/udfd.c) add_library(function STATIC ${FUNCTION_SRC}) target_include_directories( function @@ -9,4 +10,16 @@ target_include_directories( target_link_libraries( function PRIVATE os util common nodes +) + +add_executable(udfd src/udfd.c) +target_include_directories( + udfd + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + udfd + PRIVATE os util common nodes function ) \ No newline at end of file diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 9faf8d6d39..72875239d2 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -20,10 +20,6 @@ extern "C" { #endif -#include "os.h" -#include "taoserror.h" -#include "tcommon.h" - //====================================================================================== //begin API to taosd and qworker /** @@ -38,10 +34,20 @@ int32_t startUdfService(); */ int32_t stopUdfService(); +enum { + TSDB_UDF_TYPE_SCALAR = 0, + TSDB_UDF_TYPE_AGGREGATE = 1 +}; + +enum { + TSDB_UDF_SCRIPT_BIN_LIB = 0, + TSDB_UDF_SCRIPT_LUA = 1, +}; + typedef struct SUdfInfo { char *udfName; // function name - int32_t funcType; // scalar function or aggregate function - int8_t isScript; + int32_t udfType; // scalar function or aggregate function + int8_t scriptType; char *path; int8_t resType; // result type @@ -80,16 +86,14 @@ enum { * @return error code */ +//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined typedef struct SUdfDataBlock { - int16_t numOfCols; - struct { - char* data; - int32_t length; - } *colsData; + char* data; + int32_t size; } SUdfDataBlock; -int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock *input, char **newstate, - int32_t *newStateSize, SUdfDataBlock **output); +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newstate, + int32_t *newStateSize, SUdfDataBlock *output); /** * tearn down udf @@ -100,7 +104,8 @@ int32_t teardownUdf(UdfHandle handle); // end API to taosd and qworker //============================================================================================================================= -// begin API to UDF writer +// TODO: Must change +// begin API to UDF writer. // script @@ -113,24 +118,18 @@ int32_t teardownUdf(UdfHandle handle); //typedef void (*scriptDestroyFunc)(void* pCtx); // dynamic lib -typedef int32_t (*udfInitFunc)(); -typedef void (*udfDestroyFunc)(); +typedef int32_t (*TUdfInitFunc)(); +typedef void (*TUdfDestroyFunc)(); -typedef void (*udfNormalFunc)(char *state, int32_t stateSize, SUdfDataBlock input, char **newstate, - int32_t *newStateSize, SUdfDataBlock *output); -typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput); -typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output); +typedef void (*TUdfFunc)(int8_t step, + char *state, int32_t stateSize, SUdfDataBlock input, + char **newstate, int32_t *newStateSize, SUdfDataBlock *output); + +//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput); +//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output); // end API to UDF writer //======================================================================================================================= -enum { - TSDB_UDF_FUNC_NORMAL = 0, - TSDB_UDF_FUNC_INIT, - TSDB_UDF_FUNC_FINALIZE, - TSDB_UDF_FUNC_MERGE, - TSDB_UDF_FUNC_DESTROY, - TSDB_UDF_FUNC_MAX_NUM -}; #ifdef __cplusplus } diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h new file mode 100644 index 0000000000..048522968e --- /dev/null +++ b/source/libs/function/inc/tudfInt.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TUDF_INT_H +#define TDENGINE_TUDF_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + UDF_TASK_SETUP = 0, + UDF_TASK_CALL = 1, + UDF_TASK_TEARDOWN = 2 + +}; + +typedef struct SUdfSetupRequest { + char udfName[16]; // + int8_t scriptType; // 0:c, 1: lua, 2:js + int8_t udfType; //udaf, udf + int16_t pathSize; + char *path; +} SUdfSetupRequest; + +typedef struct SUdfSetupResponse { + int64_t udfHandle; +} SUdfSetupResponse; + + +typedef struct SUdfCallRequest { + int64_t udfHandle; + int8_t step; + + int32_t inputBytes; + char *input; + + int32_t stateBytes; + char *state; +} SUdfCallRequest; + + +typedef struct SUdfCallResponse { + int32_t outputBytes; + char *output; + int32_t newStateBytes; + char *newState; +} SUdfCallResponse; + + +typedef struct SUdfTeardownRequest { + int64_t udfHandle; +} SUdfTeardownRequest; + + +typedef struct SUdfTeardownResponse { +} SUdfTeardownResponse; + +typedef struct SUdfRequest { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + void *subReq; +} SUdfRequest; + +typedef struct SUdfResponse { + int32_t msgLen; + int64_t seqNum; + + int8_t type; + int32_t code; + void *subRsp; +} SUdfResponse; + +int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest); +int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response); +int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request); +int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TUDF_INT_H diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index b65e637a57..b4e1c72483 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1,195 +1,930 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "uv.h" +#include "os.h" #include "tudf.h" +#include "tudfInt.h" -static char* getUdfFuncName(char* funcname, char* name, int type) { - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - strcpy(funcname, name); - break; - case TSDB_UDF_FUNC_INIT: - sprintf(funcname, "%s_init", name); - break; - case TSDB_UDF_FUNC_FINALIZE: - sprintf(funcname, "%s_finalize", name); - break; - case TSDB_UDF_FUNC_MERGE: - sprintf(funcname, "%s_merge", name); - break; - case TSDB_UDF_FUNC_DESTROY: - sprintf(funcname, "%s_destroy", name); - break; - default: - assert(0); - break; - } +//TODO replace them with qDebug +#define DEBUG +#ifdef DEBUG +#define debugPrint(...) fprintf(__VA_ARGS__) +#else +#define debugPrint(...) /**/ +#endif - return funcname; +//TODO: when startup, set thread poll size. add it to cfg +//TODO: udfd restart when exist or aborts +//TODO: network error processing. +//TODO: add unit test +//TODO: add lua support +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); + +enum { + UV_TASK_CONNECT = 0, + UV_TASK_REQ_RSP = 1, + UV_TASK_DISCONNECT = 2 +}; + +typedef struct SUdfUvSession { + int64_t severHandle; + uv_pipe_t *udfSvcPipe; +} SUdfUvSession; + +typedef struct SClientUvTaskNode { + int8_t type; + int errCode; + + uv_pipe_t *pipe; + + int64_t seqNum; + uv_buf_t reqBuf; + + uv_sem_t taskSem; + uv_buf_t rspBuf; + + struct SClientUvTaskNode *prev; + struct SClientUvTaskNode *next; +} SClientUvTaskNode; + +typedef struct SClientUdfTask { + int8_t type; + + SUdfUvSession *session; + + int32_t errCode; + + union { + struct { + SUdfSetupRequest req; + SUdfSetupResponse rsp; + } _setup; + struct { + SUdfCallRequest req; + SUdfCallResponse rsp; + } _call; + struct { + SUdfTeardownRequest req; + SUdfTeardownResponse rsp; + } _teardown; + }; + + +} SClientUdfTask; + +typedef struct SClientConnBuf { + char *buf; + int32_t len; + int32_t cap; + int32_t total; +} SClientConnBuf; + +typedef struct SClientUvConn { + uv_pipe_t *pipe; + SClientUvTaskNode taskQueue; + SClientConnBuf readBuf; +} SClientUvConn; + +uv_process_t gUdfdProcess; + +uv_barrier_t gUdfInitBarrier; + +uv_loop_t gUdfdLoop; +uv_thread_t gUdfLoopThread; +uv_async_t gUdfLoopTaskAync; + +uv_async_t gUdfLoopStopAsync; + +uv_mutex_t gUdfTaskQueueMutex; +int64_t gUdfTaskSeqNum = 0; + +//double circular linked list +typedef SClientUvTaskNode *SClientUvTaskQueue; +SClientUvTaskNode gUdfQueueNode; +SClientUvTaskQueue gUdfTaskQueue = &gUdfQueueNode; + +//add SClientUvTaskNode task that close conn + + + +void udfTaskQueueInit(SClientUvTaskQueue q) { + q->next = q; + q->prev = q; } -#if 0 -int32_t initUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { - return TSDB_CODE_SUCCESS; - } - ////qError("script len: %d", pUdfInfo->contLen); - if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) { - pUdfInfo->isScript = 1; - pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes); - if (pUdfInfo->pScriptCtx == NULL) { - return TSDB_CODE_QRY_SYS_ERROR; - } - tfree(pUdfInfo->content); +bool udfTaskQueueIsEmpty(SClientUvTaskQueue q) { + return q == q->next; +} - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit; - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL - || (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_QRY_SYS_ERROR; - } +void udfTaskQueueInsertTail(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q; + e->prev = q->prev; + e->prev->next = e; + q->prev = e; +} - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; +void udfTaskQueueInsertTaskAtHead(SClientUvTaskQueue q, SClientUvTaskNode *e) { + e->next = q->next; + e->prev = q; + q->next->prev = e; + q->next = e; +} - if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; - } - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy; +void udfTaskQueueRemoveTask(SClientUvTaskNode *e) { + e->prev->next = e->next; + e->next->prev = e->prev; +} +void udfTaskQueueSplit(SClientUvTaskQueue q, SClientUvTaskNode *from, SClientUvTaskQueue n) { + n->prev = q->prev; + n->prev->next = n; + n->next = from; + q->prev = from->prev; + q->prev->next = q; + from->prev = n; +} + +SClientUvTaskNode *udfTaskQueueHeadTask(SClientUvTaskQueue q) { + return q->next; +} + +SClientUvTaskNode *udfTaskQueueTailTask(SClientUvTaskQueue q) { + return q->prev; +} + +SClientUvTaskNode *udfTaskQueueNext(SClientUvTaskNode *e) { + return e->next; +} + +void udfTaskQueueMove(SClientUvTaskQueue q, SClientUvTaskQueue n) { + if (udfTaskQueueIsEmpty(q)) { + udfTaskQueueInit(n); } else { - char path[PATH_MAX] = {0}; - taosGetTmpfilePath("script", path, tsTempDir); + SClientUvTaskNode *h = udfTaskQueueHeadTask(q); + udfTaskQueueSplit(q, h, n); + } +} - FILE* file = fopen(path, "w+"); - // TODO check for failure of flush to disk - /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); - fclose(file); - tfree(pUdfInfo->content); +int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { + debugPrint(stdout, "%s\n", "encoding request"); - pUdfInfo->path = strdup(path); + int len = sizeof(SUdfRequest) - sizeof(void *); + switch (request->type) { + case 0: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + len += sizeof(SUdfSetupRequest) - 1 * sizeof(char *) + setup->pathSize; + break; + } + case 1: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + len += sizeof(SUdfCallRequest) - 2 * sizeof(char *) + call->inputBytes + call->stateBytes; + break; + } + case 2: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + len += sizeof(SUdfTeardownRequest); + break; + } + default: + break; + } - pUdfInfo->handle = taosLoadDll(path); + char *bufBegin = malloc(len); + char *buf = bufBegin; - if (NULL == pUdfInfo->handle) { - return TSDB_CODE_QRY_SYS_ERROR; + //skip msgLen first + buf += sizeof(int32_t); + + *(int64_t *) buf = request->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = request->type; + buf += sizeof(int8_t); + + switch (request->type) { + case 0: { + SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); + memcpy(buf, setup->udfName, 16); + buf += 16; + *(int8_t *) buf = setup->scriptType; + buf += sizeof(int8_t); + *(int8_t *) buf = setup->udfType; + buf += sizeof(int8_t); + *(int16_t *) buf = setup->pathSize; + buf += sizeof(int16_t); + memcpy(buf, setup->path, setup->pathSize); + buf += setup->pathSize; + break; } - char funcname[FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; - pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); - if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { - return TSDB_CODE_QRY_SYS_ERROR; + case 1: { + SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); + *(int64_t *) buf = call->udfHandle; + buf += sizeof(int64_t); + *(int8_t *) buf = call->step; + buf += sizeof(int8_t); + *(int32_t *) buf = call->inputBytes; + buf += sizeof(int32_t); + memcpy(buf, call->input, call->inputBytes); + buf += call->inputBytes; + *(int32_t *) buf = call->stateBytes; + buf += sizeof(int32_t); + memcpy(buf, call->state, call->stateBytes); + buf += call->stateBytes; + break; } - pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + case 2: { + SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); + *(int64_t *) buf = teardown->udfHandle; + buf += sizeof(int64_t); + break; + } + default: + break; + } - if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + request->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = request->msgLen; + *pBuf = bufBegin; + *pBufLen = request->msgLen; + return 0; +} + +int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { + debugPrint(stdout, "%s\n", "decoding request"); + if (*(int32_t *) bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "dequest request error"); + return -1; + } + char *buf = bufMsg; + SUdfRequest *request = malloc(sizeof(SUdfRequest)); + request->subReq = NULL; + request->msgLen = *(int32_t *) (buf); + buf += sizeof(int32_t); + request->seqNum = *(int64_t *) (buf); + buf += sizeof(int64_t); + request->type = *(int8_t *) (buf); + buf += sizeof(int8_t); + + switch (request->type) { + case 0: { + SUdfSetupRequest *setup = malloc(sizeof(SUdfSetupRequest)); + + memcpy(setup->udfName, buf, 16); + buf += 16; + setup->scriptType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->udfType = *(int8_t *) buf; + buf += sizeof(int8_t); + setup->pathSize = *(int16_t *) buf; + buf += sizeof(int16_t); + setup->path = buf; + buf += setup->pathSize; + + request->subReq = setup; + break; + } + case 1: { + SUdfCallRequest *call = malloc(sizeof(SUdfCallRequest)); + + call->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + call->step = *(int8_t *) buf; + buf += sizeof(int8_t); + call->inputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->input = buf; + buf += call->inputBytes; + call->stateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + call->state = buf; + buf += call->stateBytes; + + request->subReq = call; + break; } - pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); + case 2: { + SUdfTeardownRequest *teardown = malloc(sizeof(SUdfTeardownRequest)); - if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { - return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); + teardown->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + + request->subReq = teardown; + } + + } + if (buf - bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "decode request error"); + free(request->subReq); + free(request); + return -1; + } + *pRequest = request; + return 0; +} + +int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { + debugPrint(stdout, "%s\n", "encoding response"); + + int32_t len = sizeof(SUdfResponse) - sizeof(void *); + + switch (response->type) { + case 0: { + len += sizeof(SUdfSetupResponse); + break; + } + case 1: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + len += sizeof(SUdfCallResponse) - 2 * sizeof(char *) + + callResp->outputBytes + callResp->newStateBytes; + break; + } + case 2: { + len += sizeof(SUdfTeardownResponse); + break; } } - return TSDB_CODE_SUCCESS; + char *bufBegin = malloc(len); + char *buf = bufBegin; + + //skip msgLen + buf += sizeof(int32_t); + + *(int64_t *) buf = response->seqNum; + buf += sizeof(int64_t); + *(int8_t *) buf = response->type; + buf += sizeof(int8_t); + *(int32_t *) buf = response->code; + buf += sizeof(int32_t); + + + switch (response->type) { + case 0: { + SUdfSetupResponse *setupResp = (SUdfSetupResponse *) (response->subRsp); + *(int64_t *) buf = setupResp->udfHandle; + buf += sizeof(int64_t); + break; + } + case 1: { + SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); + *(int32_t *) buf = callResp->outputBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->output, callResp->outputBytes); + buf += callResp->outputBytes; + + *(int32_t *) buf = callResp->newStateBytes; + buf += sizeof(int32_t); + memcpy(buf, callResp->newState, callResp->newStateBytes); + buf += callResp->newStateBytes; + break; + } + case 2: { + SUdfTeardownResponse *teardownResp = (SUdfTeardownResponse *) (response->subRsp); + break; + } + default: + break; + } + response->msgLen = buf - bufBegin; + *(int32_t *) bufBegin = response->msgLen; + *pBuf = bufBegin; + *pBufLen = response->msgLen; + return 0; } -void destroyUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { +int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { + debugPrint(stdout, "%s\n", "decoding response"); + + if (*(int32_t *) bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "can not decode response"); + return -1; + } + char *buf = bufMsg; + SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + rsp->msgLen = *(int32_t *) buf; + buf += sizeof(int32_t); + rsp->seqNum = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->type = *(int8_t *) buf; + buf += sizeof(int8_t); + rsp->code = *(int32_t *) buf; + buf += sizeof(int32_t); + + switch (rsp->type) { + case 0: { + SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) malloc(sizeof(SUdfSetupResponse)); + setupRsp->udfHandle = *(int64_t *) buf; + buf += sizeof(int64_t); + rsp->subRsp = (char *) setupRsp; + break; + } + case 1: { + SUdfCallResponse *callRsp = (SUdfCallResponse *) malloc(sizeof(SUdfCallResponse)); + callRsp->outputBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + + callRsp->output = buf; + buf += callRsp->outputBytes; + + callRsp->newStateBytes = *(int32_t *) buf; + buf += sizeof(int32_t); + + callRsp->newState = buf; + buf += callRsp->newStateBytes; + + rsp->subRsp = callRsp; + break; + } + case 2: { + SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) malloc(sizeof(SUdfTeardownResponse)); + rsp->subRsp = teardownRsp; + break; + } + default: + break; + } + if (buf - bufMsg != bufLen) { + debugPrint(stderr, "%s\n", "can not decode response"); + free(rsp->subRsp); + free(rsp); + return -1; + } + *pResponse = rsp; + return 0; +} + +void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { + debugPrint(stderr, "Process exited with status %" PRId64 ", signal %d\n", exit_status, term_signal); + uv_close((uv_handle_t *) req, NULL); +} + +void onUdfcPipeClose(uv_handle_t *handle) { + SClientUvConn *conn = handle->data; + if (!udfTaskQueueIsEmpty(&conn->taskQueue)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(&conn->taskQueue); + task->errCode = 0; + uv_sem_post(&task->taskSem); + } + + free(conn->readBuf.buf); + free(conn); + free((uv_pipe_t *) handle); + +} + +int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { + debugPrint(stdout, "%s\n", "get uv task result"); + if (uvTask->type == UV_TASK_REQ_RSP) { + if (uvTask->rspBuf.base != NULL) { + SUdfResponse *rsp; + decodeResponse(uvTask->rspBuf.base, uvTask->rspBuf.len, &rsp); + task->errCode = rsp->code; + + switch (task->type) { + case UDF_TASK_SETUP: { + //TODO: copy + task->_setup.rsp = *(SUdfSetupResponse *) (rsp->subRsp); + break; + } + case UDF_TASK_CALL: { + task->_call.rsp = *(SUdfCallResponse *) (rsp->subRsp); + //TODO: copy + break; + } + case UDF_TASK_TEARDOWN: { + task->_teardown.rsp = *(SUdfTeardownResponse *) (rsp->subRsp); + //TODO: copy or not? + break; + } + default: { + break; + } + } + + // TODO: the call buffer is setup and freed by udf invocation + free(uvTask->rspBuf.base); + free(rsp->subRsp); + free(rsp); + } else { + task->errCode = uvTask->errCode; + } + } else if (uvTask->type == UV_TASK_CONNECT) { + task->errCode = uvTask->errCode; + } else if (uvTask->type == UV_TASK_DISCONNECT) { + task->errCode = uvTask->errCode; + } + return 0; +} + +void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { + debugPrint(stdout, "%s\n", "client allocate buffer to receive from pipe"); + SClientUvConn *conn = handle->data; + SClientConnBuf *connBuf = &conn->readBuf; + + int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); + if (connBuf->cap == 0) { + connBuf->buf = malloc(msgHeadSize); + if (connBuf->buf) { + connBuf->len = 0; + connBuf->cap = msgHeadSize; + connBuf->total = -1; + + buf->base = connBuf->buf; + buf->len = connBuf->cap; + } else { + //TODO: log error + buf->base = NULL; + buf->len = 0; + } + } else { + connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; + void *resultBuf = realloc(connBuf->buf, connBuf->cap); + if (resultBuf) { + connBuf->buf = resultBuf; + buf->base = connBuf->buf + connBuf->len; + buf->len = connBuf->cap - connBuf->len; + } else { + //TODO: log error free connBuf->buf + buf->base = NULL; + buf->len = 0; + } + } + + debugPrint(stdout, "\tconn buf cap - len - total : %d - %d - %d\n", connBuf->cap, connBuf->len, connBuf->total); + +} + +bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(int32_t)) { + connBuf->total = *(int32_t *) (connBuf->buf); + } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + return true; + } + return false; +} + +void udfcUvHandleRsp(SClientUvConn *conn) { + SClientConnBuf *connBuf = &conn->readBuf; + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum + + if (udfTaskQueueIsEmpty(&conn->taskQueue)) { + //LOG error return; } + bool found = false; + SClientUvTaskNode *taskFound = NULL; + SClientUvTaskNode *task = udfTaskQueueNext(&conn->taskQueue); + while (task != &conn->taskQueue) { + if (task->seqNum == seqNum) { + if (found == false) { + found = true; + taskFound = task; + } else { + //LOG error; + continue; + } + } + task = udfTaskQueueNext(task); + } - if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { - if (pUdfInfo->isScript) { - (*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx); - tfree(pUdfInfo->content); - }else{ - (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); + if (taskFound) { + taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); + udfTaskQueueRemoveTask(taskFound); + uv_sem_post(&taskFound->taskSem); + } else { + //LOG error + } + connBuf->buf = NULL; + connBuf->total = -1; + connBuf->len = 0; + connBuf->cap = 0; +} + +void udfcUvHandleError(SClientUvConn *conn) { + uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); +} + +void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { + debugPrint(stdout, "%s, nread: %zd\n", "client read from pipe", nread); + if (nread == 0) return; + + SClientUvConn *conn = client->data; + SClientConnBuf *connBuf = &conn->readBuf; + if (nread > 0) { + connBuf->len += nread; + if (isUdfcUvMsgComplete(connBuf)) { + udfcUvHandleRsp(conn); + } + + } + if (nread < 0) { + debugPrint(stderr, "\tclient read error: %s\n", uv_strerror(nread)); + if (nread == UV_EOF) { + //TODO: + } + udfcUvHandleError(conn); + } + +} + +void onUdfClientWrite(uv_write_t *write, int status) { + debugPrint(stderr, "%s\n", "after writing to pipe"); + SClientUvTaskNode *uvTask = write->data; + if (status == 0) { + uv_pipe_t *pipe = uvTask->pipe; + SClientUvConn *conn = pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + } else { + //TODO Log error; + } + debugPrint(stdout, "\tlength:%zu\n", uvTask->reqBuf.len); + free(write); + free(uvTask->reqBuf.base); +} + +void onUdfClientConnect(uv_connect_t *connect, int status) { + SClientUvTaskNode *uvTask = connect->data; + uvTask->errCode = status; + if (status != 0) { + //TODO: LOG error + } + uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); + free(connect); + uv_sem_post(&uvTask->taskSem); +} + +int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { + SClientUvTaskNode *uvTask = calloc(1, sizeof(SClientUvTaskNode)); + uvTask->type = uvTaskType; + + if (uvTaskType == UV_TASK_CONNECT) { + } else if (uvTaskType == UV_TASK_REQ_RSP) { + uvTask->pipe = task->session->udfSvcPipe; + SUdfRequest request; + request.type = task->type; + request.seqNum = gUdfTaskSeqNum++; + + if (task->type == UDF_TASK_SETUP) { + request.subReq = &task->_setup.req; + request.type = UDF_TASK_SETUP; + } else if (task->type == UDF_TASK_CALL) { + request.subReq = &task->_call.req; + request.type = UDF_TASK_CALL; + } else if (task->type == UDF_TASK_TEARDOWN) { + request.subReq = &task->_teardown.req; + request.type = UDF_TASK_TEARDOWN; + } else { + //TODO log and return error + } + char *buf = NULL; + int32_t bufLen = 0; + encodeRequest(&buf, &bufLen, &request); + uvTask->reqBuf = uv_buf_init(buf, bufLen); + uvTask->seqNum = request.seqNum; + } else if (uvTaskType == UV_TASK_DISCONNECT) { + uvTask->pipe = task->session->udfSvcPipe; + } + uv_sem_init(&uvTask->taskSem, 0); + + *pUvTask = uvTask; + return 0; +} + +int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint(stdout, "%s, %d\n", "queue uv task", uvTask->type); + + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueInsertTail(gUdfTaskQueue, uvTask); + uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_async_send(&gUdfLoopTaskAync); + + uv_sem_wait(&uvTask->taskSem); + uv_sem_destroy(&uvTask->taskSem); + + return 0; +} + +int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { + debugPrint(stdout, "%s, type %d\n", "start uv task ", uvTask->type); + switch (uvTask->type) { + case UV_TASK_CONNECT: { + uv_pipe_t *pipe = malloc(sizeof(uv_pipe_t)); + uv_pipe_init(&gUdfdLoop, pipe, 0); + uvTask->pipe = pipe; + + SClientUvConn *conn = malloc(sizeof(SClientUvConn)); + conn->pipe = pipe; + conn->readBuf.len = 0; + conn->readBuf.cap = 0; + conn->readBuf.buf = 0; + conn->readBuf.total = -1; + udfTaskQueueInit(&conn->taskQueue); + + pipe->data = conn; + + uv_connect_t *connReq = malloc(sizeof(uv_connect_t)); + connReq->data = uvTask; + + uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); + break; + } + case UV_TASK_REQ_RSP: { + uv_pipe_t *pipe = uvTask->pipe; + uv_write_t *write = malloc(sizeof(uv_write_t)); + write->data = uvTask; + uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); + break; + } + case UV_TASK_DISCONNECT: { + SClientUvConn *conn = uvTask->pipe->data; + udfTaskQueueInsertTail(&conn->taskQueue, uvTask); + uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); + break; + } + default: { + break; } } - tfree(pUdfInfo->name); - - if (pUdfInfo->path) { - unlink(pUdfInfo->path); - } - - tfree(pUdfInfo->path); - tfree(pUdfInfo->content); - taosCloseDll(pUdfInfo->handle); - tfree(pUdfInfo); + return 0; } -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type) { - int32_t output = 0; +void udfClientAsyncCb(uv_async_t *async) { + SClientUvTaskNode node; + SClientUvTaskQueue q = &node; + udfTaskQueueInit(q); - if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { - //qError("empty udf function, type:%d", type); - return; + uv_mutex_lock(&gUdfTaskQueueMutex); + udfTaskQueueMove(gUdfTaskQueue, q); + uv_mutex_unlock(&gUdfTaskQueueMutex); + + while (!udfTaskQueueIsEmpty(q)) { + SClientUvTaskNode *task = udfTaskQueueHeadTask(q); + udfTaskQueueRemoveTask(task); + startUvUdfTask(task); } -// //qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); - - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - if (pUdfInfo->isScript) { - (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, - (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, - (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - - (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, - pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes, &pUdfInfo->init); - } - - if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { - pCtx->resultInfo->numOfRes = output; - } else { - pCtx->resultInfo->numOfRes += output; - } - - if (pCtx->resultInfo->numOfRes > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_MERGE: - if (pUdfInfo->isScript) { - (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); - } else { - (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); - } - - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_FINALIZE: { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); - } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); - } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - } - } } -#endif \ No newline at end of file +void udfStopAsyncCb(uv_async_t *async) { + uv_stop(&gUdfdLoop); + uv_loop_close(&gUdfdLoop); +} + +void startUdfd(void *argsThread) { + uv_loop_init(&gUdfdLoop); + +// uv_process_options_t options; +// static char path[256] = {0}; +// size_t cwdSize; +// uv_cwd(path, &cwdSize); +// strcat(path, "/udfd"); +// char* args[2] = {path, NULL}; +// options.args = args; +// options.file = path; +// options.exit_cb = onUdfdExit; +// +// int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); +// if (err != 0) { +// debugPrint(stderr, "can not spawn udfd. error: %s", uv_strerror(err)); +// } + + uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); + uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); + uv_mutex_init(&gUdfTaskQueueMutex); + udfTaskQueueInit(gUdfTaskQueue); + uv_barrier_wait(&gUdfInitBarrier); + uv_run(&gUdfdLoop, UV_RUN_DEFAULT); +} + +int32_t startUdfService() { + uv_barrier_init(&gUdfInitBarrier, 2); + uv_thread_create(&gUdfLoopThread, startUdfd, 0); + uv_barrier_wait(&gUdfInitBarrier); + return 0; +} + +int32_t stopUdfService() { + uv_barrier_destroy(&gUdfInitBarrier); + //uv_process_kill(&gUdfdProcess, SIGINT); + uv_async_send(&gUdfLoopStopAsync); + uv_mutex_destroy(&gUdfTaskQueueMutex); + uv_thread_join(&gUdfLoopThread); + return 0; +} + +int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { + SClientUvTaskNode *uvTask = NULL; + + createUdfcUvTask(task, uvTaskType, &uvTask); + queueUvUdfTask(uvTask); + udfcGetUvTaskResponseResult(task, uvTask); + if (uvTaskType == UV_TASK_CONNECT) { + task->session->udfSvcPipe = uvTask->pipe; + } + free(uvTask); + uvTask = NULL; + return task->errCode; +} + +int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { + debugPrint(stdout, "%s\n", "client setup udf"); + SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = malloc(sizeof(SUdfUvSession)); + task->type = UDF_TASK_SETUP; + + SUdfSetupRequest *req = &task->_setup.req; + memcpy(req->udfName, udfInfo->udfName, 16); + req->path = udfInfo->path; + req->pathSize = strlen(req->path) + 1; + req->udfType = udfInfo->udfType; + req->scriptType = udfInfo->scriptType; + + int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); + if (errCode != 0) { + //TODO: log error + return -1; + } + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfSetupResponse *rsp = &task->_setup.rsp; + task->session->severHandle = rsp->udfHandle; + *handle = task->session; + int32_t err = task->errCode; + free(task); + return err; +} + +int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newState, + int32_t *newStateSize, SUdfDataBlock *output) { + debugPrint(stdout, "%s\n", "client call udf"); + + SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_CALL; + + SUdfCallRequest *req = &task->_call.req; + + req->state = state; + req->stateBytes = stateSize; + req->inputBytes = input.size; + req->input = input.data; + req->udfHandle = task->session->severHandle; + req->step = step; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + SUdfCallResponse *rsp = &task->_call.rsp; + *newState = rsp->newState; + *newStateSize = rsp->newStateBytes; + output->size = rsp->outputBytes; + output->data = rsp->output; + int32_t err = task->errCode; + free(task); + return err; +} + +int32_t teardownUdf(UdfHandle handle) { + debugPrint(stdout, "%s\n", "client teardown udf"); + + SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + task->errCode = 0; + task->session = (SUdfUvSession *) handle; + task->type = UDF_TASK_TEARDOWN; + + SUdfTeardownRequest *req = &task->_teardown.req; + req->udfHandle = task->session->severHandle; + + udfcRunUvTask(task, UV_TASK_REQ_RSP); + + + SUdfTeardownResponse *rsp = &task->_teardown.rsp; + + int32_t err = task->errCode; + + udfcRunUvTask(task, UV_TASK_DISCONNECT); + + free(task->session); + free(task); + + return err; +} diff --git a/source/libs/function/src/udfc.c b/source/libs/function/src/udfc.c deleted file mode 100644 index b7fd1bcba1..0000000000 --- a/source/libs/function/src/udfc.c +++ /dev/null @@ -1,914 +0,0 @@ -// -// Created by shenglian on 28/02/22. -// -#include -#include -#include -#include -#include -#include -#include - -//TODO: when startup, set thread poll size. add it to cfg -//TODO: udfd restart when exist or aborts -//TODO: network error processing. -//TODO: add unit test -//TODO: add lua support -void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); - -enum { - UV_TASK_CONNECT = 0, - UV_TASK_REQ_RSP = 1, - UV_TASK_DISCONNECT = 2 -}; - -typedef struct SUdfUvSession { - int64_t severHandle; - uv_pipe_t *udfSvcPipe; -} SUdfUvSession; - -typedef struct SClientUvTaskNode { - int8_t type; - int errCode; - - uv_pipe_t *pipe; - - int64_t seqNum; - uv_buf_t reqBuf; - - uv_sem_t taskSem; - uv_buf_t rspBuf; - - struct SClientUvTaskNode *prev; - struct SClientUvTaskNode *next; -} SClientUvTaskNode; - -typedef struct SClientUdfTask { - int8_t type; - - SUdfUvSession *session; - - int32_t errCode; - - union { - struct { - SUdfSetupRequest req; - SUdfSetupResponse rsp; - } _setup; - struct { - SUdfCallRequest req; - SUdfCallResponse rsp; - } _call; - struct { - SUdfTeardownRequest req; - SUdfTeardownResponse rsp; - } _teardown; - }; - - -} SClientUdfTask; - -typedef struct SClientConnBuf { - char *buf; - int32_t len; - int32_t cap; - int32_t total; -} SClientConnBuf; - -typedef struct SClientUvConn { - uv_pipe_t *pipe; - SClientUvTaskNode taskQueue; - SClientConnBuf readBuf; -} SClientUvConn; - -uv_process_t gUdfdProcess; - -uv_barrier_t gUdfInitBarrier; - -uv_loop_t gUdfdLoop; -uv_thread_t gUdfLoopThread; -uv_async_t gUdfLoopTaskAync; - -uv_async_t gUdfLoopStopAsync; - -uv_mutex_t gUdfTaskQueueMutex; -int64_t gUdfTaskSeqNum = 0; - -//double circular linked list -typedef SClientUvTaskNode *SClientUvTaskQueue; -SClientUvTaskNode gUdfQueueNode; -SClientUvTaskQueue gUdfTaskQueue = &gUdfQueueNode; - -//add SClientUvTaskNode task that close conn - - - -void udfTaskQueueInit(SClientUvTaskQueue q) { - q->next = q; - q->prev = q; -} - -bool udfTaskQueueIsEmpty(SClientUvTaskQueue q) { - return q == q->next; -} - -void udfTaskQueueInsertTail(SClientUvTaskQueue q, SClientUvTaskNode *e) { - e->next = q; - e->prev = q->prev; - e->prev->next = e; - q->prev = e; -} - -void udfTaskQueueInsertTaskAtHead(SClientUvTaskQueue q, SClientUvTaskNode *e) { - e->next = q->next; - e->prev = q; - q->next->prev = e; - q->next = e; -} - -void udfTaskQueueRemoveTask(SClientUvTaskNode *e) { - e->prev->next = e->next; - e->next->prev = e->prev; -} - -void udfTaskQueueSplit(SClientUvTaskQueue q, SClientUvTaskNode *from, SClientUvTaskQueue n) { - n->prev = q->prev; - n->prev->next = n; - n->next = from; - q->prev = from->prev; - q->prev->next = q; - from->prev = n; -} - -SClientUvTaskNode *udfTaskQueueHeadTask(SClientUvTaskQueue q) { - return q->next; -} - -SClientUvTaskNode *udfTaskQueueTailTask(SClientUvTaskQueue q) { - return q->prev; -} - -SClientUvTaskNode *udfTaskQueueNext(SClientUvTaskNode *e) { - return e->next; -} - -void udfTaskQueueMove(SClientUvTaskQueue q, SClientUvTaskQueue n) { - if (udfTaskQueueIsEmpty(q)) { - udfTaskQueueInit(n); - } else { - SClientUvTaskNode *h = udfTaskQueueHeadTask(q); - udfTaskQueueSplit(q, h, n); - } -} - - -int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { - debugPrint(stdout, "%s\n", "encoding request"); - - int len = sizeof(SUdfRequest) - sizeof(void *); - switch (request->type) { - case 0: { - SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); - len += sizeof(SUdfSetupRequest) - 1 * sizeof(char *) + setup->pathSize; - break; - } - case 1: { - SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); - len += sizeof(SUdfCallRequest) - 2 * sizeof(char *) + call->inputBytes + call->stateBytes; - break; - } - case 2: { - SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); - len += sizeof(SUdfTeardownRequest); - break; - } - default: - break; - } - - char *bufBegin = malloc(len); - char *buf = bufBegin; - - //skip msgLen first - buf += sizeof(int32_t); - - *(int64_t *) buf = request->seqNum; - buf += sizeof(int64_t); - *(int8_t *) buf = request->type; - buf += sizeof(int8_t); - - switch (request->type) { - case 0: { - SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); - memcpy(buf, setup->udfName, 16); - buf += 16; - *(int8_t *) buf = setup->scriptType; - buf += sizeof(int8_t); - *(int8_t *) buf = setup->udfType; - buf += sizeof(int8_t); - *(int16_t *) buf = setup->pathSize; - buf += sizeof(int16_t); - memcpy(buf, setup->path, setup->pathSize); - buf += setup->pathSize; - break; - } - - case 1: { - SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); - *(int64_t *) buf = call->udfHandle; - buf += sizeof(int64_t); - *(int8_t *) buf = call->step; - buf += sizeof(int8_t); - *(int32_t *) buf = call->inputBytes; - buf += sizeof(int32_t); - memcpy(buf, call->input, call->inputBytes); - buf += call->inputBytes; - *(int32_t *) buf = call->stateBytes; - buf += sizeof(int32_t); - memcpy(buf, call->state, call->stateBytes); - buf += call->stateBytes; - break; - } - - case 2: { - SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); - *(int64_t *) buf = teardown->udfHandle; - buf += sizeof(int64_t); - break; - } - default: - break; - } - - request->msgLen = buf - bufBegin; - *(int32_t *) bufBegin = request->msgLen; - *pBuf = bufBegin; - *pBufLen = request->msgLen; - return 0; -} - -int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { - debugPrint(stdout, "%s\n", "decoding request"); - if (*(int32_t *) bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "dequest request error"); - return -1; - } - char *buf = bufMsg; - SUdfRequest *request = malloc(sizeof(SUdfRequest)); - request->subReq = NULL; - request->msgLen = *(int32_t *) (buf); - buf += sizeof(int32_t); - request->seqNum = *(int64_t *) (buf); - buf += sizeof(int64_t); - request->type = *(int8_t *) (buf); - buf += sizeof(int8_t); - - switch (request->type) { - case 0: { - SUdfSetupRequest *setup = malloc(sizeof(SUdfSetupRequest)); - - memcpy(setup->udfName, buf, 16); - buf += 16; - setup->scriptType = *(int8_t *) buf; - buf += sizeof(int8_t); - setup->udfType = *(int8_t *) buf; - buf += sizeof(int8_t); - setup->pathSize = *(int16_t *) buf; - buf += sizeof(int16_t); - setup->path = buf; - buf += setup->pathSize; - - request->subReq = setup; - break; - } - case 1: { - SUdfCallRequest *call = malloc(sizeof(SUdfCallRequest)); - - call->udfHandle = *(int64_t *) buf; - buf += sizeof(int64_t); - call->step = *(int8_t *) buf; - buf += sizeof(int8_t); - call->inputBytes = *(int32_t *) buf; - buf += sizeof(int32_t); - call->input = buf; - buf += call->inputBytes; - call->stateBytes = *(int32_t *) buf; - buf += sizeof(int32_t); - call->state = buf; - buf += call->stateBytes; - - request->subReq = call; - break; - } - - case 2: { - SUdfTeardownRequest *teardown = malloc(sizeof(SUdfTeardownRequest)); - - teardown->udfHandle = *(int64_t *) buf; - buf += sizeof(int64_t); - - request->subReq = teardown; - } - - } - if (buf - bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "decode request error"); - free(request->subReq); - free(request); - return -1; - } - *pRequest = request; - return 0; -} - -int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { - debugPrint(stdout, "%s\n", "encoding response"); - - int32_t len = sizeof(SUdfResponse) - sizeof(void *); - - switch (response->type) { - case 0: { - len += sizeof(SUdfSetupResponse); - break; - } - case 1: { - SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); - len += sizeof(SUdfCallResponse) - 2 * sizeof(char *) + - callResp->outputBytes + callResp->newStateBytes; - break; - } - case 2: { - len += sizeof(SUdfTeardownResponse); - break; - } - } - - char *bufBegin = malloc(len); - char *buf = bufBegin; - - //skip msgLen - buf += sizeof(int32_t); - - *(int64_t *) buf = response->seqNum; - buf += sizeof(int64_t); - *(int8_t *) buf = response->type; - buf += sizeof(int8_t); - *(int32_t *) buf = response->code; - buf += sizeof(int32_t); - - - switch (response->type) { - case 0: { - SUdfSetupResponse *setupResp = (SUdfSetupResponse *) (response->subRsp); - *(int64_t *) buf = setupResp->udfHandle; - buf += sizeof(int64_t); - break; - } - case 1: { - SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); - *(int32_t *) buf = callResp->outputBytes; - buf += sizeof(int32_t); - memcpy(buf, callResp->output, callResp->outputBytes); - buf += callResp->outputBytes; - - *(int32_t *) buf = callResp->newStateBytes; - buf += sizeof(int32_t); - memcpy(buf, callResp->newState, callResp->newStateBytes); - buf += callResp->newStateBytes; - break; - } - case 2: { - SUdfTeardownResponse *teardownResp = (SUdfTeardownResponse *) (response->subRsp); - break; - } - default: - break; - } - response->msgLen = buf - bufBegin; - *(int32_t *) bufBegin = response->msgLen; - *pBuf = bufBegin; - *pBufLen = response->msgLen; - return 0; -} - -int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { - debugPrint(stdout, "%s\n", "decoding response"); - - if (*(int32_t *) bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "can not decode response"); - return -1; - } - char *buf = bufMsg; - SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); - rsp->msgLen = *(int32_t *) buf; - buf += sizeof(int32_t); - rsp->seqNum = *(int64_t *) buf; - buf += sizeof(int64_t); - rsp->type = *(int8_t *) buf; - buf += sizeof(int8_t); - rsp->code = *(int32_t *) buf; - buf += sizeof(int32_t); - - switch (rsp->type) { - case 0: { - SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) malloc(sizeof(SUdfSetupResponse)); - setupRsp->udfHandle = *(int64_t *) buf; - buf += sizeof(int64_t); - rsp->subRsp = (char *) setupRsp; - break; - } - case 1: { - SUdfCallResponse *callRsp = (SUdfCallResponse *) malloc(sizeof(SUdfCallResponse)); - callRsp->outputBytes = *(int32_t *) buf; - buf += sizeof(int32_t); - - callRsp->output = buf; - buf += callRsp->outputBytes; - - callRsp->newStateBytes = *(int32_t *) buf; - buf += sizeof(int32_t); - - callRsp->newState = buf; - buf += callRsp->newStateBytes; - - rsp->subRsp = callRsp; - break; - } - case 2: { - SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) malloc(sizeof(SUdfTeardownResponse)); - rsp->subRsp = teardownRsp; - break; - } - default: - break; - } - if (buf - bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "can not decode response"); - free(rsp->subRsp); - free(rsp); - return -1; - } - *pResponse = rsp; - return 0; -} - -void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { - debugPrint(stderr, "Process exited with status %" PRId64 ", signal %d\n", exit_status, term_signal); - uv_close((uv_handle_t *) req, NULL); -} - -void onUdfcPipeClose(uv_handle_t *handle) { - SClientUvConn *conn = handle->data; - if (!udfTaskQueueIsEmpty(&conn->taskQueue)) { - SClientUvTaskNode *task = udfTaskQueueHeadTask(&conn->taskQueue); - task->errCode = 0; - uv_sem_post(&task->taskSem); - } - - free(conn->readBuf.buf); - free(conn); - free((uv_pipe_t *) handle); - -} - -int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { - debugPrint(stdout, "%s\n", "get uv task result"); - if (uvTask->type == UV_TASK_REQ_RSP) { - if (uvTask->rspBuf.base != NULL) { - SUdfResponse *rsp; - decodeResponse(uvTask->rspBuf.base, uvTask->rspBuf.len, &rsp); - task->errCode = rsp->code; - - switch (task->type) { - case UDF_TASK_SETUP: { - //TODO: copy - task->_setup.rsp = *(SUdfSetupResponse *) (rsp->subRsp); - break; - } - case UDF_TASK_CALL: { - task->_call.rsp = *(SUdfCallResponse *) (rsp->subRsp); - //TODO: copy - break; - } - case UDF_TASK_TEARDOWN: { - task->_teardown.rsp = *(SUdfTeardownResponse *) (rsp->subRsp); - //TODO: copy or not? - break; - } - default: { - break; - } - } - - // TODO: the call buffer is setup and freed by udf invocation - free(uvTask->rspBuf.base); - free(rsp->subRsp); - free(rsp); - } else { - task->errCode = uvTask->errCode; - } - } else if (uvTask->type == UV_TASK_CONNECT) { - task->errCode = uvTask->errCode; - } else if (uvTask->type == UV_TASK_DISCONNECT) { - task->errCode = uvTask->errCode; - } - return 0; -} - -void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint(stdout, "%s\n", "client allocate buffer to receive from pipe"); - SClientUvConn *conn = handle->data; - SClientConnBuf *connBuf = &conn->readBuf; - - int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); - if (connBuf->cap == 0) { - connBuf->buf = malloc(msgHeadSize); - if (connBuf->buf) { - connBuf->len = 0; - connBuf->cap = msgHeadSize; - connBuf->total = -1; - - buf->base = connBuf->buf; - buf->len = connBuf->cap; - } else { - //TODO: log error - buf->base = NULL; - buf->len = 0; - } - } else { - connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; - void *resultBuf = realloc(connBuf->buf, connBuf->cap); - if (resultBuf) { - connBuf->buf = resultBuf; - buf->base = connBuf->buf + connBuf->len; - buf->len = connBuf->cap - connBuf->len; - } else { - //TODO: log error free connBuf->buf - buf->base = NULL; - buf->len = 0; - } - } - - debugPrint(stdout, "\tconn buf cap - len - total : %d - %d - %d\n", connBuf->cap, connBuf->len, connBuf->total); - -} - -bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { - if (connBuf->total == -1 && connBuf->len >= sizeof(int32_t)) { - connBuf->total = *(int32_t *) (connBuf->buf); - } - if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { - return true; - } - return false; -} - -void udfcUvHandleRsp(SClientUvConn *conn) { - SClientConnBuf *connBuf = &conn->readBuf; - int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum - - if (udfTaskQueueIsEmpty(&conn->taskQueue)) { - //LOG error - return; - } - bool found = false; - SClientUvTaskNode *taskFound = NULL; - SClientUvTaskNode *task = udfTaskQueueNext(&conn->taskQueue); - while (task != &conn->taskQueue) { - if (task->seqNum == seqNum) { - if (found == false) { - found = true; - taskFound = task; - } else { - //LOG error; - continue; - } - } - task = udfTaskQueueNext(task); - } - - if (taskFound) { - taskFound->rspBuf = uv_buf_init(connBuf->buf, connBuf->len); - udfTaskQueueRemoveTask(taskFound); - uv_sem_post(&taskFound->taskSem); - } else { - //LOG error - } - connBuf->buf = NULL; - connBuf->total = -1; - connBuf->len = 0; - connBuf->cap = 0; -} - -void udfcUvHandleError(SClientUvConn *conn) { - uv_close((uv_handle_t *) conn->pipe, onUdfcPipeClose); -} - -void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint(stdout, "%s, nread: %zd\n", "client read from pipe", nread); - if (nread == 0) return; - - SClientUvConn *conn = client->data; - SClientConnBuf *connBuf = &conn->readBuf; - if (nread > 0) { - connBuf->len += nread; - if (isUdfcUvMsgComplete(connBuf)) { - udfcUvHandleRsp(conn); - } - - } - if (nread < 0) { - debugPrint(stderr, "\tclient read error: %s\n", uv_strerror(nread)); - if (nread == UV_EOF) { - //TODO: - } - udfcUvHandleError(conn); - } - -} - -void onUdfClientWrite(uv_write_t *write, int status) { - debugPrint(stderr, "%s\n", "after writing to pipe"); - SClientUvTaskNode *uvTask = write->data; - if (status == 0) { - uv_pipe_t *pipe = uvTask->pipe; - SClientUvConn *conn = pipe->data; - udfTaskQueueInsertTail(&conn->taskQueue, uvTask); - } else { - //TODO Log error; - } - debugPrint(stdout, "\tlength:%zu\n", uvTask->reqBuf.len); - free(write); - free(uvTask->reqBuf.base); -} - -void onUdfClientConnect(uv_connect_t *connect, int status) { - SClientUvTaskNode *uvTask = connect->data; - uvTask->errCode = status; - if (status != 0) { - //TODO: LOG error - } - uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); - free(connect); - uv_sem_post(&uvTask->taskSem); -} - -int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { - SClientUvTaskNode *uvTask = calloc(1, sizeof(SClientUvTaskNode)); - uvTask->type = uvTaskType; - - if (uvTaskType == UV_TASK_CONNECT) { - } else if (uvTaskType == UV_TASK_REQ_RSP) { - uvTask->pipe = task->session->udfSvcPipe; - SUdfRequest request; - request.type = task->type; - request.seqNum = gUdfTaskSeqNum++; - - if (task->type == UDF_TASK_SETUP) { - request.subReq = &task->_setup.req; - request.type = UDF_TASK_SETUP; - } else if (task->type == UDF_TASK_CALL) { - request.subReq = &task->_call.req; - request.type = UDF_TASK_CALL; - } else if (task->type == UDF_TASK_TEARDOWN) { - request.subReq = &task->_teardown.req; - request.type = UDF_TASK_TEARDOWN; - } else { - //TODO log and return error - } - char *buf = NULL; - int32_t bufLen = 0; - encodeRequest(&buf, &bufLen, &request); - uvTask->reqBuf = uv_buf_init(buf, bufLen); - uvTask->seqNum = request.seqNum; - } else if (uvTaskType == UV_TASK_DISCONNECT) { - uvTask->pipe = task->session->udfSvcPipe; - } - uv_sem_init(&uvTask->taskSem, 0); - - *pUvTask = uvTask; - return 0; -} - -int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint(stdout, "%s, %d\n", "queue uv task", uvTask->type); - - uv_mutex_lock(&gUdfTaskQueueMutex); - udfTaskQueueInsertTail(gUdfTaskQueue, uvTask); - uv_mutex_unlock(&gUdfTaskQueueMutex); - uv_async_send(&gUdfLoopTaskAync); - - uv_sem_wait(&uvTask->taskSem); - uv_sem_destroy(&uvTask->taskSem); - - return 0; -} - -int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint(stdout, "%s, type %d\n", "start uv task ", uvTask->type); - switch (uvTask->type) { - case UV_TASK_CONNECT: { - uv_pipe_t *pipe = malloc(sizeof(uv_pipe_t)); - uv_pipe_init(&gUdfdLoop, pipe, 0); - uvTask->pipe = pipe; - - SClientUvConn *conn = malloc(sizeof(SClientUvConn)); - conn->pipe = pipe; - conn->readBuf.len = 0; - conn->readBuf.cap = 0; - conn->readBuf.buf = 0; - conn->readBuf.total = -1; - udfTaskQueueInit(&conn->taskQueue); - - pipe->data = conn; - - uv_connect_t *connReq = malloc(sizeof(uv_connect_t)); - connReq->data = uvTask; - - uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); - break; - } - case UV_TASK_REQ_RSP: { - uv_pipe_t *pipe = uvTask->pipe; - uv_write_t *write = malloc(sizeof(uv_write_t)); - write->data = uvTask; - uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); - break; - } - case UV_TASK_DISCONNECT: { - SClientUvConn *conn = uvTask->pipe->data; - udfTaskQueueInsertTail(&conn->taskQueue, uvTask); - uv_close((uv_handle_t *) uvTask->pipe, onUdfcPipeClose); - break; - } - default: { - break; - } - } - - return 0; -} - -void udfClientAsyncCb(uv_async_t *async) { - SClientUvTaskNode node; - SClientUvTaskQueue q = &node; - udfTaskQueueInit(q); - - uv_mutex_lock(&gUdfTaskQueueMutex); - udfTaskQueueMove(gUdfTaskQueue, q); - uv_mutex_unlock(&gUdfTaskQueueMutex); - - while (!udfTaskQueueIsEmpty(q)) { - SClientUvTaskNode *task = udfTaskQueueHeadTask(q); - udfTaskQueueRemoveTask(task); - startUvUdfTask(task); - } - -} - -void udfStopAsyncCb(uv_async_t *async) { - uv_stop(&gUdfdLoop); - uv_loop_close(&gUdfdLoop); -} - -void startUdfd(void *argsThread) { - uv_loop_init(&gUdfdLoop); - -// uv_process_options_t options; -// static char path[256] = {0}; -// size_t cwdSize; -// uv_cwd(path, &cwdSize); -// strcat(path, "/udfd"); -// char* args[2] = {path, NULL}; -// options.args = args; -// options.file = path; -// options.exit_cb = onUdfdExit; -// -// int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); -// if (err != 0) { -// debugPrint(stderr, "can not spawn udfd. error: %s", uv_strerror(err)); -// } - - uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); - uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); - uv_mutex_init(&gUdfTaskQueueMutex); - udfTaskQueueInit(gUdfTaskQueue); - uv_barrier_wait(&gUdfInitBarrier); - uv_run(&gUdfdLoop, UV_RUN_DEFAULT); -} - -int32_t startUdfService() { - uv_barrier_init(&gUdfInitBarrier, 2); - uv_thread_create(&gUdfLoopThread, startUdfd, 0); - uv_barrier_wait(&gUdfInitBarrier); - return 0; -} - -int32_t stopUdfService() { - uv_barrier_destroy(&gUdfInitBarrier); - //uv_process_kill(&gUdfdProcess, SIGINT); - uv_async_send(&gUdfLoopStopAsync); - uv_mutex_destroy(&gUdfTaskQueueMutex); - uv_thread_join(&gUdfLoopThread); - return 0; -} - -int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { - SClientUvTaskNode *uvTask = NULL; - - createUdfcUvTask(task, uvTaskType, &uvTask); - queueUvUdfTask(uvTask); - udfcGetUvTaskResponseResult(task, uvTask); - if (uvTaskType == UV_TASK_CONNECT) { - task->session->udfSvcPipe = uvTask->pipe; - } - free(uvTask); - uvTask = NULL; - return task->errCode; -} - -int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { - debugPrint(stdout, "%s\n", "client setup udf"); - SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); - task->errCode = 0; - task->session = malloc(sizeof(SUdfUvSession)); - task->type = UDF_TASK_SETUP; - - SUdfSetupRequest *req = &task->_setup.req; - memcpy(req->udfName, udfInfo->udfName, 16); - req->path = udfInfo->path; - req->pathSize = strlen(req->path) + 1; - req->udfType = 0; - req->scriptType = 0; - - int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); - if (errCode != 0) { - //TODO: log error - return -1; - } - - udfcRunUvTask(task, UV_TASK_REQ_RSP); - - SUdfSetupResponse *rsp = &task->_setup.rsp; - task->session->severHandle = rsp->udfHandle; - *handle = task->session; - int32_t err = task->errCode; - free(task); - return err; -} - -int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newState, - int32_t *newStateSize, SSDataBlock *output) { - debugPrint(stdout, "%s\n", "client call udf"); - - SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); - task->errCode = 0; - task->session = (SUdfUvSession *) handle; - task->type = UDF_TASK_CALL; - - SUdfCallRequest *req = &task->_call.req; - - req->state = state; - req->stateBytes = stateSize; - req->inputBytes = input.size; - req->input = input.data; - req->udfHandle = task->session->severHandle; - req->step = step; - - udfcRunUvTask(task, UV_TASK_REQ_RSP); - - SUdfCallResponse *rsp = &task->_call.rsp; - *newState = rsp->newState; - *newStateSize = rsp->newStateBytes; - output->size = rsp->outputBytes; - output->data = rsp->output; - int32_t err = task->errCode; - free(task); - return err; -} - -int32_t teardownUdf(UdfHandle handle) { - debugPrint(stdout, "%s\n", "client teardown udf"); - - SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); - task->errCode = 0; - task->session = (SUdfUvSession *) handle; - task->type = UDF_TASK_TEARDOWN; - - SUdfTeardownRequest *req = &task->_teardown.req; - req->udfHandle = task->session->severHandle; - - udfcRunUvTask(task, UV_TASK_REQ_RSP); - - - SUdfTeardownResponse *rsp = &task->_teardown.rsp; - - int32_t err = task->errCode; - - udfcRunUvTask(task, UV_TASK_DISCONNECT); - - free(task->session); - free(task); - - return err; -} diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 1442914888..18c6004b25 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1,11 +1,32 @@ -#include -#include -#include -#include -#include -#include +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ -uv_loop_t *loop; +#include "uv.h" +#include "os.h" +#include "tudf.h" +#include "tudfInt.h" + +//TODO replaces them with qDebug +#define DEBUG +#ifdef DEBUG +#define debugPrint(...) fprintf(__VA_ARGS__) +#else +#define debugPrint(...) /**/ +#endif + +static uv_loop_t *loop; typedef struct SUdfdUvConn { uv_stream_t *client; @@ -21,10 +42,6 @@ typedef struct SUvUdfWork { uv_buf_t output; } SUvUdfWork; -typedef void (*TUdfNormalFunc)(char *state, int32_t stateSize, char **newstate, int32_t *newStateSize, - SSDataBlock input, SSDataBlock *output); - - typedef struct SUdf { int32_t refCount; @@ -32,8 +49,7 @@ typedef struct SUdf { int8_t type; uv_lib_t lib; - TUdfNormalFunc normalFunc; - + TUdfFunc normalFunc; } SUdf; //TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix @@ -43,10 +59,6 @@ typedef struct SUdfHandle { } SUdfHandle; -typedef void (*TUdfNormalFunc)(char *state, int32_t stateSize, char **newstate, int32_t *newStateSize, - SSDataBlock input, SSDataBlock *output); - - void udfdProcessRequest(uv_work_t *req) { SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data); SUdfRequest *request = NULL; @@ -67,8 +79,7 @@ void udfdProcessRequest(uv_work_t *req) { char normalFuncName[32] = {0}; strcpy(normalFuncName, setup->udfName); - strcat(normalFuncName, "_normal"); - //TODO error, + //TODO error, //TODO find all functions normal, init, destroy, normal, merge, finalize uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc)); @@ -104,10 +115,10 @@ void udfdProcessRequest(uv_work_t *req) { SUdf *udf = handle->udf; char *newState; int32_t newStateSize; - SSDataBlock input = {.data = call->input, .size= call->inputBytes}; - SSDataBlock output; + SUdfDataBlock input = {.data = call->input, .size= call->inputBytes}; + SUdfDataBlock output; //TODO: call different functions according to the step - udf->normalFunc(call->state, call->stateBytes, &newState, &newStateSize, input, &output); + udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output); SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); rsp->seqNum = request->seqNum; From b3ec7c8261d6562a389caabef2f36f4ccaabf6eb Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Mar 2022 17:30:50 +0800 Subject: [PATCH 05/15] sync refactor --- source/libs/sync/src/syncMain.c | 67 ++++-- source/libs/sync/src/syncRaftLog.c | 14 +- source/libs/sync/test/CMakeLists.txt | 14 ++ source/libs/sync/test/syncEntryTest.cpp | 2 +- .../libs/sync/test/syncReplicateLoadTest.cpp | 5 +- source/libs/sync/test/syncReplicateTest.cpp | 12 +- source/libs/sync/test/syncReplicateTest2.cpp | 10 +- source/libs/sync/test/syncReplicateTest3.cpp | 217 ++++++++++++++++++ 8 files changed, 304 insertions(+), 37 deletions(-) create mode 100644 source/libs/sync/test/syncReplicateTest3.cpp diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3080cbcd0d..58e02b3866 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -37,15 +37,16 @@ static int32_t tsNodeRefId = -1; // ------ local funciton --------- // enqueue message ---- -static void syncNodeEqPingTimer(void* param, void* tmrId); -static void syncNodeEqElectTimer(void* param, void* tmrId); -static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +static void syncNodeEqPingTimer(void* param, void* tmrId); +static void syncNodeEqElectTimer(void* param, void* tmrId); +static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); +static int32_t syncNodeEqNoop(SSyncNode* ths); +static int32_t syncNodeAppendNoop(SSyncNode* ths); // process message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg); -static int32_t syncNodeAppendNoop(SSyncNode* ths); // life cycle static void syncFreeNode(void* param); @@ -673,6 +674,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { // Raft 3.6.2 Committing entries from previous terms syncNodeAppendNoop(pSyncNode); + // syncNodeEqNoop(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { @@ -807,6 +809,47 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } } +static int32_t syncNodeEqNoop(SSyncNode* ths) { + int32_t ret = 0; + assert(ths->state == TAOS_SYNC_STATE_LEADER); + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + assert(pEntry != NULL); + + uint32_t entryLen; + char* serialized = syncEntrySerialize(pEntry, &entryLen); + SyncClientRequest* pSyncMsg = syncClientRequestBuild(entryLen); + assert(pSyncMsg->dataLen == entryLen); + memcpy(pSyncMsg->data, serialized, entryLen); + + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); + ths->FpEqMsg(ths->queue, &rpcMsg); + + free(serialized); + syncClientRequestDestroy(pSyncMsg); + + return ret; +} + +static int32_t syncNodeAppendNoop(SSyncNode* ths) { + int32_t ret = 0; + + SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncTerm term = ths->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); + assert(pEntry != NULL); + + if (ths->state == TAOS_SYNC_STATE_LEADER) { + ths->pLogStore->appendEntry(ths->pLogStore, pEntry); + syncNodeReplicate(ths); + } + + return ret; +} + // on message ---- static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { int32_t ret = 0; @@ -881,22 +924,6 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg return ret; } -static int32_t syncNodeAppendNoop(SSyncNode* ths) { - int32_t ret = 0; - - SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; - SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index); - assert(pEntry != NULL); - - if (ths->state == TAOS_SYNC_STATE_LEADER) { - ths->pLogStore->appendEntry(ths->pLogStore, pEntry); - syncNodeReplicate(ths); - } - - return ret; -} - static void syncFreeNode(void* param) { SSyncNode* pNode = param; syncNodePrint2((char*)"==syncFreeNode==", pNode); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index cad2d5da41..a788b91c01 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -48,14 +48,18 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - assert(pEntry->index == logStoreLastIndex(pLogStore) + 1); + SyncIndex lastIndex = logStoreLastIndex(pLogStore); + assert(pEntry->index == lastIndex + 1); uint32_t len; char* serialized = syncEntrySerialize(pEntry, &len); assert(serialized != NULL); - int code; - code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len); - assert(code == 0); + int code = 0; + /* + code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len); + assert(code == 0); + */ + assert(walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len) == 0); walFsync(pWal, true); free(serialized); @@ -69,7 +73,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - walReadWithHandle(pWalHandle, index); + assert(walReadWithHandle(pWalHandle, index) == 0); pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); assert(pEntry != NULL); diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index dcf380e7e4..7341565b1d 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(syncEncodeTest "") add_executable(syncWriteTest "") add_executable(syncReplicateTest "") add_executable(syncReplicateTest2 "") +add_executable(syncReplicateTest3 "") add_executable(syncReplicateLoadTest "") add_executable(syncRefTest "") add_executable(syncLogStoreCheck "") @@ -183,6 +184,10 @@ target_sources(syncReplicateTest2 PRIVATE "syncReplicateTest2.cpp" ) +target_sources(syncReplicateTest3 + PRIVATE + "syncReplicateTest3.cpp" +) target_sources(syncReplicateLoadTest PRIVATE "syncReplicateLoadTest.cpp" @@ -377,6 +382,11 @@ target_include_directories(syncReplicateTest2 "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncReplicateTest3 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncReplicateLoadTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" @@ -538,6 +548,10 @@ target_link_libraries(syncReplicateTest2 sync gtest_main ) +target_link_libraries(syncReplicateTest3 + sync + gtest_main +) target_link_libraries(syncReplicateLoadTest sync gtest_main diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index 4acdb6052e..2c8433426a 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -90,7 +90,7 @@ int main(int argc, char** argv) { test1(); test2(); - test3(); + test3(); test4(); return 0; diff --git a/source/libs/sync/test/syncReplicateLoadTest.cpp b/source/libs/sync/test/syncReplicateLoadTest.cpp index d53ceca473..b2d3f1a98d 100644 --- a/source/libs/sync/test/syncReplicateLoadTest.cpp +++ b/source/libs/sync/test/syncReplicateLoadTest.cpp @@ -178,9 +178,10 @@ int main(int argc, char **argv) { while (1) { sTrace( "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "electTimerMS:%d, commitIndex:%ld", pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 47399eeb3c..09aec6534f 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -183,18 +183,20 @@ int main(int argc, char **argv) { taosMsleep(1000); sTrace( - "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, - gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, + gSyncNode->commitIndex); } while (1) { sTrace( "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "electTimerMS:%d, commitIndex:%ld", gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, - gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS, + gSyncNode->commitIndex); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncReplicateTest2.cpp b/source/libs/sync/test/syncReplicateTest2.cpp index 09dbc0e2ed..8739a8d12f 100644 --- a/source/libs/sync/test/syncReplicateTest2.cpp +++ b/source/libs/sync/test/syncReplicateTest2.cpp @@ -185,17 +185,19 @@ int main(int argc, char **argv) { sTrace( "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "electTimerMS:%d, commitIndex:%ld", pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); } while (1) { sTrace( "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " - "electTimerMS:%d", + "electTimerMS:%d, commitIndex:%ld", pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncReplicateTest3.cpp b/source/libs/sync/test/syncReplicateTest3.cpp new file mode 100644 index 0000000000..ad2172ccd2 --- /dev/null +++ b/source/libs/sync/test/syncReplicateTest3.cpp @@ -0,0 +1,217 @@ +#define ALLOW_FORBID_FUNC + +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM *pFsm; +SWal * pWal; + +void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, index, isWeak, + code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, bool isWeak, int32_t code, + ESyncState state) { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + pFsm, index, isWeak, code, state, syncUtilState2String(state)); + syncRpcMsgPrint2(logBuf, (SRpcMsg *)pMsg); +} + +void initFsm() { + pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm->FpCommitCb = CommitCb; + pFsm->FpPreCommitCb = PreCommitCb; + pFsm->FpRollBackCb = RollBackCb; +} + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./replicate2_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./replicate2_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + // pSyncNode->hbBaseLine = 500; + // pSyncNode->electBaseLine = 1500; + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->FpOnSyncClientRequest = pSyncNode->FpOnClientRequest; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode *pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char *s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +SRpcMsg *step0(int i) { + SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + memset(pMsg, 0, sizeof(SRpcMsg)); + pMsg->msgType = 9999; + pMsg->contLen = 128; + pMsg->pCont = malloc(pMsg->contLen); + snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); + return pMsg; +} + +SyncClientRequest *step1(const SRpcMsg *pMsg) { + SyncClientRequest *pRetMsg = syncClientRequestBuild2(pMsg, 123, true); + return pRetMsg; +} + +int main(int argc, char **argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + void logTest(); + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int recordCount = 100; + if (argc >= 3) { + recordCount = atoi(argv[2]); + } + + int sleepMS = 10; + if (argc >= 4) { + sleepMS = atoi(argv[3]); + } + + int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + initFsm(); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode *pSyncNode = (SSyncNode *)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char *)"", pSyncNode); + initRaftId(pSyncNode); + + for (int i = 0; i < recordCount; ++i) { + // step0 + SRpcMsg *pMsg0 = step0(i); + syncRpcMsgPrint2((char *)"==step0==", pMsg0); + + syncPropose(rid, pMsg0, true); + taosMsleep(sleepMS); + + free(pMsg0); + + sTrace( + "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + } + + while (1) { + sTrace( + "replicate sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " + "electTimerMS:%d, commitIndex:%ld", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, + pSyncNode->commitIndex); + taosMsleep(1000); + } + + return 0; +} From 14031ca6681042d2742447462e0187dc89fd9322 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 24 Mar 2022 16:59:43 +0800 Subject: [PATCH 06/15] add simple test --- source/libs/function/CMakeLists.txt | 22 +++++++++++++- source/libs/function/test/runUdf.c | 45 +++++++++++++++++++++++++++++ source/libs/function/test/udf1.c | 20 +++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 source/libs/function/test/runUdf.c create mode 100644 source/libs/function/test/udf1.c diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index f44cbe6d63..d69383f69b 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -12,6 +12,25 @@ target_link_libraries( PRIVATE os util common nodes ) +add_executable(runUdf test/runUdf.c) +target_include_directories( + runUdf + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + runUdf + PRIVATE os util common nodes function +) + +add_library(udf1 MODULE test/udf1.c) +target_include_directories( + udf1 + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +#SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) add_executable(udfd src/udfd.c) target_include_directories( udfd @@ -22,4 +41,5 @@ target_include_directories( target_link_libraries( udfd PRIVATE os util common nodes function -) \ No newline at end of file +) + diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c new file mode 100644 index 0000000000..b7d651e55e --- /dev/null +++ b/source/libs/function/test/runUdf.c @@ -0,0 +1,45 @@ +#include +#include +#include + +#include "uv.h" +#include "tudf.h" + +int main(int argc, char *argv[]) { + startUdfService(); + uv_sleep(1000); + char path[256] = {0}; + size_t cwdSize = 256; + int err = uv_cwd(path, &cwdSize); + if (err != 0) { + fprintf(stderr, "err cwd: %s\n", uv_strerror(err)); + return err; + } + fprintf(stdout, "current working directory:%s\n", path); + strcat(path, "/libudf1.so"); + SUdfInfo udfInfo = {.udfName="udf1", .path=path}; + + UdfHandle handle; + setupUdf(&udfInfo, &handle); + + //char state[5000000] = "state"; + //char input[5000000] = "input"; + int dataSize = 500; + int callCount = 2; + if (argc > 1) dataSize = atoi(argv[1]); + if (argc > 2) callCount = atoi(argv[2]); + char *state = malloc(dataSize); + char *input = malloc(dataSize); + SUdfDataBlock blockInput = {.data = input, .size = dataSize}; + SUdfDataBlock blockOutput; + char* newState; + int32_t newStateSize; + for (int l = 0; l < callCount; ++l) { + callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput); + } + free(state); + free(input); + teardownUdf(handle); + + stopUdfService(); +} diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c new file mode 100644 index 0000000000..15e96e3bd1 --- /dev/null +++ b/source/libs/function/test/udf1.c @@ -0,0 +1,20 @@ +#include +#include +#include + +#include "tudf.h" + +void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, + char **newState, int32_t *newStateSize, SUdfDataBlock *output) { + fprintf(stdout, "%s, step:%d\n", "udf function called", step); + char *newStateBuf = malloc(stateSize); + memcpy(newStateBuf, state, stateSize); + *newState = newStateBuf; + *newStateSize = stateSize; + + char *outputBuf = malloc(input.size); + memcpy(outputBuf, input.data, input.size); + output->data = outputBuf; + output->size = input.size; + return; +} From 61f227816d2fd7052e73bbbdd675aa3909a2a3b2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Mar 2022 18:41:20 +0800 Subject: [PATCH 07/15] sync refactor --- source/libs/sync/src/syncRaftLog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a788b91c01..50b08d0e87 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -87,7 +87,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - walRollback(pWal, fromIndex); + assert(walRollback(pWal, fromIndex) == 0); return 0; // to avoid compiler error } From 8058a0f1a700b8886d64a4ed17129bb2b4929406 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Mar 2022 18:32:14 +0800 Subject: [PATCH 08/15] xshm --- source/dnode/mgmt/container/inc/dnd.h | 9 ++------- source/dnode/mgmt/container/inc/dndInt.h | 2 +- source/dnode/mgmt/container/src/dndInt.c | 15 +++++++-------- source/dnode/mgmt/container/src/dndObj.c | 2 +- source/dnode/mgmt/container/src/dndTransport.c | 2 +- source/dnode/mgmt/mnode/src/mmInt.c | 1 + 6 files changed, 13 insertions(+), 18 deletions(-) diff --git a/source/dnode/mgmt/container/inc/dnd.h b/source/dnode/mgmt/container/inc/dnd.h index 6805c4a232..c89448ab3c 100644 --- a/source/dnode/mgmt/container/inc/dnd.h +++ b/source/dnode/mgmt/container/inc/dnd.h @@ -19,7 +19,6 @@ #include "os.h" #include "cJSON.h" -#include "monitor.h" #include "tcache.h" #include "tcrc32c.h" #include "tdatablock.h" @@ -36,8 +35,7 @@ #include "tworker.h" #include "dnode.h" -#include "tfs.h" -#include "wal.h" +#include "monitor.h" #ifdef __cplusplus extern "C" { @@ -53,7 +51,6 @@ extern "C" { typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus; -typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType; typedef struct SMgmtFp SMgmtFp; @@ -127,7 +124,6 @@ typedef struct SDnode { bool dropped; EDndStatus status; EDndEvent event; - EProcType procType; SStartupReq startup; TdFilePtr pLockFile; STransMgmt trans; @@ -138,7 +134,7 @@ EDndStatus dndGetStatus(SDnode *pDnode); void dndSetStatus(SDnode *pDnode, EDndStatus stat); SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId); -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); void dndSendMonitorReport(SDnode *pDnode); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); @@ -146,7 +142,6 @@ int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg) void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); - int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h index 9ae70874fe..0a25a989c1 100644 --- a/source/dnode/mgmt/container/inc/dndInt.h +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -34,7 +34,7 @@ int32_t dndInit(); void dndCleanup(); const char *dndStatStr(EDndStatus stat); void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); -TdFilePtr dndCheckRunning(char *dataDir); +TdFilePtr dndCheckRunning(const char *dataDir); void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); // dndMsg.c diff --git a/source/dnode/mgmt/container/src/dndInt.c b/source/dnode/mgmt/container/src/dndInt.c index 8ad4351a88..ab19415168 100644 --- a/source/dnode/mgmt/container/src/dndInt.c +++ b/source/dnode/mgmt/container/src/dndInt.c @@ -15,11 +15,12 @@ #define _DEFAULT_SOURCE #include "dndInt.h" +#include "wal.h" static int8_t once = DND_ENV_INIT; int32_t dndInit() { - dDebug("start to init dnode env"); + dInfo("start to init dnode env"); if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { terrno = TSDB_CODE_REPEAT_INIT; dError("failed to init dnode env since %s", terrstr()); @@ -52,7 +53,7 @@ int32_t dndInit() { } void dndCleanup() { - dDebug("start to cleanup dnode env"); + dInfo("start to cleanup dnode env"); if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { dError("dnode env is already cleaned up"); return; @@ -92,7 +93,7 @@ const char *dndStatStr(EDndStatus status) { } } -void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { SStartupReq *pStartup = &pDnode->startup; tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); @@ -104,21 +105,21 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); } -TdFilePtr dndCheckRunning(char *dataDir) { +TdFilePtr dndCheckRunning(const char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + dError("failed to lock file:%s since %s", filepath, terrstr()); return NULL; } int32_t ret = taosLockFile(pFile); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s, quit", filepath, terrstr()); + dError("failed to lock file:%s since %s", filepath, terrstr()); taosCloseFile(&pFile); return NULL; } @@ -129,12 +130,10 @@ TdFilePtr dndCheckRunning(char *dataDir) { void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); dndGetStartup(pDnode, pStartup); dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; rpcSendResponse(&rpcRsp); } diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c index 15acbfccad..d618f4e503 100644 --- a/source/dnode/mgmt/container/src/dndObj.c +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -58,7 +58,7 @@ static void dndClearMemory(SDnode *pDnode) { SDnode *dndCreate(const SDnodeOpt *pOption) { dInfo("start to create dnode object"); int32_t code = -1; - char path[PATH_MAX]; + char path[PATH_MAX] = {0}; SDnode *pDnode = NULL; pDnode = calloc(1, sizeof(SDnode)); diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c index 25edb82baa..0f6aee4fc8 100644 --- a/source/dnode/mgmt/container/src/dndTransport.c +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -271,7 +271,7 @@ int32_t dndInitMsgHandle(SDnode *pDnode) { int32_t vgId = pWrapper->msgVgIds[msgIndex]; if (msgFp == NULL) continue; - dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); + // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; if (vgId == QND_VGID) { diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c index aa348ae879..81d1abc692 100644 --- a/source/dnode/mgmt/mnode/src/mmInt.c +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" +#include "wal.h" static bool mmDeployRequired(SDnode *pDnode) { if (pDnode->dnodeId > 0) return false; From 83ecb8dd309b4ea2d18af1948a355618d88492b1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 24 Mar 2022 19:40:37 +0800 Subject: [PATCH 09/15] sync refactor --- source/libs/sync/src/syncIO.c | 33 ++++++++++++++++-------------- source/libs/sync/src/syncRaftLog.c | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index b05d2e397d..da54aa9668 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -75,6 +75,7 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) { syncRpcMsgPrint2(logBuf, pMsg); pMsg->handle = NULL; + pMsg->noResp = 1; rpcSendRequest(clientRpc, pEpSet, pMsg, NULL); return ret; } @@ -234,9 +235,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -324,19 +325,21 @@ static void *syncIOConsumerFunc(void *param) { taosGetQitem(qall, (void **)&pRpcMsg); rpcFreeCont(pRpcMsg->pCont); - if (pRpcMsg->handle != NULL) { - int msgSize = 32; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.msgType = SYNC_RESPONSE; - rpcMsg.pCont = rpcMallocCont(msgSize); - rpcMsg.contLen = msgSize; - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); - rpcMsg.handle = pRpcMsg->handle; - rpcMsg.code = 0; + /* + if (pRpcMsg->handle != NULL) { + int msgSize = 32; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.msgType = SYNC_RESPONSE; + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply"); + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; - syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); - rpcSendResponse(&rpcMsg); - } + syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg); + rpcSendResponse(&rpcMsg); + } + */ taosFreeQitem(pRpcMsg); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 50b08d0e87..b28f899aa9 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -111,7 +111,7 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - walCommit(pWal, index); + assert(walCommit(pWal, index) == 0); return 0; // to avoid compiler error } From 4fa5121c7245eb7a8fef3c8739698f5ffe37bcf6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Mar 2022 19:59:37 +0800 Subject: [PATCH 10/15] adjust refcount --- source/common/src/tglobal.c | 4 +++ source/dnode/mgmt/bnode/src/bmWorker.c | 6 +++- source/dnode/mgmt/container/inc/dnd.h | 15 ++++---- source/dnode/mgmt/container/inc/dndInt.h | 4 --- source/dnode/mgmt/container/src/dndMonitor.c | 33 ++++++++++++++---- source/dnode/mgmt/container/src/dndMsg.c | 2 +- .../dnode/mgmt/container/src/dndTransport.c | 34 +++++++++++++++---- source/dnode/mgmt/dnode/src/dmFile.c | 2 +- source/dnode/mgmt/dnode/src/dmInt.c | 4 +-- source/dnode/mgmt/dnode/src/dmMsg.c | 8 +++-- source/dnode/mgmt/dnode/src/dmWorker.c | 4 ++- source/dnode/mgmt/mnode/src/mmInt.c | 2 +- source/dnode/mgmt/mnode/src/mmWorker.c | 2 ++ source/dnode/mgmt/qnode/src/qmWorker.c | 2 ++ source/dnode/mgmt/snode/src/smWorker.c | 2 ++ source/dnode/mgmt/vnode/src/vmInt.c | 8 ++--- source/dnode/mgmt/vnode/src/vmWorker.c | 4 +-- 17 files changed, 97 insertions(+), 39 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 506e017deb..179a50c422 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -180,6 +180,10 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) { memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); if (pCfg->level == 0 && pCfg->primary == 1) { tstrncpy(tsDataDir, pCfg->dir, PATH_MAX); + if (taosMkDir(tsDataDir) != 0) { + uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); + return -1; + } } if (taosMkDir(pCfg->dir) != 0) { uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); diff --git a/source/dnode/mgmt/bnode/src/bmWorker.c b/source/dnode/mgmt/bnode/src/bmWorker.c index 2099787c0d..7698aa9dbd 100644 --- a/source/dnode/mgmt/bnode/src/bmWorker.c +++ b/source/dnode/mgmt/bnode/src/bmWorker.c @@ -77,7 +77,11 @@ int32_t bmStartWorker(SBnodeMgmt *pMgmt) { return -1; } + dDebug("bnode workers are initialized"); return 0; } -void bmStopWorker(SBnodeMgmt *pMgmt) { tMultiWorkerCleanup(&pMgmt->writeWorker); } +void bmStopWorker(SBnodeMgmt *pMgmt) { + tMultiWorkerCleanup(&pMgmt->writeWorker); + dDebug("bnode workers are closed"); +} diff --git a/source/dnode/mgmt/container/inc/dnd.h b/source/dnode/mgmt/container/inc/dnd.h index c89448ab3c..f6c8897f64 100644 --- a/source/dnode/mgmt/container/inc/dnd.h +++ b/source/dnode/mgmt/container/inc/dnd.h @@ -130,12 +130,11 @@ typedef struct SDnode { SMgmtWrapper wrappers[NODE_MAX]; } SDnode; -EDndStatus dndGetStatus(SDnode *pDnode); -void dndSetStatus(SDnode *pDnode, EDndStatus stat); -SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); -void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId); -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -void dndSendMonitorReport(SDnode *pDnode); +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); +void dndSendMonitorReport(SDnode *pDnode); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg); @@ -145,6 +144,10 @@ int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); +void dndReleaseWrapper(SMgmtWrapper *pWrapper); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h index 0a25a989c1..d10835b67f 100644 --- a/source/dnode/mgmt/container/inc/dndInt.h +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -50,10 +50,6 @@ SDnode *dndCreate(const SDnodeOpt *pOption); void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); -SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); -int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); -void dndReleaseWrapper(SMgmtWrapper *pWrapper); - // dndTransport.c int32_t dndInitServer(SDnode *pDnode); void dndCleanupServer(SDnode *pDnode); diff --git a/source/dnode/mgmt/container/src/dndMonitor.c b/source/dnode/mgmt/container/src/dndMonitor.c index ef3db387de..c01f840794 100644 --- a/source/dnode/mgmt/container/src/dndMonitor.c +++ b/source/dnode/mgmt/container/src/dndMonitor.c @@ -22,7 +22,12 @@ static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); pInfo->tempdir.size = tsTempSpace.size; - return vmMonitorTfsInfo(dndAcquireWrapper(pDnode, VNODES), pInfo); + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + vmMonitorTfsInfo(pWrapper, pInfo); + dndReleaseWrapper(pWrapper); + } + return 0; } static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { @@ -45,8 +50,17 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); - vmMonitorVnodeReqs(dndAcquireWrapper(pDnode, VNODES), pInfo); - pInfo->has_mnode = (dndAcquireWrapper(pDnode, MNODE)->required); + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + vmMonitorVnodeReqs(pWrapper, pInfo); + dndReleaseWrapper(pWrapper); + } + + pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + pInfo->has_mnode = pWrapper->required; + dndReleaseWrapper(pWrapper); + } } void dndSendMonitorReport(SDnode *pDnode) { @@ -63,10 +77,15 @@ void dndSendMonitorReport(SDnode *pDnode) { SMonClusterInfo clusterInfo = {0}; SMonVgroupInfo vgroupInfo = {0}; SMonGrantInfo grantInfo = {0}; - if (mmMonitorMnodeInfo(dndAcquireWrapper(pDnode, MNODE), &clusterInfo, &vgroupInfo, &grantInfo) == 0) { - monSetClusterInfo(pMonitor, &clusterInfo); - monSetVgroupInfo(pMonitor, &vgroupInfo); - monSetGrantInfo(pMonitor, &grantInfo); + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + if (mmMonitorMnodeInfo(pWrapper, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { + monSetClusterInfo(pMonitor, &clusterInfo); + monSetVgroupInfo(pMonitor, &vgroupInfo); + monSetGrantInfo(pMonitor, &grantInfo); + } + dndReleaseWrapper(pWrapper); } SMonDnodeInfo dnodeInfo = {0}; diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c index 21e9cc71a5..37ff4ebc05 100644 --- a/source/dnode/mgmt/container/src/dndMsg.c +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -20,8 +20,8 @@ static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); if (pWrapper != NULL) { dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); + dndReleaseWrapper(pWrapper); } - dndReleaseWrapper(pWrapper); } static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c index 0f6aee4fc8..4acb1f459e 100644 --- a/source/dnode/mgmt/container/src/dndTransport.c +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -146,9 +146,14 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { STransMgmt *pMgmt = &pDnode->trans; + SEpSet epSet = {0}; + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + dndReleaseWrapper(pWrapper); + } - SEpSet epSet = {0}; - dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); } @@ -182,9 +187,14 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char return 0; } - if (mmGetUserAuth(dndAcquireWrapper(pDnode, MNODE), user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) { + dndReleaseWrapper(pWrapper); + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + return 0; + } + dndReleaseWrapper(pWrapper); } if (terrno != TSDB_CODE_APP_NOT_READY) { @@ -328,7 +338,12 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { SDnode *pDnode = pWrapper->pDnode; STransMgmt *pTrans = &pDnode->trans; SEpSet epSet = {0}; - dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet); + + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); + if (pWrapper != NULL) { + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + dndReleaseWrapper(pWrapper); + } return dndSendRpcReq(pTrans, &epSet, pReq); } } @@ -336,7 +351,12 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { if (pRsp->code == TSDB_CODE_APP_NOT_READY) { SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); - dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); + if (pDnodeWrapper != NULL) { + dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); + dndReleaseWrapper(pDnodeWrapper); + } else { + rpcSendResponse(pRsp); + } } else { rpcSendResponse(pRsp); } diff --git a/source/dnode/mgmt/dnode/src/dmFile.c b/source/dnode/mgmt/dnode/src/dmFile.c index aa2247bcd5..9acfc2960f 100644 --- a/source/dnode/mgmt/dnode/src/dmFile.c +++ b/source/dnode/mgmt/dnode/src/dmFile.c @@ -209,7 +209,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) { } pMgmt->updateTime = taosGetTimestampMs(); - dDebug("successed to write %s", file); + dDebug("successed to write %s", realfile); return 0; } diff --git a/source/dnode/mgmt/dnode/src/dmInt.c b/source/dnode/mgmt/dnode/src/dmInt.c index 1746cbe6e1..8aa87cb7bc 100644 --- a/source/dnode/mgmt/dnode/src/dmInt.c +++ b/source/dnode/mgmt/dnode/src/dmInt.c @@ -74,14 +74,14 @@ void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) { } static int32_t dmStart(SMgmtWrapper *pWrapper) { - dDebug("dnode mgmt start to run"); + dDebug("dnode-mgmt start to run"); return dmStartThread(pWrapper->pMgmt); } int32_t dmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt)); - dInfo("dnode-mgmt is initialized"); + dInfo("dnode-mgmt start to init"); pDnode->dnodeId = 0; pDnode->dropped = 0; diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c index 1df0798310..836817e772 100644 --- a/source/dnode/mgmt/dnode/src/dmMsg.c +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -41,8 +41,12 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); - req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); - vmMonitorVnodeLoads(dndAcquireWrapper(pDnode, VNODES), req.pVloads); + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); + if (pWrapper != NULL) { + req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); + vmMonitorVnodeLoads(pWrapper, req.pVloads); + dndReleaseWrapper(pWrapper); + } int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); diff --git a/source/dnode/mgmt/dnode/src/dmWorker.c b/source/dnode/mgmt/dnode/src/dmWorker.c index b62c18655a..be08aad7cb 100644 --- a/source/dnode/mgmt/dnode/src/dmWorker.c +++ b/source/dnode/mgmt/dnode/src/dmWorker.c @@ -114,6 +114,7 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { return -1; } + dDebug("dnode workers are initialized"); return 0; } @@ -136,6 +137,7 @@ void dmStopWorker(SDnodeMgmt *pMgmt) { taosDestoryThread(pMgmt->threadId); pMgmt->threadId = NULL; } + dDebug("dnode workers are closed"); } int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { @@ -144,6 +146,6 @@ int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { pWorker = &pMgmt->statusWorker; } - dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); return taosWriteQitem(pWorker->queue, pMsg); } diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c index 81d1abc692..1f60007be1 100644 --- a/source/dnode/mgmt/mnode/src/mmInt.c +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -227,7 +227,7 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) { } static int32_t mmStart(SMgmtWrapper *pWrapper) { - dDebug("mnode mgmt start to run"); + dDebug("mnode-mgmt start to run"); SMnodeMgmt *pMgmt = pWrapper->pMgmt; return mndStart(pMgmt->pMnode); } diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c index b9a3a4f14c..d6b150106d 100644 --- a/source/dnode/mgmt/mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -108,6 +108,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { return -1; } + dDebug("mnode workers are initialized"); return 0; } @@ -115,4 +116,5 @@ void mmStopWorker(SMnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->readWorker); tSingleWorkerCleanup(&pMgmt->writeWorker); tSingleWorkerCleanup(&pMgmt->syncWorker); + dDebug("mnode workers are closed"); } diff --git a/source/dnode/mgmt/qnode/src/qmWorker.c b/source/dnode/mgmt/qnode/src/qmWorker.c index fff469a902..aa4da82790 100644 --- a/source/dnode/mgmt/qnode/src/qmWorker.c +++ b/source/dnode/mgmt/qnode/src/qmWorker.c @@ -132,10 +132,12 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { return -1; } + dDebug("qnode workers are initialized"); return 0; } void qmStopWorker(SQnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->queryWorker); tSingleWorkerCleanup(&pMgmt->fetchWorker); + dDebug("qnode workers are closed"); } diff --git a/source/dnode/mgmt/snode/src/smWorker.c b/source/dnode/mgmt/snode/src/smWorker.c index ceec6b82c3..18ce71b8e7 100644 --- a/source/dnode/mgmt/snode/src/smWorker.c +++ b/source/dnode/mgmt/snode/src/smWorker.c @@ -80,6 +80,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { return -1; } + dDebug("snode workers are initialized"); return 0; } @@ -90,6 +91,7 @@ void smStopWorker(SSnodeMgmt *pMgmt) { } taosArrayDestroy(pMgmt->uniqueWorkers); tSingleWorkerCleanup(&pMgmt->sharedWorker); + dDebug("snode workers are closed"); } static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c index 746fcd4855..71f8f23b24 100644 --- a/source/dnode/mgmt/vnode/src/vmInt.c +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -257,14 +257,14 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; if (pMgmt == NULL) return; - dInfo("vnodes-mgmt start to cleanup"); + dInfo("vnode-mgmt start to cleanup"); vmCloseVnodes(pMgmt); vmStopWorker(pMgmt); vnodeCleanup(); // walCleanUp(); free(pMgmt); pWrapper->pMgmt = NULL; - dInfo("vnodes-mgmt is cleaned up"); + dInfo("vnode-mgmt is cleaned up"); } static int32_t vmInit(SMgmtWrapper *pWrapper) { @@ -272,7 +272,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt)); int32_t code = -1; - dInfo("vnodes-mgmt start to init"); + dInfo("vnode-mgmt start to init"); if (pMgmt == NULL) goto _OVER; pMgmt->path = pWrapper->path; @@ -312,7 +312,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { } if (vmOpenVnodes(pMgmt) != 0) { - dError("failed to open vnodes since %s", terrstr()); + dError("failed to open vnode since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index 6c7d513c58..12e15e5adf 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -356,7 +356,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { return -1; } - dDebug("vnode workers is initialized"); + dDebug("vnode workers are initialized"); return 0; } @@ -366,5 +366,5 @@ void vmStopWorker(SVnodesMgmt *pMgmt) { tQWorkerCleanup(&pMgmt->queryPool); tWWorkerCleanup(&pMgmt->writePool); tWWorkerCleanup(&pMgmt->syncPool); - dDebug("vnode workers is closed"); + dDebug("vnode workers are closed"); } From 2d255d179bbf7b440cd2bc7ef41de8c1be48011f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Mar 2022 20:05:25 +0800 Subject: [PATCH 11/15] refcount --- source/dnode/mgmt/container/src/dndExec.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index d5c882398e..6d07ca1898 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -50,16 +50,23 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) { } void dndCloseNode(SMgmtWrapper *pWrapper) { + dDebug("node:%s, start to close", pWrapper->name); taosWLockLatch(&pWrapper->latch); if (pWrapper->deployed) { (*pWrapper->fp.closeFp)(pWrapper); pWrapper->deployed = false; } + taosWUnLockLatch(&pWrapper->latch); + + while (pWrapper->refCount > 0) { + taosMsleep(10); + } + if (pWrapper->pProc) { taosProcCleanup(pWrapper->pProc); pWrapper->pProc = NULL; } - taosWUnLockLatch(&pWrapper->latch); + dDebug("node:%s, has been closed", pWrapper->name); } static int32_t dndRunInSingleProcess(SDnode *pDnode) { From abedeb23d65b0b23ab07930ffb74933b2f5141f4 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 25 Mar 2022 08:34:46 +0800 Subject: [PATCH 12/15] libuv dependency refactoring --- source/libs/function/CMakeLists.txt | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index d69383f69b..6305d242ac 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -3,30 +3,37 @@ list(REMOVE_ITEM FUNCTION_SRC src/udfd.c) add_library(function STATIC ${FUNCTION_SRC}) target_include_directories( function - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( function + PUBLIC uv_a PRIVATE os util common nodes ) add_executable(runUdf test/runUdf.c) target_include_directories( runUdf - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( runUdf + PUBLIC uv_a PRIVATE os util common nodes function ) add_library(udf1 MODULE test/udf1.c) target_include_directories( udf1 - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) @@ -34,12 +41,15 @@ target_include_directories( add_executable(udfd src/udfd.c) target_include_directories( udfd - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/contrib/libuv/include" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( udfd + PUBLIC uv_a PRIVATE os util common nodes function ) From 16b44c9e28574561c33ae8e2b957a4fdcd516a30 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 25 Mar 2022 10:37:26 +0800 Subject: [PATCH 13/15] sync refactor --- source/libs/sync/src/syncAppendEntries.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 0a83e09605..2e9cd63de6 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -198,7 +198,9 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { if (ths->pFsm->FpRollBackCb != NULL) { SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index); assert(pRollBackEntry != NULL); - assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA); + + // maybe is a NOOP ENTRY + // assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA); SRpcMsg rpcMsg; syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); From 9c12e2bf6e3111632c5f79be9b239a5d53ef2e7c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 25 Mar 2022 10:50:25 +0800 Subject: [PATCH 14/15] start/stop process --- source/libs/function/inc/tudfInt.h | 3 + source/libs/function/src/tudf.c | 123 ++++++++++++++--------------- source/libs/function/src/udfd.c | 41 +++++----- 3 files changed, 80 insertions(+), 87 deletions(-) diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 048522968e..5f757c1ef0 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -20,6 +20,9 @@ extern "C" { #endif +//TODO replaces them with fnDebug +//#define debugPrint(...) taosPrintLog("Function", DEBUG_INFO, 135, __VA_ARGS__) +#define debugPrint(...) {fprintf(stderr, __VA_ARGS__);fprintf(stderr, "\n");} enum { UDF_TASK_SETUP = 0, UDF_TASK_CALL = 1, diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index b4e1c72483..6fcdb34529 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -14,17 +14,10 @@ */ #include "uv.h" #include "os.h" +#include "tlog.h" #include "tudf.h" #include "tudfInt.h" -//TODO replace them with qDebug -#define DEBUG -#ifdef DEBUG -#define debugPrint(...) fprintf(__VA_ARGS__) -#else -#define debugPrint(...) /**/ -#endif - //TODO: when startup, set thread poll size. add it to cfg //TODO: udfd restart when exist or aborts //TODO: network error processing. @@ -179,21 +172,21 @@ void udfTaskQueueMove(SClientUvTaskQueue q, SClientUvTaskQueue n) { int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { - debugPrint(stdout, "%s\n", "encoding request"); + debugPrint("%s", "encoding request"); int len = sizeof(SUdfRequest) - sizeof(void *); switch (request->type) { - case 0: { + case UDF_TASK_SETUP: { SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); len += sizeof(SUdfSetupRequest) - 1 * sizeof(char *) + setup->pathSize; break; } - case 1: { + case UDF_TASK_CALL: { SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); len += sizeof(SUdfCallRequest) - 2 * sizeof(char *) + call->inputBytes + call->stateBytes; break; } - case 2: { + case UDF_TASK_TEARDOWN: { SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); len += sizeof(SUdfTeardownRequest); break; @@ -214,7 +207,7 @@ int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { buf += sizeof(int8_t); switch (request->type) { - case 0: { + case UDF_TASK_SETUP: { SUdfSetupRequest *setup = (SUdfSetupRequest *) (request->subReq); memcpy(buf, setup->udfName, 16); buf += 16; @@ -229,7 +222,7 @@ int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { break; } - case 1: { + case UDF_TASK_CALL: { SUdfCallRequest *call = (SUdfCallRequest *) (request->subReq); *(int64_t *) buf = call->udfHandle; buf += sizeof(int64_t); @@ -246,7 +239,7 @@ int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { break; } - case 2: { + case UDF_TASK_TEARDOWN: { SUdfTeardownRequest *teardown = (SUdfTeardownRequest *) (request->subReq); *(int64_t *) buf = teardown->udfHandle; buf += sizeof(int64_t); @@ -264,9 +257,9 @@ int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { } int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { - debugPrint(stdout, "%s\n", "decoding request"); + debugPrint("%s", "decoding request"); if (*(int32_t *) bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "dequest request error"); + debugPrint("%s", "decoding request error"); return -1; } char *buf = bufMsg; @@ -280,7 +273,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { buf += sizeof(int8_t); switch (request->type) { - case 0: { + case UDF_TASK_SETUP: { SUdfSetupRequest *setup = malloc(sizeof(SUdfSetupRequest)); memcpy(setup->udfName, buf, 16); @@ -297,7 +290,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { request->subReq = setup; break; } - case 1: { + case UDF_TASK_CALL: { SUdfCallRequest *call = malloc(sizeof(SUdfCallRequest)); call->udfHandle = *(int64_t *) buf; @@ -317,7 +310,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { break; } - case 2: { + case UDF_TASK_TEARDOWN: { SUdfTeardownRequest *teardown = malloc(sizeof(SUdfTeardownRequest)); teardown->udfHandle = *(int64_t *) buf; @@ -328,7 +321,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { } if (buf - bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "decode request error"); + debugPrint("%s", "decode request error"); free(request->subReq); free(request); return -1; @@ -338,22 +331,22 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { } int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { - debugPrint(stdout, "%s\n", "encoding response"); + debugPrint("%s", "encoding response"); int32_t len = sizeof(SUdfResponse) - sizeof(void *); switch (response->type) { - case 0: { + case UDF_TASK_SETUP: { len += sizeof(SUdfSetupResponse); break; } - case 1: { + case UDF_TASK_CALL: { SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); len += sizeof(SUdfCallResponse) - 2 * sizeof(char *) + callResp->outputBytes + callResp->newStateBytes; break; } - case 2: { + case UDF_TASK_TEARDOWN: { len += sizeof(SUdfTeardownResponse); break; } @@ -374,13 +367,13 @@ int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { switch (response->type) { - case 0: { + case UDF_TASK_SETUP: { SUdfSetupResponse *setupResp = (SUdfSetupResponse *) (response->subRsp); *(int64_t *) buf = setupResp->udfHandle; buf += sizeof(int64_t); break; } - case 1: { + case UDF_TASK_CALL: { SUdfCallResponse *callResp = (SUdfCallResponse *) (response->subRsp); *(int32_t *) buf = callResp->outputBytes; buf += sizeof(int32_t); @@ -393,7 +386,7 @@ int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { buf += callResp->newStateBytes; break; } - case 2: { + case UDF_TASK_TEARDOWN: { SUdfTeardownResponse *teardownResp = (SUdfTeardownResponse *) (response->subRsp); break; } @@ -408,10 +401,10 @@ int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { } int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { - debugPrint(stdout, "%s\n", "decoding response"); + debugPrint("%s", "decoding response"); if (*(int32_t *) bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "can not decode response"); + debugPrint("%s", "can not decode response"); return -1; } char *buf = bufMsg; @@ -426,14 +419,14 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { buf += sizeof(int32_t); switch (rsp->type) { - case 0: { + case UDF_TASK_SETUP: { SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) malloc(sizeof(SUdfSetupResponse)); setupRsp->udfHandle = *(int64_t *) buf; buf += sizeof(int64_t); rsp->subRsp = (char *) setupRsp; break; } - case 1: { + case UDF_TASK_CALL: { SUdfCallResponse *callRsp = (SUdfCallResponse *) malloc(sizeof(SUdfCallResponse)); callRsp->outputBytes = *(int32_t *) buf; buf += sizeof(int32_t); @@ -450,7 +443,7 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { rsp->subRsp = callRsp; break; } - case 2: { + case UDF_TASK_TEARDOWN: { SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) malloc(sizeof(SUdfTeardownResponse)); rsp->subRsp = teardownRsp; break; @@ -459,7 +452,7 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { break; } if (buf - bufMsg != bufLen) { - debugPrint(stderr, "%s\n", "can not decode response"); + debugPrint("%s", "can not decode response"); free(rsp->subRsp); free(rsp); return -1; @@ -469,8 +462,9 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { } void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { - debugPrint(stderr, "Process exited with status %" PRId64 ", signal %d\n", exit_status, term_signal); + debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal); uv_close((uv_handle_t *) req, NULL); + //TODO: restart the udfd process } void onUdfcPipeClose(uv_handle_t *handle) { @@ -488,7 +482,7 @@ void onUdfcPipeClose(uv_handle_t *handle) { } int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { - debugPrint(stdout, "%s\n", "get uv task result"); + debugPrint("%s", "get uv task result"); if (uvTask->type == UV_TASK_REQ_RSP) { if (uvTask->rspBuf.base != NULL) { SUdfResponse *rsp; @@ -497,13 +491,13 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT switch (task->type) { case UDF_TASK_SETUP: { - //TODO: copy + //TODO: copy or not task->_setup.rsp = *(SUdfSetupResponse *) (rsp->subRsp); break; } case UDF_TASK_CALL: { task->_call.rsp = *(SUdfCallResponse *) (rsp->subRsp); - //TODO: copy + //TODO: copy or not break; } case UDF_TASK_TEARDOWN: { @@ -532,7 +526,7 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT } void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint(stdout, "%s\n", "client allocate buffer to receive from pipe"); + debugPrint("%s", "client allocate buffer to receive from pipe"); SClientUvConn *conn = handle->data; SClientConnBuf *connBuf = &conn->readBuf; @@ -565,7 +559,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf } } - debugPrint(stdout, "\tconn buf cap - len - total : %d - %d - %d\n", connBuf->cap, connBuf->len, connBuf->total); + debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); } @@ -621,7 +615,7 @@ void udfcUvHandleError(SClientUvConn *conn) { } void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint(stdout, "%s, nread: %zd\n", "client read from pipe", nread); + debugPrint("%s, nread: %zd", "client read from pipe", nread); if (nread == 0) return; SClientUvConn *conn = client->data; @@ -634,7 +628,7 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - debugPrint(stderr, "\tclient read error: %s\n", uv_strerror(nread)); + debugPrint("\tclient read error: %s", uv_strerror(nread)); if (nread == UV_EOF) { //TODO: } @@ -644,7 +638,7 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } void onUdfClientWrite(uv_write_t *write, int status) { - debugPrint(stderr, "%s\n", "after writing to pipe"); + debugPrint("%s", "after writing to pipe"); SClientUvTaskNode *uvTask = write->data; if (status == 0) { uv_pipe_t *pipe = uvTask->pipe; @@ -653,7 +647,7 @@ void onUdfClientWrite(uv_write_t *write, int status) { } else { //TODO Log error; } - debugPrint(stdout, "\tlength:%zu\n", uvTask->reqBuf.len); + debugPrint("\tlength:%zu", uvTask->reqBuf.len); free(write); free(uvTask->reqBuf.base); } @@ -707,7 +701,7 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN } int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint(stdout, "%s, %d\n", "queue uv task", uvTask->type); + debugPrint("%s, %d", "queue uv task", uvTask->type); uv_mutex_lock(&gUdfTaskQueueMutex); udfTaskQueueInsertTail(gUdfTaskQueue, uvTask); @@ -721,7 +715,7 @@ int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { } int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint(stdout, "%s, type %d\n", "start uv task ", uvTask->type); + debugPrint("%s, type %d", "start uv task ", uvTask->type); switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = malloc(sizeof(uv_pipe_t)); @@ -790,20 +784,21 @@ void udfStopAsyncCb(uv_async_t *async) { void startUdfd(void *argsThread) { uv_loop_init(&gUdfdLoop); -// uv_process_options_t options; -// static char path[256] = {0}; -// size_t cwdSize; -// uv_cwd(path, &cwdSize); -// strcat(path, "/udfd"); -// char* args[2] = {path, NULL}; -// options.args = args; -// options.file = path; -// options.exit_cb = onUdfdExit; -// -// int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); -// if (err != 0) { -// debugPrint(stderr, "can not spawn udfd. error: %s", uv_strerror(err)); -// } + //TODO: path + uv_process_options_t options; + static char path[256] = {0}; + size_t cwdSize; + uv_cwd(path, &cwdSize); + strcat(path, "./udfd"); + char* args[2] = {path, NULL}; + options.args = args; + options.file = path; + options.exit_cb = onUdfdExit; + + int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); + if (err != 0) { + debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); + } uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); @@ -822,7 +817,7 @@ int32_t startUdfService() { int32_t stopUdfService() { uv_barrier_destroy(&gUdfInitBarrier); - //uv_process_kill(&gUdfdProcess, SIGINT); + uv_process_kill(&gUdfdProcess, SIGINT); uv_async_send(&gUdfLoopStopAsync); uv_mutex_destroy(&gUdfTaskQueueMutex); uv_thread_join(&gUdfLoopThread); @@ -844,7 +839,7 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { } int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { - debugPrint(stdout, "%s\n", "client setup udf"); + debugPrint("%s", "client setup udf"); SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = malloc(sizeof(SUdfUvSession)); @@ -875,7 +870,7 @@ int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newState, int32_t *newStateSize, SUdfDataBlock *output) { - debugPrint(stdout, "%s\n", "client call udf"); + debugPrint("%s", "client call udf"); SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); task->errCode = 0; @@ -904,7 +899,7 @@ int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, S } int32_t teardownUdf(UdfHandle handle) { - debugPrint(stdout, "%s\n", "client teardown udf"); + debugPrint("%s", "client teardown udf"); SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); task->errCode = 0; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 18c6004b25..27385325f5 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -15,16 +15,11 @@ #include "uv.h" #include "os.h" +#include "tlog.h" + #include "tudf.h" #include "tudfInt.h" -//TODO replaces them with qDebug -#define DEBUG -#ifdef DEBUG -#define debugPrint(...) fprintf(__VA_ARGS__) -#else -#define debugPrint(...) /**/ -#endif static uv_loop_t *loop; @@ -66,14 +61,14 @@ void udfdProcessRequest(uv_work_t *req) { switch (request->type) { case UDF_TASK_SETUP: { - debugPrint(stdout, "%s\n", "process setup request"); + debugPrint("%s", "process setup request"); SUdf *udf = malloc(sizeof(SUdf)); udf->refCount = 0; SUdfSetupRequest *setup = request->subReq; strcpy(udf->name, setup->udfName); int err = uv_dlopen(setup->path, &udf->lib); if (err != 0) { - debugPrint(stderr, "can not load library %s. error: %s", setup->path, uv_strerror(err)); + debugPrint("can not load library %s. error: %s", setup->path, uv_strerror(err)); //TODO set error } @@ -109,7 +104,7 @@ void udfdProcessRequest(uv_work_t *req) { } case UDF_TASK_CALL: { - debugPrint(stdout, "%s\n", "process call request"); + debugPrint("%s", "process call request"); SUdfCallRequest *call = request->subReq; SUdfHandle *handle = (SUdfHandle *) (call->udfHandle); SUdf *udf = handle->udf; @@ -146,7 +141,7 @@ void udfdProcessRequest(uv_work_t *req) { break; } case UDF_TASK_TEARDOWN: { - debugPrint(stdout, "%s\n", "process teardown request"); + debugPrint("%s", "process teardown request"); SUdfTeardownRequest *teardown = request->subReq; SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle); @@ -186,12 +181,12 @@ void udfdProcessRequest(uv_work_t *req) { } void udfdOnWrite(uv_write_t *req, int status) { - debugPrint(stdout, "%s\n", "after writing to pipe"); + debugPrint("%s", "after writing to pipe"); if (status < 0) { - debugPrint(stderr, "Write error %s\n", uv_err_name(status)); + debugPrint("Write error %s", uv_err_name(status)); } SUvUdfWork *work = (SUvUdfWork *) req->data; - debugPrint(stdout, "\tlength: %zu\n", work->output.len); + debugPrint("\tlength: %zu", work->output.len); free(work->output.base); free(work); free(req); @@ -199,7 +194,7 @@ void udfdOnWrite(uv_write_t *req, int status) { void udfdSendResponse(uv_work_t *work, int status) { - debugPrint(stdout, "%s\n", "send response"); + debugPrint("%s", "send response"); SUvUdfWork *udfWork = (SUvUdfWork *) (work->data); uv_write_t *write_req = malloc(sizeof(uv_write_t)); @@ -210,7 +205,7 @@ void udfdSendResponse(uv_work_t *work, int status) { } void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint(stdout, "%s\n", "allocate buffer for read"); + debugPrint("%s", "allocate buffer for read"); SUdfdUvConn *ctx = handle->data; int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); if (ctx->inputCap == 0) { @@ -240,7 +235,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->len = 0; } } - debugPrint(stdout, "\tinput buf cap - len - total : %d - %d - %d\n", ctx->inputCap, ctx->inputLen, ctx->inputTotal); + debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal); } @@ -279,7 +274,7 @@ void udfdUvHandleError(SUdfdUvConn *conn) { } void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint(stdout, "%s, nread: %zd\n", "read from pipe", nread); + debugPrint("%s, nread: %zd", "read from pipe", nread); if (nread == 0) return; @@ -296,7 +291,7 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - debugPrint(stderr, "Read error %s\n", uv_err_name(nread)); + debugPrint("Read error %s", uv_err_name(nread)); if (nread == UV_EOF) { //TODO check more when close } else { @@ -306,7 +301,7 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } void udfdOnNewConnection(uv_stream_t *server, int status) { - debugPrint(stdout, "%s\n", "on new connection"); + debugPrint("%s", "on new connection"); if (status < 0) { // TODO return; @@ -335,7 +330,7 @@ void removeListeningPipe(int sig) { } int main() { - debugPrint(stderr, "libuv version: %x\n", UV_VERSION_HEX); + debugPrint("libuv version: %x", UV_VERSION_HEX); loop = uv_default_loop(); uv_fs_t req; @@ -348,12 +343,12 @@ int main() { int r; if ((r = uv_pipe_bind(&server, "udf.sock"))) { - debugPrint(stderr, "Bind error %s\n", uv_err_name(r)); + debugPrint("Bind error %s\n", uv_err_name(r)); removeListeningPipe(0); return 1; } if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) { - debugPrint(stderr, "Listen error %s\n", uv_err_name(r)); + debugPrint("Listen error %s", uv_err_name(r)); return 2; } uv_run(loop, UV_RUN_DEFAULT); From c98626b0810da75f1653d3876c85dd643447e6ab Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 25 Mar 2022 13:19:41 +0800 Subject: [PATCH 15/15] tSma create bug fix --- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 3 +++ source/libs/parser/src/parTranslater.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index ebd34fb2a5..4e3ab8eae4 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -69,7 +69,7 @@ void mndCleanupSma(SMnode *pMnode) {} static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = sizeof(SSmaObj) + pSma->exprLen + pSma->tagsFilterLen + TSDB_SMA_RESERVE_SIZE; + int32_t size = sizeof(SSmaObj) + pSma->exprLen + pSma->tagsFilterLen + pSma->sqlLen + pSma->astLen + TSDB_SMA_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SMA, TSDB_SMA_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index bb7891ad2d..be03539b4d 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -269,6 +269,9 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) { int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { SNode *pAst = NULL; +#if 1 // TODO: remove debug info later + printf("ast = %s\n", ast); +#endif if (nodesStringToNode(ast, &pAst) < 0) { return -1; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 55303ddc2d..8b97b88b09 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1519,7 +1519,7 @@ static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; - pCxt->pCmdMsg->msgType = TDMT_VND_CREATE_SMA; + pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_SMA; pCxt->pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, &createSmaReq); pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) {