xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: core caused by invalid total rows of data block when executing sql like select udf2(f2) from udf.t2 group by 1-udf1(f1)

This commit is contained in:
slzhou 2022-05-14 09:31:30 +08:00
parent ee8e7d8dca
commit b53e7192fc
2 changed files with 78 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
source/libs/function/src/tudf.c
View File

@ -310,28 +310,28 @@ enum {
}; };
int64_t gUdfTaskSeqNum = 0; int64_t gUdfTaskSeqNum = 0;
typedef struct SUdfdProxy { typedef struct SUdfcProxy {
char udfdPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2]; char udfdPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2];
uv_barrier_t gUdfInitBarrier; uv_barrier_t initBarrier;
uv_loop_t gUdfdLoop; uv_loop_t uvLoop;
uv_thread_t gUdfLoopThread; uv_thread_t loopThread;
uv_async_t gUdfLoopTaskAync; uv_async_t loopTaskAync;
uv_async_t gUdfLoopStopAsync; uv_async_t loopStopAsync;
uv_mutex_t gUdfTaskQueueMutex; uv_mutex_t taskQueueMutex;
int8_t gUdfcState; int8_t udfcState;
QUEUE gUdfTaskQueue; QUEUE taskQueue;
QUEUE gUvProcTaskQueue; QUEUE uvProcTaskQueue;
int8_t initialized; int8_t initialized;
} SUdfdProxy; } SUdfcProxy;
SUdfdProxy gUdfdProxy = {0}; SUdfcProxy gUdfdProxy = {0};
typedef struct SClientUdfUvSession { typedef struct SClientUdfUvSession {
SUdfdProxy *udfc; SUdfcProxy *udfc;
int64_t severHandle; int64_t severHandle;
uv_pipe_t *udfUvPipe; uv_pipe_t *udfUvPipe;
@ -341,7 +341,7 @@ typedef struct SClientUdfUvSession {
} SClientUdfUvSession; } SClientUdfUvSession;
typedef struct SClientUvTaskNode { typedef struct SClientUvTaskNode {
SUdfdProxy *udfc; SUdfcProxy *udfc;
int8_t type; int8_t type;
int errCode; int errCode;
@ -1055,11 +1055,11 @@ int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) { int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) {
fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask); fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask);
SUdfdProxy *udfc = uvTask->udfc; SUdfcProxy *udfc = uvTask->udfc;
uv_mutex_lock(&udfc->gUdfTaskQueueMutex); uv_mutex_lock(&udfc->taskQueueMutex);
QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); QUEUE_INSERT_TAIL(&udfc->taskQueue, &uvTask->recvTaskQueue);
uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); uv_mutex_unlock(&udfc->taskQueueMutex);
uv_async_send(&udfc->gUdfLoopTaskAync); uv_async_send(&udfc->loopTaskAync);
uv_sem_wait(&uvTask->taskSem); uv_sem_wait(&uvTask->taskSem);
fnInfo("udfc uv task finished. task: %d, %p", uvTask->type, uvTask); fnInfo("udfc uv task finished. task: %d, %p", uvTask->type, uvTask);
@ -1073,7 +1073,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
switch (uvTask->type) { switch (uvTask->type) {
case UV_TASK_CONNECT: { case UV_TASK_CONNECT: {
uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
uv_pipe_init(&uvTask->udfc->gUdfdLoop, pipe, 0); uv_pipe_init(&uvTask->udfc->uvLoop, pipe, 0);
uvTask->pipe = pipe; uvTask->pipe = pipe;
SClientUvConn *conn = taosMemoryCalloc(1, sizeof(SClientUvConn)); SClientUvConn *conn = taosMemoryCalloc(1, sizeof(SClientUvConn));
@ -1113,46 +1113,46 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
} }
void udfClientAsyncCb(uv_async_t *async) { void udfClientAsyncCb(uv_async_t *async) {
SUdfdProxy *udfc = async->data; SUdfcProxy *udfc = async->data;
QUEUE wq; QUEUE wq;
uv_mutex_lock(&udfc->gUdfTaskQueueMutex); uv_mutex_lock(&udfc->taskQueueMutex);
QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); QUEUE_MOVE(&udfc->taskQueue, &wq);
uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); uv_mutex_unlock(&udfc->taskQueueMutex);
while (!QUEUE_EMPTY(&wq)) { while (!QUEUE_EMPTY(&wq)) {
QUEUE* h = QUEUE_HEAD(&wq); QUEUE* h = QUEUE_HEAD(&wq);
QUEUE_REMOVE(h); QUEUE_REMOVE(h);
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue);
udfcStartUvTask(task); udfcStartUvTask(task);
QUEUE_INSERT_TAIL(&udfc->gUvProcTaskQueue, &task->procTaskQueue); QUEUE_INSERT_TAIL(&udfc->uvProcTaskQueue, &task->procTaskQueue);
} }
} }
void cleanUpUvTasks(SUdfdProxy *udfc) { void cleanUpUvTasks(SUdfcProxy *udfc) {
fnDebug("clean up uv tasks") fnDebug("clean up uv tasks")
QUEUE wq; QUEUE wq;
uv_mutex_lock(&udfc->gUdfTaskQueueMutex); uv_mutex_lock(&udfc->taskQueueMutex);
QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); QUEUE_MOVE(&udfc->taskQueue, &wq);
uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); uv_mutex_unlock(&udfc->taskQueueMutex);
while (!QUEUE_EMPTY(&wq)) { while (!QUEUE_EMPTY(&wq)) {
QUEUE* h = QUEUE_HEAD(&wq); QUEUE* h = QUEUE_HEAD(&wq);
QUEUE_REMOVE(h); QUEUE_REMOVE(h);
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue);
if (udfc->gUdfcState == UDFC_STATE_STOPPING) { if (udfc->udfcState == UDFC_STATE_STOPPING) {
task->errCode = TSDB_CODE_UDF_STOPPING; task->errCode = TSDB_CODE_UDF_STOPPING;
} }
uv_sem_post(&task->taskSem); uv_sem_post(&task->taskSem);
} }
while (!QUEUE_EMPTY(&udfc->gUvProcTaskQueue)) { while (!QUEUE_EMPTY(&udfc->uvProcTaskQueue)) {
QUEUE* h = QUEUE_HEAD(&udfc->gUvProcTaskQueue); QUEUE* h = QUEUE_HEAD(&udfc->uvProcTaskQueue);
QUEUE_REMOVE(h); QUEUE_REMOVE(h);
SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue);
if (udfc->gUdfcState == UDFC_STATE_STOPPING) { if (udfc->udfcState == UDFC_STATE_STOPPING) {
task->errCode = TSDB_CODE_UDF_STOPPING; task->errCode = TSDB_CODE_UDF_STOPPING;
} }
uv_sem_post(&task->taskSem); uv_sem_post(&task->taskSem);
@ -1160,28 +1160,28 @@ void cleanUpUvTasks(SUdfdProxy *udfc) {
} }
void udfStopAsyncCb(uv_async_t *async) { void udfStopAsyncCb(uv_async_t *async) {
SUdfdProxy *udfc = async->data; SUdfcProxy *udfc = async->data;
cleanUpUvTasks(udfc); cleanUpUvTasks(udfc);
if (udfc->gUdfcState == UDFC_STATE_STOPPING) { if (udfc->udfcState == UDFC_STATE_STOPPING) {
uv_stop(&udfc->gUdfdLoop); uv_stop(&udfc->uvLoop);
} }
} }
void constructUdfService(void *argsThread) { void constructUdfService(void *argsThread) {
SUdfdProxy *udfc = (SUdfdProxy*)argsThread; SUdfcProxy *udfc = (SUdfcProxy *)argsThread;
uv_loop_init(&udfc->gUdfdLoop); uv_loop_init(&udfc->uvLoop);
uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopTaskAync, udfClientAsyncCb); uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfClientAsyncCb);
udfc->gUdfLoopTaskAync.data = udfc; udfc->loopTaskAync.data = udfc;
uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopStopAsync, udfStopAsyncCb); uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb);
udfc->gUdfLoopStopAsync.data = udfc; udfc->loopStopAsync.data = udfc;
uv_mutex_init(&udfc->gUdfTaskQueueMutex); uv_mutex_init(&udfc->taskQueueMutex);
QUEUE_INIT(&udfc->gUdfTaskQueue); QUEUE_INIT(&udfc->taskQueue);
QUEUE_INIT(&udfc->gUvProcTaskQueue); QUEUE_INIT(&udfc->uvProcTaskQueue);
uv_barrier_wait(&udfc->gUdfInitBarrier); uv_barrier_wait(&udfc->initBarrier);
//TODO return value of uv_run //TODO return value of uv_run
uv_run(&udfc->gUdfdLoop, UV_RUN_DEFAULT); uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
uv_loop_close(&udfc->gUdfdLoop); uv_loop_close(&udfc->uvLoop);
} }
int32_t udfcOpen() { int32_t udfcOpen() {
@ -1189,14 +1189,14 @@ int32_t udfcOpen() {
if (old == 1) { if (old == 1) {
return 0; return 0;
} }
SUdfdProxy *proxy = &gUdfdProxy; SUdfcProxy *proxy = &gUdfdProxy;
getUdfdPipeName(proxy->udfdPipeName, sizeof(proxy->udfdPipeName)); getUdfdPipeName(proxy->udfdPipeName, sizeof(proxy->udfdPipeName));
proxy->gUdfcState = UDFC_STATE_STARTNG; proxy->udfcState = UDFC_STATE_STARTNG;
uv_barrier_init(&proxy->gUdfInitBarrier, 2); uv_barrier_init(&proxy->initBarrier, 2);
uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); uv_thread_create(&proxy->loopThread, constructUdfService, proxy);
atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY); atomic_store_8(&proxy->udfcState, UDFC_STATE_READY);
proxy->gUdfcState = UDFC_STATE_READY; proxy->udfcState = UDFC_STATE_READY;
uv_barrier_wait(&proxy->gUdfInitBarrier); uv_barrier_wait(&proxy->initBarrier);
fnInfo("udfc initialized") fnInfo("udfc initialized")
return 0; return 0;
} }
@ -1207,13 +1207,13 @@ int32_t udfcClose() {
return 0; return 0;
} }
SUdfdProxy *udfc = &gUdfdProxy; SUdfcProxy *udfc = &gUdfdProxy;
udfc->gUdfcState = UDFC_STATE_STOPPING; udfc->udfcState = UDFC_STATE_STOPPING;
uv_async_send(&udfc->gUdfLoopStopAsync); uv_async_send(&udfc->loopStopAsync);
uv_thread_join(&udfc->gUdfLoopThread); uv_thread_join(&udfc->loopThread);
uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); uv_mutex_destroy(&udfc->taskQueueMutex);
uv_barrier_destroy(&udfc->gUdfInitBarrier); uv_barrier_destroy(&udfc->initBarrier);
udfc->gUdfcState = UDFC_STATE_INITAL; udfc->udfcState = UDFC_STATE_INITAL;
fnInfo("udfc cleaned up"); fnInfo("udfc cleaned up");
return 0; return 0;
} }
@ -1236,7 +1236,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
fnInfo("udfc setup udf. udfName: %s", udfName); fnInfo("udfc setup udf. udfName: %s", udfName);
if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) { if (gUdfdProxy.udfcState != UDFC_STATE_READY) {
return TSDB_CODE_UDF_INVALID_STATE; return TSDB_CODE_UDF_INVALID_STATE;
} }
SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask)); SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask));
@ -1484,7 +1484,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
SSDataBlock tempBlock = {0}; SSDataBlock tempBlock = {0};
tempBlock.info.numOfCols = numOfCols; tempBlock.info.numOfCols = numOfCols;
tempBlock.info.rows = numOfRows; tempBlock.info.rows = pInput->totalRows;
tempBlock.info.uid = pInput->uid; tempBlock.info.uid = pInput->uid;
bool hasVarCol = false; bool hasVarCol = false;
tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));

15
tests/script/tsim/query/udf.sim
View File

@ -109,6 +109,7 @@ if $data01 != 18.547236991 then
endi endi
sql select udf2(udf1(f2-f1)), udf2(udf1(f2/f1)) from t2; sql select udf2(udf1(f2-f1)), udf2(udf1(f2/f1)) from t2;
print $rows , $data00 , $data01
if $rows != 1 then if $rows != 1 then
return -1 return -1
endi endi
@ -118,7 +119,19 @@ endi
if $data01 != 152.420471066 then if $data01 != 152.420471066 then
return -1 return -1
endi endi
print $rows , $data00 , $data01
sql select udf2(f2) from udf.t2 group by 1-udf1(f1);
print $rows , $data00 , $data10
if $rows != 2 then
return -1
endi
if $data00 != 2.000000000 then
return -1
endi
if $data10 != 12.083045974 then
return -1
endi
sql drop function udf1; sql drop function udf1;
sql show functions; sql show functions;
if $rows != 1 then if $rows != 1 then