From e7b3a3aec838d882a4b94e5d2e5457d311c12a70 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 1 Nov 2024 18:58:22 +0800 Subject: [PATCH 001/108] fix(stream): reset task add the failed chkptId info. --- include/common/tmsg.h | 9 ++++++- include/libs/stream/tstream.h | 1 + source/dnode/mnode/impl/inc/mndStream.h | 5 ++-- source/dnode/mnode/impl/src/mndStreamHb.c | 21 ++++++++------- .../dnode/mnode/impl/src/mndStreamTransAct.c | 7 ++--- source/dnode/vnode/src/tqCommon/tqCommon.c | 12 +++------ source/libs/stream/inc/streamInt.h | 1 - source/libs/stream/src/streamCheckpoint.c | 26 ++++++++++++------- source/libs/stream/src/streamTask.c | 4 +-- 9 files changed, 51 insertions(+), 35 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b7ea6d4719..a5ba89b870 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3769,7 +3769,14 @@ typedef struct { SMsgHead head; int64_t streamId; int32_t taskId; -} SVPauseStreamTaskReq, SVResetStreamTaskReq; +} SVPauseStreamTaskReq; + +typedef struct { + SMsgHead head; + int64_t streamId; + int32_t taskId; + int64_t chkptId; +} SVResetStreamTaskReq; typedef struct { char name[TSDB_STREAM_FNAME_LEN]; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index a189cee0bb..c43d2bde71 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -712,6 +712,7 @@ int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask); void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId); int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId, SRpcHandleInfo* pInfo, int32_t code); +void streamTaskSetFailedCheckpointId(SStreamTask* pTask, int64_t failedId); int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue); diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index b97eaf31d1..012a9f166d 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -56,6 +56,7 @@ typedef struct SStreamTransMgmt { typedef struct SStreamTaskResetMsg { int64_t streamId; int32_t transId; + int64_t checkpointId; } SStreamTaskResetMsg; typedef struct SChkptReportInfo { @@ -141,9 +142,9 @@ int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pSt int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); -int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream, int64_t chkptId); int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream, int64_t chkptId); int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStreamTask* pTask, int64_t checkpointId, int64_t ts); int32_t mndStreamSetRestartAction(SMnode* pMnode, STrans *pTrans, SStreamObj* pStream); int32_t mndStreamSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, int64_t checkpointId, diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 941956ae2b..03b418f13f 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -24,7 +24,7 @@ typedef struct SFailedCheckpointInfo { static int32_t mndStreamSendUpdateChkptInfoMsg(SMnode *pMnode); static int32_t mndSendDropOrphanTasksMsg(SMnode *pMnode, SArray *pList); -static int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId); +static int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId, int64_t checkpointId); static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage); static void addIntoFailedChkptList(SArray *pList, const SFailedCheckpointInfo *pInfo); static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); @@ -68,7 +68,7 @@ void addIntoFailedChkptList(SArray *pList, const SFailedCheckpointInfo *pInfo) { } } -int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { +int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream, int64_t chkptId) { STrans *pTrans = NULL; int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint", &pTrans); @@ -84,7 +84,7 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return code; } - code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); + code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream, chkptId); if (code) { sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); @@ -115,7 +115,7 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { return code; } -int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId) { +int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t transId, int64_t checkpointId) { int32_t size = sizeof(SStreamTaskResetMsg); int32_t num = taosArrayGetSize(execInfo.pKilledChkptTrans); @@ -135,8 +135,9 @@ int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t taosArrayRemove(execInfo.pKilledChkptTrans, 0); // remove this first, append new reset trans in the tail } - SStreamTaskResetMsg p = {.streamId = streamId, .transId = transId}; + SStreamTaskResetMsg p = {.streamId = streamId, .transId = transId, .checkpointId = checkpointId}; + // let's remember that this trans had been killed already void *px = taosArrayPush(execInfo.pKilledChkptTrans, &p); if (px == NULL) { mError("failed to push reset-msg trans:%d into the killed chkpt trans list, size:%d", transId, num - 1); @@ -150,6 +151,7 @@ int32_t mndSendResetFromCheckpointMsg(SMnode *pMnode, int64_t streamId, int32_t pReq->streamId = streamId; pReq->transId = transId; + pReq->checkpointId = checkpointId; SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_TASK_RESET, .pCont = pReq, .contLen = size}; int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -234,7 +236,7 @@ int32_t mndProcessResetStatusReq(SRpcMsg *pReq) { } else { mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name, pStream->uid, pMsg->transId); - code = mndCreateStreamResetStatusTrans(pMnode, pStream); + code = mndCreateStreamResetStatusTrans(pMnode, pStream, pMsg->checkpointId); } } @@ -495,10 +497,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { continue; } - mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", - pInfo->checkpointId, pInfo->transId); + mInfo("stream:0x%" PRIx64 "checkpointId:%" PRId64 + " transId:%d failed issue task-reset trans to reset all tasks status", + pInfo->streamUid, pInfo->checkpointId, pInfo->transId); - code = mndSendResetFromCheckpointMsg(pMnode, pInfo->streamUid, pInfo->transId); + code = mndSendResetFromCheckpointMsg(pMnode, pInfo->streamUid, pInfo->transId, pInfo->checkpointId); if (code) { mError("failed to create reset task trans, code:%s", tstrerror(code)); } diff --git a/source/dnode/mnode/impl/src/mndStreamTransAct.c b/source/dnode/mnode/impl/src/mndStreamTransAct.c index 139ea4f147..7ee60c6f14 100644 --- a/source/dnode/mnode/impl/src/mndStreamTransAct.c +++ b/source/dnode/mnode/impl/src/mndStreamTransAct.c @@ -295,7 +295,7 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas return code; } -static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { +static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, int64_t chkptId) { SVResetStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq)); if (pReq == NULL) { mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq), @@ -306,6 +306,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; + pReq->chkptId = chkptId; SEpSet epset = {0}; bool hasEpset = false; @@ -544,7 +545,7 @@ int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* p return 0; } -int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { +int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream, int64_t chkptId) { SStreamTaskIter *pIter = NULL; taosWLockLatch(&pStream->lock); @@ -564,7 +565,7 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj * return code; } - code = doSetResetAction(pMnode, pTrans, pTask); + code = doSetResetAction(pMnode, pTrans, pTask, chkptId); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a00e92997c..7390e4948a 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -935,7 +935,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { } int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { - SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; + SVResetStreamTaskReq* pReq = (SVResetStreamTaskReq*)pMsg; SStreamTask* pTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); @@ -950,17 +950,13 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { streamMutexLock(&pTask->lock); streamTaskClearCheckInfo(pTask, true); + streamTaskSetFailedCheckpointId(pTask, pReq->chkptId); + // clear flag set during do checkpoint, and open inputQ for all upstream tasks SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state == TASK_STATUS__CK) { - int32_t tranId = 0; - int64_t activeChkId = 0; - streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId); - - tqDebug("s-task:%s reset task status from checkpoint, current checkpointingId:%" PRId64 ", transId:%d", - pTask->id.idStr, activeChkId, tranId); - streamTaskSetStatusReady(pTask); + tqDebug("s-task:%s reset checkpoint status to ready", pTask->id.idStr); } else if (pState.state == TASK_STATUS__UNINIT) { // tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); // tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 94c196d280..bbfb9f2e15 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -191,7 +191,6 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask); int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask); int32_t streamTaskSendCheckpointReq(SStreamTask* pTask); -void streamTaskSetFailedCheckpointId(SStreamTask* pTask); int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask); int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask); int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 302090bb37..aa3b4fbe46 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -560,7 +560,7 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { } streamMutexUnlock(&pInfo->lock); - stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", current checkpointId:%" PRId64, + stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", latest checkpointId:%" PRId64, pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId); } @@ -680,15 +680,22 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV return TSDB_CODE_SUCCESS; } -void streamTaskSetFailedCheckpointId(SStreamTask* pTask) { +void streamTaskSetFailedCheckpointId(SStreamTask* pTask, int64_t failedId) { struct SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; - if (pInfo->activeId <= 0) { - stWarn("s-task:%s checkpoint-info is cleared now, not set the failed checkpoint info", pTask->id.idStr); + if (failedId <= 0) { + stWarn("s-task:%s failedId is 0, not update the failed checkpoint info, current failedId:%" PRId64 + " activeId:%" PRId64, + pTask->id.idStr, pInfo->failedId, pInfo->activeId); } else { - pInfo->failedId = pInfo->activeId; - stDebug("s-task:%s mark and set the failed checkpointId:%" PRId64 " (transId:%d)", pTask->id.idStr, pInfo->activeId, - pInfo->transId); + if (failedId <= pInfo->failedId) { + stDebug("s-task:%s failedId:%" PRId64 " not update to:%" PRId64, pTask->id.idStr, pInfo->failedId, failedId); + } else { + stDebug("s-task:%s mark and set the failed checkpointId:%" PRId64 " (transId:%d) activeId:%" PRId64 + " prev failedId:%" PRId64, + pTask->id.idStr, failedId, pInfo->transId, pInfo->activeId, pInfo->failedId); + pInfo->failedId = failedId; + } } } @@ -696,7 +703,7 @@ void streamTaskSetCheckpointFailed(SStreamTask* pTask) { streamMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask).state; if (status == TASK_STATUS__CK) { - streamTaskSetFailedCheckpointId(pTask); + streamTaskSetFailedCheckpointId(pTask, pTask->chkInfo.pActiveInfo->activeId); } streamMutexUnlock(&pTask->lock); } @@ -874,8 +881,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); } } else { // clear the checkpoint info if failed + // set failed checkpoint id before clear the checkpoint info streamMutexLock(&pTask->lock); - streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info + streamTaskSetFailedCheckpointId(pTask, ckId); streamMutexUnlock(&pTask->lock); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index b359cdfc81..0c3dd3709c 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -1248,13 +1248,13 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { taosMemoryFree(pInfo); } -//NOTE: clear the checkpoint id, and keep the failed id +// NOTE: clear the checkpoint id, and keep the failed id +// failedId for a task will increase as the checkpoint I.D. increases. void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) { pInfo->activeId = 0; pInfo->transId = 0; pInfo->allUpstreamTriggerRecv = 0; pInfo->dispatchTrigger = false; -// pInfo->failedId = 0; taosArrayClear(pInfo->pDispatchTriggerList); taosArrayClear(pInfo->pCheckpointReadyRecvList); From bb05b9b912f205143cfad19e3845c80e2d39e860 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Nov 2024 10:11:22 +0800 Subject: [PATCH 002/108] fix(stream): record the failed checkpointId, after receive reset task info. --- include/util/taoserror.h | 1 + .../mnode/impl/src/mndStreamErrorInjection.c | 72 +++++++++++++++++++ source/dnode/mnode/impl/src/mndStreamHb.c | 5 +- source/dnode/mnode/impl/src/mndStreamUtil.c | 70 ------------------ source/libs/stream/inc/streamInt.h | 2 + source/libs/stream/src/streamCheckpoint.c | 8 ++- source/libs/stream/src/streamErrorInjection.c | 17 +++++ source/util/src/terror.c | 3 +- 8 files changed, 103 insertions(+), 75 deletions(-) create mode 100644 source/dnode/mnode/impl/src/mndStreamErrorInjection.c create mode 100644 source/libs/stream/src/streamErrorInjection.c diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2a67a3cae1..269a0ad49c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -1008,6 +1008,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_STREAM_CONFLICT_EVENT TAOS_DEF_ERROR_CODE(0, 0x4106) #define TSDB_CODE_STREAM_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x4107) #define TSDB_CODE_STREAM_INPUTQ_FULL TAOS_DEF_ERROR_CODE(0, 0x4108) +#define TSDB_CODE_STREAM_INVLD_CHKPT TAOS_DEF_ERROR_CODE(0, 0x4109) // TDLite #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) diff --git a/source/dnode/mnode/impl/src/mndStreamErrorInjection.c b/source/dnode/mnode/impl/src/mndStreamErrorInjection.c new file mode 100644 index 0000000000..c68416369d --- /dev/null +++ b/source/dnode/mnode/impl/src/mndStreamErrorInjection.c @@ -0,0 +1,72 @@ +#include "mndTrans.h" + +uint32_t seed = 0; + +static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) { + SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature}; + rpcMsg.pCont = rpcMallocCont(pAction->contLen); + if (rpcMsg.pCont == NULL) { + return rpcMsg; + } + + rpcMsg.info.traceId.rootId = traceId; + rpcMsg.info.notFreeAhandle = 1; + + memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); + return rpcMsg; +} + +void streamTransRandomErrorGen(STransAction *pAction, STrans *pTrans, int64_t signature) { + if ((pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT && pAction->id > 2) || + (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) || + (pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE && pAction->id > 2)) { + if (seed == 0) { + seed = taosGetTimestampSec(); + } + + uint32_t v = taosRandR(&seed); + int32_t choseItem = v % 5; + + if (choseItem == 0) { + // 1. one of update-checkpoint not send, restart and send it again + taosMsleep(5000); + if (pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT) { + mError( + "***sleep 5s and core dump, following tasks will not recv update-checkpoint info, so the checkpoint will " + "rollback***"); + exit(-1); + } else if (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) { // pAction->msgType == TDMT_STREAM_CONSEN_CHKPT + mError( + "***sleep 5s and core dump, following tasks will not recv consen-checkpoint info, so the tasks will " + "not started***"); + } else { // pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE + mError( + "***sleep 5s and core dump, following tasks will not recv checkpoint-source info, so the tasks will " + "started after restart***"); + exit(-1); + } + } else if (choseItem == 1) { + // 2. repeat send update chkpt msg + mError("***repeat send update-checkpoint/consensus/checkpoint trans msg 3times to vnode***"); + + mError("***repeat 1***"); + SRpcMsg rpcMsg1 = createRpcMsg(pAction, pTrans->mTraceId, signature); + int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg1); + + mError("***repeat 2***"); + SRpcMsg rpcMsg2 = createRpcMsg(pAction, pTrans->mTraceId, signature); + code = tmsgSendReq(&pAction->epSet, &rpcMsg2); + + mError("***repeat 3***"); + SRpcMsg rpcMsg3 = createRpcMsg(pAction, pTrans->mTraceId, signature); + code = tmsgSendReq(&pAction->epSet, &rpcMsg3); + } else if (choseItem == 2) { + // 3. sleep 40s and then send msg + mError("***idle for 30s, and then send msg***"); + taosMsleep(30000); + } else { + // do nothing + // mInfo("no error triggered"); + } + } +} diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 03b418f13f..0f903632ed 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -381,9 +381,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { } if ((pEntry->lastHbMsgId == req.msgId) && (pEntry->lastHbMsgTs == req.ts)) { - mError("vgId:%d HbMsgId:%d already handled, bh msg discard", pEntry->nodeId, req.msgId); + mError("vgId:%d HbMsgId:%d already handled, bh msg discard, and send HbRsp", pEntry->nodeId, req.msgId); - terrno = TSDB_CODE_INVALID_MSG; + // return directly and after the vnode to continue to send the next HbMsg. + terrno = TSDB_CODE_SUCCESS; doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); streamMutexUnlock(&execInfo.lock); diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 6e48c58b30..930d678ead 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -1496,73 +1496,3 @@ bool mndStreamNodeIsUpdated(SMnode *pMnode) { taosArrayDestroy(pNodeSnapshot); return updated; } - -uint32_t seed = 0; -static SRpcMsg createRpcMsg(STransAction* pAction, int64_t traceId, int64_t signature) { - SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .info.ahandle = (void *)signature}; - rpcMsg.pCont = rpcMallocCont(pAction->contLen); - if (rpcMsg.pCont == NULL) { - return rpcMsg; - } - - rpcMsg.info.traceId.rootId = traceId; - rpcMsg.info.notFreeAhandle = 1; - - memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); - return rpcMsg; -} - -void streamTransRandomErrorGen(STransAction *pAction, STrans *pTrans, int64_t signature) { - if ((pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT && pAction->id > 2) || - (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) || - (pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE && pAction->id > 2)) { - if (seed == 0) { - seed = taosGetTimestampSec(); - } - - uint32_t v = taosRandR(&seed); - int32_t choseItem = v % 5; - - if (choseItem == 0) { - // 1. one of update-checkpoint not send, restart and send it again - taosMsleep(5000); - if (pAction->msgType == TDMT_STREAM_TASK_UPDATE_CHKPT) { - mError( - "***sleep 5s and core dump, following tasks will not recv update-checkpoint info, so the checkpoint will " - "rollback***"); - exit(-1); - } else if (pAction->msgType == TDMT_STREAM_CONSEN_CHKPT) { // pAction->msgType == TDMT_STREAM_CONSEN_CHKPT - mError( - "***sleep 5s and core dump, following tasks will not recv consen-checkpoint info, so the tasks will " - "not started***"); - } else { // pAction->msgType == TDMT_VND_STREAM_CHECK_POINT_SOURCE - mError( - "***sleep 5s and core dump, following tasks will not recv checkpoint-source info, so the tasks will " - "started after restart***"); - exit(-1); - } - } else if (choseItem == 1) { - // 2. repeat send update chkpt msg - mError("***repeat send update-checkpoint/consensus/checkpoint trans msg 3times to vnode***"); - - mError("***repeat 1***"); - SRpcMsg rpcMsg1 = createRpcMsg(pAction, pTrans->mTraceId, signature); - int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg1); - - mError("***repeat 2***"); - SRpcMsg rpcMsg2 = createRpcMsg(pAction, pTrans->mTraceId, signature); - code = tmsgSendReq(&pAction->epSet, &rpcMsg2); - - mError("***repeat 3***"); - SRpcMsg rpcMsg3 = createRpcMsg(pAction, pTrans->mTraceId, signature); - code = tmsgSendReq(&pAction->epSet, &rpcMsg3); - } else if (choseItem == 2) { - // 3. sleep 40s and then send msg - mError("***idle for 30s, and then send msg***"); - taosMsleep(30000); - } else { - // do nothing - // mInfo("no error triggered"); - } - } -} \ No newline at end of file diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index bbfb9f2e15..52262c9120 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -238,6 +238,8 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32 int32_t flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); +// inject stream errors +void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index aa3b4fbe46..0146bf3564 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -222,14 +222,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check stError("s-task:%s vgId:%d current checkpointId:%" PRId64 " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (pActiveInfo->failedId >= checkpointId) { stError("s-task:%s vgId:%d checkpointId:%" PRId64 " transId:%d, has been marked failed, failedId:%" PRId64 " discard the checkpoint-trigger block", id, vgId, checkpointId, transId, pActiveInfo->failedId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (pTask->chkInfo.checkpointId == checkpointId) { @@ -371,6 +371,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock return code; } +#if 0 + chkptFailedByRetrieveReqToSource(pTask, checkpointId); +#endif + if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure diff --git a/source/libs/stream/src/streamErrorInjection.c b/source/libs/stream/src/streamErrorInjection.c new file mode 100644 index 0000000000..515845ba2b --- /dev/null +++ b/source/libs/stream/src/streamErrorInjection.c @@ -0,0 +1,17 @@ +#include "streamInt.h" + +/** + * pre-request: checkpoint interval should be 60s + * @param pTask + * @param checkpointId + */ +void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId) { + streamMutexLock(&pTask->lock); + + // set current checkpoint failed immediately, set failed checkpoint id before clear the checkpoint info + streamTaskSetFailedCheckpointId(pTask, checkpointId); + streamMutexUnlock(&pTask->lock); + + // the checkpoint interval should be 60s, and the next checkpoint req should be issued by mnode + taosMsleep(65*1000); +} \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4d8b3627ef..7ca13806f2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -850,7 +850,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_IVLD_STATUS, "Invalid task status TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_CONFLICT_EVENT, "Stream conflict event") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INTERNAL_ERROR, "Stream internal error") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_NOT_LEADER, "Stream task not on leader vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INPUTQ_FULL, "Task input queue is full") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INPUTQ_FULL, "Task input queue is full") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INVLD_CHKPT, "Invalid checkpoint trigger msg") // TDLite TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags") From 7fc7197b596660a37898483af76cb335347cd6e2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Nov 2024 14:12:05 +0800 Subject: [PATCH 003/108] fix(test): fix error in unit test. --- source/dnode/mnode/impl/test/stream/stream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index d508cf7390..45bc4c2ce2 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -246,7 +246,7 @@ TEST_F(StreamTest, kill_checkpoint_trans) { px = taosArrayPush(pStream->tasks, &pLevel); ASSERT(px != NULL); - code = mndCreateStreamResetStatusTrans(pMnode, pStream); + code = mndCreateStreamResetStatusTrans(pMnode, pStream, 1); ASSERT(code != 0); tFreeStreamObj(pStream); From af69ea78ad334266b9afd58f65326a886e4747cb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 11 Nov 2024 15:28:33 +0800 Subject: [PATCH 004/108] other: merge main --- contrib/lemon/lemon.c | 6038 ++++++++++++++++++++++++++++++++++++++++ contrib/lemon/lempar.c | 1086 ++++++++ 2 files changed, 7124 insertions(+) create mode 100644 contrib/lemon/lemon.c create mode 100644 contrib/lemon/lempar.c diff --git a/contrib/lemon/lemon.c b/contrib/lemon/lemon.c new file mode 100644 index 0000000000..d92df2a1a7 --- /dev/null +++ b/contrib/lemon/lemon.c @@ -0,0 +1,6038 @@ +/* +** This file contains all sources (including headers) to the LEMON +** LALR(1) parser generator. The sources have been combined into a +** single file to make it easy to include LEMON in the source tree +** and Makefile of another program. +** +** The author of this program disclaims copyright. +*/ +#include +#include +#include +#include +#include +#include + +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) +#define ISALNUM(X) isalnum((unsigned char)(X)) +#define ISALPHA(X) isalpha((unsigned char)(X)) +#define ISUPPER(X) isupper((unsigned char)(X)) +#define ISLOWER(X) islower((unsigned char)(X)) + + +#ifndef __WIN32__ +# if defined(_WIN32) || defined(WIN32) +# define __WIN32__ +# endif +#endif + +#ifdef __WIN32__ +#ifdef __cplusplus +extern "C" { +#endif +extern int access(const char *path, int mode); +#ifdef __cplusplus +} +#endif +#else +#include +#endif + +/* #define PRIVATE static */ +#define PRIVATE + +#ifdef TEST +#define MAXRHS 5 /* Set low to exercise exception code */ +#else +#define MAXRHS 1000 +#endif + +extern void memory_error(); +static int showPrecedenceConflict = 0; +static char *msort(char*,char**,int(*)(const char*,const char*)); + +/* +** Compilers are getting increasingly pedantic about type conversions +** as C evolves ever closer to Ada.... To work around the latest problems +** we have to define the following variant of strlen(). +*/ +#define lemonStrlen(X) ((int)strlen(X)) + +/* +** Header on the linked list of memory allocations. +*/ +typedef struct MemChunk MemChunk; +struct MemChunk { + MemChunk *pNext; + size_t sz; + /* Actually memory follows */ +}; + +/* +** Global linked list of all memory allocations. +*/ +static MemChunk *memChunkList = 0; + +/* +** Wrappers around malloc(), calloc(), realloc() and free(). +** +** All memory allocations are kept on a doubly-linked list. The +** lemon_free_all() function can be called prior to exit to clean +** up any memory leaks. +** +** This is not necessary. But compilers and getting increasingly +** fussy about memory leaks, even in command-line programs like Lemon +** where they do not matter. So this code is provided to hush the +** warnings. +*/ +static void *lemon_malloc(size_t nByte){ + MemChunk *p; + if( nByte<0 ) return 0; + p = malloc( nByte + sizeof(MemChunk) ); + if( p==0 ){ + fprintf(stderr, "Out of memory. Failed to allocate %lld bytes.\n", + (long long int)nByte); + exit(1); + } + p->pNext = memChunkList; + p->sz = nByte; + memChunkList = p; + return (void*)&p[1]; +} +static void *lemon_calloc(size_t nElem, size_t sz){ + void *p = lemon_malloc(nElem*sz); + memset(p, 0, nElem*sz); + return p; +} +static void lemon_free(void *pOld){ + if( pOld ){ + MemChunk *p = (MemChunk*)pOld; + p--; + memset(pOld, 0, p->sz); + } +} +static void *lemon_realloc(void *pOld, size_t nNew){ + void *pNew; + MemChunk *p; + if( pOld==0 ) return lemon_malloc(nNew); + p = (MemChunk*)pOld; + p--; + if( p->sz>=nNew ) return pOld; + pNew = lemon_malloc( nNew ); + memcpy(pNew, pOld, p->sz); + return pNew; +} + +/* Free all outstanding memory allocations. +** Do this right before exiting. +*/ +static void lemon_free_all(void){ + while( memChunkList ){ + MemChunk *pNext = memChunkList->pNext; + free( memChunkList ); + memChunkList = pNext; + } +} + +/* +** Compilers are starting to complain about the use of sprintf() and strcpy(), +** saying they are unsafe. So we define our own versions of those routines too. +** +** There are three routines here: lemon_sprintf(), lemon_vsprintf(), and +** lemon_addtext(). The first two are replacements for sprintf() and vsprintf(). +** The third is a helper routine for vsnprintf() that adds texts to the end of a +** buffer, making sure the buffer is always zero-terminated. +** +** The string formatter is a minimal subset of stdlib sprintf() supporting only +** a few simply conversions: +** +** %d +** %s +** %.*s +** +*/ +static void lemon_addtext( + char *zBuf, /* The buffer to which text is added */ + int *pnUsed, /* Slots of the buffer used so far */ + const char *zIn, /* Text to add */ + int nIn, /* Bytes of text to add. -1 to use strlen() */ + int iWidth /* Field width. Negative to left justify */ +){ + if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){} + while( iWidth>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth--; } + if( nIn==0 ) return; + memcpy(&zBuf[*pnUsed], zIn, nIn); + *pnUsed += nIn; + while( (-iWidth)>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth++; } + zBuf[*pnUsed] = 0; +} +static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ + int i, j, k, c; + int nUsed = 0; + const char *z; + char zTemp[50]; + str[0] = 0; + for(i=j=0; (c = zFormat[i])!=0; i++){ + if( c=='%' ){ + int iWidth = 0; + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + c = zFormat[++i]; + if( ISDIGIT(c) || (c=='-' && ISDIGIT(zFormat[i+1])) ){ + if( c=='-' ) i++; + while( ISDIGIT(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0'; + if( c=='-' ) iWidth = -iWidth; + c = zFormat[i]; + } + if( c=='d' ){ + int v = va_arg(ap, int); + if( v<0 ){ + lemon_addtext(str, &nUsed, "-", 1, iWidth); + v = -v; + }else if( v==0 ){ + lemon_addtext(str, &nUsed, "0", 1, iWidth); + } + k = 0; + while( v>0 ){ + k++; + zTemp[sizeof(zTemp)-k] = (v%10) + '0'; + v /= 10; + } + lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth); + }else if( c=='s' ){ + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, -1, iWidth); + }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){ + i += 2; + k = va_arg(ap, int); + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, k, iWidth); + }else if( c=='%' ){ + lemon_addtext(str, &nUsed, "%", 1, 0); + }else{ + fprintf(stderr, "illegal format\n"); + exit(1); + } + j = i+1; + } + } + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + return nUsed; +} +static int lemon_sprintf(char *str, const char *format, ...){ + va_list ap; + int rc; + va_start(ap, format); + rc = lemon_vsprintf(str, format, ap); + va_end(ap); + return rc; +} +static void lemon_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} +static void lemon_strcat(char *dest, const char *src){ + while( *dest ) dest++; + lemon_strcpy(dest, src); +} + + +/* a few forward declarations... */ +struct rule; +struct lemon; +struct action; + +static struct action *Action_new(void); +static struct action *Action_sort(struct action *); + +/********** From the file "build.h" ************************************/ +void FindRulePrecedences(struct lemon*); +void FindFirstSets(struct lemon*); +void FindStates(struct lemon*); +void FindLinks(struct lemon*); +void FindFollowSets(struct lemon*); +void FindActions(struct lemon*); + +/********* From the file "configlist.h" *********************************/ +void Configlist_init(void); +struct config *Configlist_add(struct rule *, int); +struct config *Configlist_addbasis(struct rule *, int); +void Configlist_closure(struct lemon *); +void Configlist_sort(void); +void Configlist_sortbasis(void); +struct config *Configlist_return(void); +struct config *Configlist_basis(void); +void Configlist_eat(struct config *); +void Configlist_reset(void); + +/********* From the file "error.h" ***************************************/ +void ErrorMsg(const char *, int,const char *, ...); + +/****** From the file "option.h" ******************************************/ +enum option_type { OPT_FLAG=1, OPT_INT, OPT_DBL, OPT_STR, + OPT_FFLAG, OPT_FINT, OPT_FDBL, OPT_FSTR}; +struct s_options { + enum option_type type; + const char *label; + char *arg; + const char *message; +}; +int OptInit(char**,struct s_options*,FILE*); +int OptNArgs(void); +char *OptArg(int); +void OptErr(int); +void OptPrint(void); + +/******** From the file "parse.h" *****************************************/ +void Parse(struct lemon *lemp); + +/********* From the file "plink.h" ***************************************/ +struct plink *Plink_new(void); +void Plink_add(struct plink **, struct config *); +void Plink_copy(struct plink **, struct plink *); +void Plink_delete(struct plink *); + +/********** From the file "report.h" *************************************/ +void Reprint(struct lemon *); +void ReportOutput(struct lemon *); +void ReportTable(struct lemon *, int, int); +void ReportHeader(struct lemon *); +void CompressTables(struct lemon *); +void ResortStates(struct lemon *); + +/********** From the file "set.h" ****************************************/ +void SetSize(int); /* All sets will be of size N */ +char *SetNew(void); /* A new set for element 0..N */ +void SetFree(char*); /* Deallocate a set */ +int SetAdd(char*,int); /* Add element to a set */ +int SetUnion(char *,char *); /* A <- A U B, thru element N */ +#define SetFind(X,Y) (X[Y]) /* True if Y is in set X */ + +/********** From the file "struct.h" *************************************/ +/* +** Principal data structures for the LEMON parser generator. +*/ + +typedef enum {LEMON_FALSE=0, LEMON_TRUE} Boolean; + +/* Symbols (terminals and nonterminals) of the grammar are stored +** in the following: */ +enum symbol_type { + TERMINAL, + NONTERMINAL, + MULTITERMINAL +}; +enum e_assoc { + LEFT, + RIGHT, + NONE, + UNK +}; +struct symbol { + const char *name; /* Name of the symbol */ + int index; /* Index number for this symbol */ + enum symbol_type type; /* Symbols are all either TERMINALS or NTs */ + struct rule *rule; /* Linked list of rules of this (if an NT) */ + struct symbol *fallback; /* fallback token in case this token doesn't parse */ + int prec; /* Precedence if defined (-1 otherwise) */ + enum e_assoc assoc; /* Associativity if precedence is defined */ + char *firstset; /* First-set for all rules of this symbol */ + Boolean lambda; /* True if NT and can generate an empty string */ + int useCnt; /* Number of times used */ + char *destructor; /* Code which executes whenever this symbol is + ** popped from the stack during error processing */ + int destLineno; /* Line number for start of destructor. Set to + ** -1 for duplicate destructors. */ + char *datatype; /* The data type of information held by this + ** object. Only used if type==NONTERMINAL */ + int dtnum; /* The data type number. In the parser, the value + ** stack is a union. The .yy%d element of this + ** union is the correct data type for this object */ + int bContent; /* True if this symbol ever carries content - if + ** it is ever more than just syntax */ + /* The following fields are used by MULTITERMINALs only */ + int nsubsym; /* Number of constituent symbols in the MULTI */ + struct symbol **subsym; /* Array of constituent symbols */ +}; + +/* Each production rule in the grammar is stored in the following +** structure. */ +struct rule { + struct symbol *lhs; /* Left-hand side of the rule */ + const char *lhsalias; /* Alias for the LHS (NULL if none) */ + int lhsStart; /* True if left-hand side is the start symbol */ + int ruleline; /* Line number for the rule */ + int nrhs; /* Number of RHS symbols */ + struct symbol **rhs; /* The RHS symbols */ + const char **rhsalias; /* An alias for each RHS symbol (NULL if none) */ + int line; /* Line number at which code begins */ + const char *code; /* The code executed when this rule is reduced */ + const char *codePrefix; /* Setup code before code[] above */ + const char *codeSuffix; /* Breakdown code after code[] above */ + struct symbol *precsym; /* Precedence symbol for this rule */ + int index; /* An index number for this rule */ + int iRule; /* Rule number as used in the generated tables */ + Boolean noCode; /* True if this rule has no associated C code */ + Boolean codeEmitted; /* True if the code has been emitted already */ + Boolean canReduce; /* True if this rule is ever reduced */ + Boolean doesReduce; /* Reduce actions occur after optimization */ + Boolean neverReduce; /* Reduce is theoretically possible, but prevented + ** by actions or other outside implementation */ + struct rule *nextlhs; /* Next rule with the same LHS */ + struct rule *next; /* Next rule in the global list */ +}; + +/* A configuration is a production rule of the grammar together with +** a mark (dot) showing how much of that rule has been processed so far. +** Configurations also contain a follow-set which is a list of terminal +** symbols which are allowed to immediately follow the end of the rule. +** Every configuration is recorded as an instance of the following: */ +enum cfgstatus { + COMPLETE, + INCOMPLETE +}; +struct config { + struct rule *rp; /* The rule upon which the configuration is based */ + int dot; /* The parse point */ + char *fws; /* Follow-set for this configuration only */ + struct plink *fplp; /* Follow-set forward propagation links */ + struct plink *bplp; /* Follow-set backwards propagation links */ + struct state *stp; /* Pointer to state which contains this */ + enum cfgstatus status; /* used during followset and shift computations */ + struct config *next; /* Next configuration in the state */ + struct config *bp; /* The next basis configuration */ +}; + +enum e_action { + SHIFT, + ACCEPT, + REDUCE, + ERROR, + SSCONFLICT, /* A shift/shift conflict */ + SRCONFLICT, /* Was a reduce, but part of a conflict */ + RRCONFLICT, /* Was a reduce, but part of a conflict */ + SH_RESOLVED, /* Was a shift. Precedence resolved conflict */ + RD_RESOLVED, /* Was reduce. Precedence resolved conflict */ + NOT_USED, /* Deleted by compression */ + SHIFTREDUCE /* Shift first, then reduce */ +}; + +/* Every shift or reduce operation is stored as one of the following */ +struct action { + struct symbol *sp; /* The look-ahead symbol */ + enum e_action type; + union { + struct state *stp; /* The new state, if a shift */ + struct rule *rp; /* The rule, if a reduce */ + } x; + struct symbol *spOpt; /* SHIFTREDUCE optimization to this symbol */ + struct action *next; /* Next action for this state */ + struct action *collide; /* Next action with the same hash */ +}; + +/* Each state of the generated parser's finite state machine +** is encoded as an instance of the following structure. */ +struct state { + struct config *bp; /* The basis configurations for this state */ + struct config *cfp; /* All configurations in this set */ + int statenum; /* Sequential number for this state */ + struct action *ap; /* List of actions for this state */ + int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ + int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ + int iDfltReduce; /* Default action is to REDUCE by this rule */ + struct rule *pDfltReduce;/* The default REDUCE rule. */ + int autoReduce; /* True if this is an auto-reduce state */ +}; +#define NO_OFFSET (-2147483647) + +/* A followset propagation link indicates that the contents of one +** configuration followset should be propagated to another whenever +** the first changes. */ +struct plink { + struct config *cfp; /* The configuration to which linked */ + struct plink *next; /* The next propagate link */ +}; + +/* The state vector for the entire parser generator is recorded as +** follows. (LEMON uses no global variables and makes little use of +** static variables. Fields in the following structure can be thought +** of as begin global variables in the program.) */ +struct lemon { + struct state **sorted; /* Table of states sorted by state number */ + struct rule *rule; /* List of all rules */ + struct rule *startRule; /* First rule */ + int nstate; /* Number of states */ + int nxstate; /* nstate with tail degenerate states removed */ + int nrule; /* Number of rules */ + int nruleWithAction; /* Number of rules with actions */ + int nsymbol; /* Number of terminal and nonterminal symbols */ + int nterminal; /* Number of terminal symbols */ + int minShiftReduce; /* Minimum shift-reduce action value */ + int errAction; /* Error action value */ + int accAction; /* Accept action value */ + int noAction; /* No-op action value */ + int minReduce; /* Minimum reduce action */ + int maxAction; /* Maximum action value of any kind */ + struct symbol **symbols; /* Sorted array of pointers to symbols */ + int errorcnt; /* Number of errors */ + struct symbol *errsym; /* The error symbol */ + struct symbol *wildcard; /* Token that matches anything */ + char *name; /* Name of the generated parser */ + char *arg; /* Declaration of the 3rd argument to parser */ + char *ctx; /* Declaration of 2nd argument to constructor */ + char *tokentype; /* Type of terminal symbols in the parser stack */ + char *vartype; /* The default type of non-terminal symbols */ + char *start; /* Name of the start symbol for the grammar */ + char *stacksize; /* Size of the parser stack */ + char *include; /* Code to put at the start of the C file */ + char *error; /* Code to execute when an error is seen */ + char *overflow; /* Code to execute on a stack overflow */ + char *failure; /* Code to execute on parser failure */ + char *accept; /* Code to execute when the parser excepts */ + char *extracode; /* Code appended to the generated file */ + char *tokendest; /* Code to execute to destroy token data */ + char *vardest; /* Code for the default non-terminal destructor */ + char *filename; /* Name of the input file */ + char *outname; /* Name of the current output file */ + char *tokenprefix; /* A prefix added to token names in the .h file */ + char *reallocFunc; /* Function to use to allocate stack space */ + char *freeFunc; /* Function to use to free stack space */ + int nconflict; /* Number of parsing conflicts */ + int nactiontab; /* Number of entries in the yy_action[] table */ + int nlookaheadtab; /* Number of entries in yy_lookahead[] */ + int tablesize; /* Total table size of all tables in bytes */ + int basisflag; /* Print only basis configurations */ + int printPreprocessed; /* Show preprocessor output on stdout */ + int has_fallback; /* True if any %fallback is seen in the grammar */ + int nolinenosflag; /* True if #line statements should not be printed */ + int argc; /* Number of command-line arguments */ + char **argv; /* Command-line arguments */ +}; + +#define MemoryCheck(X) if((X)==0){ \ + extern void memory_error(); \ + memory_error(); \ +} + +/**************** From the file "table.h" *********************************/ +/* +** All code in this file has been automatically generated +** from a specification in the file +** "table.q" +** by the associative array code building program "aagen". +** Do not edit this file! Instead, edit the specification +** file, then rerun aagen. +*/ +/* +** Code for processing tables in the LEMON parser generator. +*/ +/* Routines for handling a strings */ + +const char *Strsafe(const char *); + +void Strsafe_init(void); +int Strsafe_insert(const char *); +const char *Strsafe_find(const char *); + +/* Routines for handling symbols of the grammar */ + +struct symbol *Symbol_new(const char *); +int Symbolcmpp(const void *, const void *); +void Symbol_init(void); +int Symbol_insert(struct symbol *, const char *); +struct symbol *Symbol_find(const char *); +struct symbol *Symbol_Nth(int); +int Symbol_count(void); +struct symbol **Symbol_arrayof(void); + +/* Routines to manage the state table */ + +int Configcmp(const char *, const char *); +struct state *State_new(void); +void State_init(void); +int State_insert(struct state *, struct config *); +struct state *State_find(struct config *); +struct state **State_arrayof(void); + +/* Routines used for efficiency in Configlist_add */ + +void Configtable_init(void); +int Configtable_insert(struct config *); +struct config *Configtable_find(struct config *); +void Configtable_clear(int(*)(struct config *)); + +/****************** From the file "action.c" *******************************/ +/* +** Routines processing parser actions in the LEMON parser generator. +*/ + +/* Allocate a new parser action */ +static struct action *Action_new(void){ + static struct action *actionfreelist = 0; + struct action *newaction; + + if( actionfreelist==0 ){ + int i; + int amt = 100; + actionfreelist = (struct action *)lemon_calloc(amt, sizeof(struct action)); + if( actionfreelist==0 ){ + fprintf(stderr,"Unable to allocate memory for a new parser action."); + exit(1); + } + for(i=0; inext; + return newaction; +} + +/* Compare two actions for sorting purposes. Return negative, zero, or +** positive if the first action is less than, equal to, or greater than +** the first +*/ +static int actioncmp( + struct action *ap1, + struct action *ap2 +){ + int rc; + rc = ap1->sp->index - ap2->sp->index; + if( rc==0 ){ + rc = (int)ap1->type - (int)ap2->type; + } + if( rc==0 && (ap1->type==REDUCE || ap1->type==SHIFTREDUCE) ){ + rc = ap1->x.rp->index - ap2->x.rp->index; + } + if( rc==0 ){ + rc = (int) (ap2 - ap1); + } + return rc; +} + +/* Sort parser actions */ +static struct action *Action_sort( + struct action *ap +){ + ap = (struct action *)msort((char *)ap,(char **)&ap->next, + (int(*)(const char*,const char*))actioncmp); + return ap; +} + +void Action_add( + struct action **app, + enum e_action type, + struct symbol *sp, + char *arg +){ + struct action *newaction; + newaction = Action_new(); + newaction->next = *app; + *app = newaction; + newaction->type = type; + newaction->sp = sp; + newaction->spOpt = 0; + if( type==SHIFT ){ + newaction->x.stp = (struct state *)arg; + }else{ + newaction->x.rp = (struct rule *)arg; + } +} +/********************** New code to implement the "acttab" module ***********/ +/* +** This module implements routines use to construct the yy_action[] table. +*/ + +/* +** The state of the yy_action table under construction is an instance of +** the following structure. +** +** The yy_action table maps the pair (state_number, lookahead) into an +** action_number. The table is an array of integers pairs. The state_number +** determines an initial offset into the yy_action array. The lookahead +** value is then added to this initial offset to get an index X into the +** yy_action array. If the aAction[X].lookahead equals the value of the +** of the lookahead input, then the value of the action_number output is +** aAction[X].action. If the lookaheads do not match then the +** default action for the state_number is returned. +** +** All actions associated with a single state_number are first entered +** into aLookahead[] using multiple calls to acttab_action(). Then the +** actions for that single state_number are placed into the aAction[] +** array with a single call to acttab_insert(). The acttab_insert() call +** also resets the aLookahead[] array in preparation for the next +** state number. +*/ +struct lookahead_action { + int lookahead; /* Value of the lookahead token */ + int action; /* Action to take on the given lookahead */ +}; +typedef struct acttab acttab; +struct acttab { + int nAction; /* Number of used slots in aAction[] */ + int nActionAlloc; /* Slots allocated for aAction[] */ + struct lookahead_action + *aAction, /* The yy_action[] table under construction */ + *aLookahead; /* A single new transaction set */ + int mnLookahead; /* Minimum aLookahead[].lookahead */ + int mnAction; /* Action associated with mnLookahead */ + int mxLookahead; /* Maximum aLookahead[].lookahead */ + int nLookahead; /* Used slots in aLookahead[] */ + int nLookaheadAlloc; /* Slots allocated in aLookahead[] */ + int nterminal; /* Number of terminal symbols */ + int nsymbol; /* total number of symbols */ +}; + +/* Return the number of entries in the yy_action table */ +#define acttab_lookahead_size(X) ((X)->nAction) + +/* The value for the N-th entry in yy_action */ +#define acttab_yyaction(X,N) ((X)->aAction[N].action) + +/* The value for the N-th entry in yy_lookahead */ +#define acttab_yylookahead(X,N) ((X)->aAction[N].lookahead) + +/* Free all memory associated with the given acttab */ +void acttab_free(acttab *p){ + lemon_free( p->aAction ); + lemon_free( p->aLookahead ); + lemon_free( p ); +} + +/* Allocate a new acttab structure */ +acttab *acttab_alloc(int nsymbol, int nterminal){ + acttab *p = (acttab *) lemon_calloc( 1, sizeof(*p) ); + if( p==0 ){ + fprintf(stderr,"Unable to allocate memory for a new acttab."); + exit(1); + } + memset(p, 0, sizeof(*p)); + p->nsymbol = nsymbol; + p->nterminal = nterminal; + return p; +} + +/* Add a new action to the current transaction set. +** +** This routine is called once for each lookahead for a particular +** state. +*/ +void acttab_action(acttab *p, int lookahead, int action){ + if( p->nLookahead>=p->nLookaheadAlloc ){ + p->nLookaheadAlloc += 25; + p->aLookahead = (struct lookahead_action *) lemon_realloc( p->aLookahead, + sizeof(p->aLookahead[0])*p->nLookaheadAlloc ); + if( p->aLookahead==0 ){ + fprintf(stderr,"malloc failed\n"); + exit(1); + } + } + if( p->nLookahead==0 ){ + p->mxLookahead = lookahead; + p->mnLookahead = lookahead; + p->mnAction = action; + }else{ + if( p->mxLookaheadmxLookahead = lookahead; + if( p->mnLookahead>lookahead ){ + p->mnLookahead = lookahead; + p->mnAction = action; + } + } + p->aLookahead[p->nLookahead].lookahead = lookahead; + p->aLookahead[p->nLookahead].action = action; + p->nLookahead++; +} + +/* +** Add the transaction set built up with prior calls to acttab_action() +** into the current action table. Then reset the transaction set back +** to an empty set in preparation for a new round of acttab_action() calls. +** +** Return the offset into the action table of the new transaction. +** +** If the makeItSafe parameter is true, then the offset is chosen so that +** it is impossible to overread the yy_lookaside[] table regardless of +** the lookaside token. This is done for the terminal symbols, as they +** come from external inputs and can contain syntax errors. When makeItSafe +** is false, there is more flexibility in selecting offsets, resulting in +** a smaller table. For non-terminal symbols, which are never syntax errors, +** makeItSafe can be false. +*/ +int acttab_insert(acttab *p, int makeItSafe){ + int i, j, k, n, end; + assert( p->nLookahead>0 ); + + /* Make sure we have enough space to hold the expanded action table + ** in the worst case. The worst case occurs if the transaction set + ** must be appended to the current action table + */ + n = p->nsymbol + 1; + if( p->nAction + n >= p->nActionAlloc ){ + int oldAlloc = p->nActionAlloc; + p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; + p->aAction = (struct lookahead_action *) lemon_realloc( p->aAction, + sizeof(p->aAction[0])*p->nActionAlloc); + if( p->aAction==0 ){ + fprintf(stderr,"malloc failed\n"); + exit(1); + } + for(i=oldAlloc; inActionAlloc; i++){ + p->aAction[i].lookahead = -1; + p->aAction[i].action = -1; + } + } + + /* Scan the existing action table looking for an offset that is a + ** duplicate of the current transaction set. Fall out of the loop + ** if and when the duplicate is found. + ** + ** i is the index in p->aAction[] where p->mnLookahead is inserted. + */ + end = makeItSafe ? p->mnLookahead : 0; + for(i=p->nAction-1; i>=end; i--){ + if( p->aAction[i].lookahead==p->mnLookahead ){ + /* All lookaheads and actions in the aLookahead[] transaction + ** must match against the candidate aAction[i] entry. */ + if( p->aAction[i].action!=p->mnAction ) continue; + for(j=0; jnLookahead; j++){ + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + if( k<0 || k>=p->nAction ) break; + if( p->aLookahead[j].lookahead!=p->aAction[k].lookahead ) break; + if( p->aLookahead[j].action!=p->aAction[k].action ) break; + } + if( jnLookahead ) continue; + + /* No possible lookahead value that is not in the aLookahead[] + ** transaction is allowed to match aAction[i] */ + n = 0; + for(j=0; jnAction; j++){ + if( p->aAction[j].lookahead<0 ) continue; + if( p->aAction[j].lookahead==j+p->mnLookahead-i ) n++; + } + if( n==p->nLookahead ){ + break; /* An exact match is found at offset i */ + } + } + } + + /* If no existing offsets exactly match the current transaction, find an + ** an empty offset in the aAction[] table in which we can add the + ** aLookahead[] transaction. + */ + if( inAction, which means the + ** transaction will be appended. */ + i = makeItSafe ? p->mnLookahead : 0; + for(; inActionAlloc - p->mxLookahead; i++){ + if( p->aAction[i].lookahead<0 ){ + for(j=0; jnLookahead; j++){ + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + if( k<0 ) break; + if( p->aAction[k].lookahead>=0 ) break; + } + if( jnLookahead ) continue; + for(j=0; jnAction; j++){ + if( p->aAction[j].lookahead==j+p->mnLookahead-i ) break; + } + if( j==p->nAction ){ + break; /* Fits in empty slots */ + } + } + } + } + /* Insert transaction set at index i. */ +#if 0 + printf("Acttab:"); + for(j=0; jnLookahead; j++){ + printf(" %d", p->aLookahead[j].lookahead); + } + printf(" inserted at %d\n", i); +#endif + for(j=0; jnLookahead; j++){ + k = p->aLookahead[j].lookahead - p->mnLookahead + i; + p->aAction[k] = p->aLookahead[j]; + if( k>=p->nAction ) p->nAction = k+1; + } + if( makeItSafe && i+p->nterminal>=p->nAction ) p->nAction = i+p->nterminal+1; + p->nLookahead = 0; + + /* Return the offset that is added to the lookahead in order to get the + ** index into yy_action of the action */ + return i - p->mnLookahead; +} + +/* +** Return the size of the action table without the trailing syntax error +** entries. +*/ +int acttab_action_size(acttab *p){ + int n = p->nAction; + while( n>0 && p->aAction[n-1].lookahead<0 ){ n--; } + return n; +} + +/********************** From the file "build.c" *****************************/ +/* +** Routines to construction the finite state machine for the LEMON +** parser generator. +*/ + +/* Find a precedence symbol of every rule in the grammar. +** +** Those rules which have a precedence symbol coded in the input +** grammar using the "[symbol]" construct will already have the +** rp->precsym field filled. Other rules take as their precedence +** symbol the first RHS symbol with a defined precedence. If there +** are not RHS symbols with a defined precedence, the precedence +** symbol field is left blank. +*/ +void FindRulePrecedences(struct lemon *xp) +{ + struct rule *rp; + for(rp=xp->rule; rp; rp=rp->next){ + if( rp->precsym==0 ){ + int i, j; + for(i=0; inrhs && rp->precsym==0; i++){ + struct symbol *sp = rp->rhs[i]; + if( sp->type==MULTITERMINAL ){ + for(j=0; jnsubsym; j++){ + if( sp->subsym[j]->prec>=0 ){ + rp->precsym = sp->subsym[j]; + break; + } + } + }else if( sp->prec>=0 ){ + rp->precsym = rp->rhs[i]; + } + } + } + } + return; +} + +/* Find all nonterminals which will generate the empty string. +** Then go back and compute the first sets of every nonterminal. +** The first set is the set of all terminal symbols which can begin +** a string generated by that nonterminal. +*/ +void FindFirstSets(struct lemon *lemp) +{ + int i, j; + struct rule *rp; + int progress; + + for(i=0; insymbol; i++){ + lemp->symbols[i]->lambda = LEMON_FALSE; + } + for(i=lemp->nterminal; insymbol; i++){ + lemp->symbols[i]->firstset = SetNew(); + } + + /* First compute all lambdas */ + do{ + progress = 0; + for(rp=lemp->rule; rp; rp=rp->next){ + if( rp->lhs->lambda ) continue; + for(i=0; inrhs; i++){ + struct symbol *sp = rp->rhs[i]; + assert( sp->type==NONTERMINAL || sp->lambda==LEMON_FALSE ); + if( sp->lambda==LEMON_FALSE ) break; + } + if( i==rp->nrhs ){ + rp->lhs->lambda = LEMON_TRUE; + progress = 1; + } + } + }while( progress ); + + /* Now compute all first sets */ + do{ + struct symbol *s1, *s2; + progress = 0; + for(rp=lemp->rule; rp; rp=rp->next){ + s1 = rp->lhs; + for(i=0; inrhs; i++){ + s2 = rp->rhs[i]; + if( s2->type==TERMINAL ){ + progress += SetAdd(s1->firstset,s2->index); + break; + }else if( s2->type==MULTITERMINAL ){ + for(j=0; jnsubsym; j++){ + progress += SetAdd(s1->firstset,s2->subsym[j]->index); + } + break; + }else if( s1==s2 ){ + if( s1->lambda==LEMON_FALSE ) break; + }else{ + progress += SetUnion(s1->firstset,s2->firstset); + if( s2->lambda==LEMON_FALSE ) break; + } + } + } + }while( progress ); + return; +} + +/* Compute all LR(0) states for the grammar. Links +** are added to between some states so that the LR(1) follow sets +** can be computed later. +*/ +PRIVATE struct state *getstate(struct lemon *); /* forward reference */ +void FindStates(struct lemon *lemp) +{ + struct symbol *sp; + struct rule *rp; + + Configlist_init(); + + /* Find the start symbol */ + if( lemp->start ){ + sp = Symbol_find(lemp->start); + if( sp==0 ){ + ErrorMsg(lemp->filename,0, + "The specified start symbol \"%s\" is not " + "in a nonterminal of the grammar. \"%s\" will be used as the start " + "symbol instead.",lemp->start,lemp->startRule->lhs->name); + lemp->errorcnt++; + sp = lemp->startRule->lhs; + } + }else if( lemp->startRule ){ + sp = lemp->startRule->lhs; + }else{ + ErrorMsg(lemp->filename,0,"Internal error - no start rule\n"); + exit(1); + } + + /* Make sure the start symbol doesn't occur on the right-hand side of + ** any rule. Report an error if it does. (YACC would generate a new + ** start symbol in this case.) */ + for(rp=lemp->rule; rp; rp=rp->next){ + int i; + for(i=0; inrhs; i++){ + if( rp->rhs[i]==sp ){ /* FIX ME: Deal with multiterminals */ + ErrorMsg(lemp->filename,0, + "The start symbol \"%s\" occurs on the " + "right-hand side of a rule. This will result in a parser which " + "does not work properly.",sp->name); + lemp->errorcnt++; + } + } + } + + /* The basis configuration set for the first state + ** is all rules which have the start symbol as their + ** left-hand side */ + for(rp=sp->rule; rp; rp=rp->nextlhs){ + struct config *newcfp; + rp->lhsStart = 1; + newcfp = Configlist_addbasis(rp,0); + SetAdd(newcfp->fws,0); + } + + /* Compute the first state. All other states will be + ** computed automatically during the computation of the first one. + ** The returned pointer to the first state is not used. */ + (void)getstate(lemp); + return; +} + +/* Return a pointer to a state which is described by the configuration +** list which has been built from calls to Configlist_add. +*/ +PRIVATE void buildshifts(struct lemon *, struct state *); /* Forwd ref */ +PRIVATE struct state *getstate(struct lemon *lemp) +{ + struct config *cfp, *bp; + struct state *stp; + + /* Extract the sorted basis of the new state. The basis was constructed + ** by prior calls to "Configlist_addbasis()". */ + Configlist_sortbasis(); + bp = Configlist_basis(); + + /* Get a state with the same basis */ + stp = State_find(bp); + if( stp ){ + /* A state with the same basis already exists! Copy all the follow-set + ** propagation links from the state under construction into the + ** preexisting state, then return a pointer to the preexisting state */ + struct config *x, *y; + for(x=bp, y=stp->bp; x && y; x=x->bp, y=y->bp){ + Plink_copy(&y->bplp,x->bplp); + Plink_delete(x->fplp); + x->fplp = x->bplp = 0; + } + cfp = Configlist_return(); + Configlist_eat(cfp); + }else{ + /* This really is a new state. Construct all the details */ + Configlist_closure(lemp); /* Compute the configuration closure */ + Configlist_sort(); /* Sort the configuration closure */ + cfp = Configlist_return(); /* Get a pointer to the config list */ + stp = State_new(); /* A new state structure */ + MemoryCheck(stp); + stp->bp = bp; /* Remember the configuration basis */ + stp->cfp = cfp; /* Remember the configuration closure */ + stp->statenum = lemp->nstate++; /* Every state gets a sequence number */ + stp->ap = 0; /* No actions, yet. */ + State_insert(stp,stp->bp); /* Add to the state table */ + buildshifts(lemp,stp); /* Recursively compute successor states */ + } + return stp; +} + +/* +** Return true if two symbols are the same. +*/ +int same_symbol(struct symbol *a, struct symbol *b) +{ + int i; + if( a==b ) return 1; + if( a->type!=MULTITERMINAL ) return 0; + if( b->type!=MULTITERMINAL ) return 0; + if( a->nsubsym!=b->nsubsym ) return 0; + for(i=0; insubsym; i++){ + if( a->subsym[i]!=b->subsym[i] ) return 0; + } + return 1; +} + +/* Construct all successor states to the given state. A "successor" +** state is any state which can be reached by a shift action. +*/ +PRIVATE void buildshifts(struct lemon *lemp, struct state *stp) +{ + struct config *cfp; /* For looping thru the config closure of "stp" */ + struct config *bcfp; /* For the inner loop on config closure of "stp" */ + struct config *newcfg; /* */ + struct symbol *sp; /* Symbol following the dot in configuration "cfp" */ + struct symbol *bsp; /* Symbol following the dot in configuration "bcfp" */ + struct state *newstp; /* A pointer to a successor state */ + + /* Each configuration becomes complete after it contributes to a successor + ** state. Initially, all configurations are incomplete */ + for(cfp=stp->cfp; cfp; cfp=cfp->next) cfp->status = INCOMPLETE; + + /* Loop through all configurations of the state "stp" */ + for(cfp=stp->cfp; cfp; cfp=cfp->next){ + if( cfp->status==COMPLETE ) continue; /* Already used by inner loop */ + if( cfp->dot>=cfp->rp->nrhs ) continue; /* Can't shift this config */ + Configlist_reset(); /* Reset the new config set */ + sp = cfp->rp->rhs[cfp->dot]; /* Symbol after the dot */ + + /* For every configuration in the state "stp" which has the symbol "sp" + ** following its dot, add the same configuration to the basis set under + ** construction but with the dot shifted one symbol to the right. */ + for(bcfp=cfp; bcfp; bcfp=bcfp->next){ + if( bcfp->status==COMPLETE ) continue; /* Already used */ + if( bcfp->dot>=bcfp->rp->nrhs ) continue; /* Can't shift this one */ + bsp = bcfp->rp->rhs[bcfp->dot]; /* Get symbol after dot */ + if( !same_symbol(bsp,sp) ) continue; /* Must be same as for "cfp" */ + bcfp->status = COMPLETE; /* Mark this config as used */ + newcfg = Configlist_addbasis(bcfp->rp,bcfp->dot+1); + Plink_add(&newcfg->bplp,bcfp); + } + + /* Get a pointer to the state described by the basis configuration set + ** constructed in the preceding loop */ + newstp = getstate(lemp); + + /* The state "newstp" is reached from the state "stp" by a shift action + ** on the symbol "sp" */ + if( sp->type==MULTITERMINAL ){ + int i; + for(i=0; insubsym; i++){ + Action_add(&stp->ap,SHIFT,sp->subsym[i],(char*)newstp); + } + }else{ + Action_add(&stp->ap,SHIFT,sp,(char *)newstp); + } + } +} + +/* +** Construct the propagation links +*/ +void FindLinks(struct lemon *lemp) +{ + int i; + struct config *cfp, *other; + struct state *stp; + struct plink *plp; + + /* Housekeeping detail: + ** Add to every propagate link a pointer back to the state to + ** which the link is attached. */ + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + for(cfp=stp?stp->cfp:0; cfp; cfp=cfp->next){ + cfp->stp = stp; + } + } + + /* Convert all backlinks into forward links. Only the forward + ** links are used in the follow-set computation. */ + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + for(cfp=stp?stp->cfp:0; cfp; cfp=cfp->next){ + for(plp=cfp->bplp; plp; plp=plp->next){ + other = plp->cfp; + Plink_add(&other->fplp,cfp); + } + } + } +} + +/* Compute all followsets. +** +** A followset is the set of all symbols which can come immediately +** after a configuration. +*/ +void FindFollowSets(struct lemon *lemp) +{ + int i; + struct config *cfp; + struct plink *plp; + int progress; + int change; + + for(i=0; instate; i++){ + assert( lemp->sorted[i]!=0 ); + for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){ + cfp->status = INCOMPLETE; + } + } + + do{ + progress = 0; + for(i=0; instate; i++){ + assert( lemp->sorted[i]!=0 ); + for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){ + if( cfp->status==COMPLETE ) continue; + for(plp=cfp->fplp; plp; plp=plp->next){ + change = SetUnion(plp->cfp->fws,cfp->fws); + if( change ){ + plp->cfp->status = INCOMPLETE; + progress = 1; + } + } + cfp->status = COMPLETE; + } + } + }while( progress ); +} + +static int resolve_conflict(struct action *,struct action *); + +/* Compute the reduce actions, and resolve conflicts. +*/ +void FindActions(struct lemon *lemp) +{ + int i,j; + struct config *cfp; + struct state *stp; + struct symbol *sp; + struct rule *rp; + + /* Add all of the reduce actions + ** A reduce action is added for each element of the followset of + ** a configuration which has its dot at the extreme right. + */ + for(i=0; instate; i++){ /* Loop over all states */ + stp = lemp->sorted[i]; + for(cfp=stp->cfp; cfp; cfp=cfp->next){ /* Loop over all configurations */ + if( cfp->rp->nrhs==cfp->dot ){ /* Is dot at extreme right? */ + for(j=0; jnterminal; j++){ + if( SetFind(cfp->fws,j) ){ + /* Add a reduce action to the state "stp" which will reduce by the + ** rule "cfp->rp" if the lookahead symbol is "lemp->symbols[j]" */ + Action_add(&stp->ap,REDUCE,lemp->symbols[j],(char *)cfp->rp); + } + } + } + } + } + + /* Add the accepting token */ + if( lemp->start ){ + sp = Symbol_find(lemp->start); + if( sp==0 ){ + if( lemp->startRule==0 ){ + fprintf(stderr, "internal error on source line %d: no start rule\n", + __LINE__); + exit(1); + } + sp = lemp->startRule->lhs; + } + }else{ + sp = lemp->startRule->lhs; + } + /* Add to the first state (which is always the starting state of the + ** finite state machine) an action to ACCEPT if the lookahead is the + ** start nonterminal. */ + Action_add(&lemp->sorted[0]->ap,ACCEPT,sp,0); + + /* Resolve conflicts */ + for(i=0; instate; i++){ + struct action *ap, *nap; + stp = lemp->sorted[i]; + /* assert( stp->ap ); */ + stp->ap = Action_sort(stp->ap); + for(ap=stp->ap; ap && ap->next; ap=ap->next){ + for(nap=ap->next; nap && nap->sp==ap->sp; nap=nap->next){ + /* The two actions "ap" and "nap" have the same lookahead. + ** Figure out which one should be used */ + lemp->nconflict += resolve_conflict(ap,nap); + } + } + } + + /* Report an error for each rule that can never be reduced. */ + for(rp=lemp->rule; rp; rp=rp->next) rp->canReduce = LEMON_FALSE; + for(i=0; instate; i++){ + struct action *ap; + for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){ + if( ap->type==REDUCE ) ap->x.rp->canReduce = LEMON_TRUE; + } + } + for(rp=lemp->rule; rp; rp=rp->next){ + if( rp->canReduce ) continue; + ErrorMsg(lemp->filename,rp->ruleline,"This rule can not be reduced.\n"); + lemp->errorcnt++; + } +} + +/* Resolve a conflict between the two given actions. If the +** conflict can't be resolved, return non-zero. +** +** NO LONGER TRUE: +** To resolve a conflict, first look to see if either action +** is on an error rule. In that case, take the action which +** is not associated with the error rule. If neither or both +** actions are associated with an error rule, then try to +** use precedence to resolve the conflict. +** +** If either action is a SHIFT, then it must be apx. This +** function won't work if apx->type==REDUCE and apy->type==SHIFT. +*/ +static int resolve_conflict( + struct action *apx, + struct action *apy +){ + struct symbol *spx, *spy; + int errcnt = 0; + assert( apx->sp==apy->sp ); /* Otherwise there would be no conflict */ + if( apx->type==SHIFT && apy->type==SHIFT ){ + apy->type = SSCONFLICT; + errcnt++; + } + if( apx->type==SHIFT && apy->type==REDUCE ){ + spx = apx->sp; + spy = apy->x.rp->precsym; + if( spy==0 || spx->prec<0 || spy->prec<0 ){ + /* Not enough precedence information. */ + apy->type = SRCONFLICT; + errcnt++; + }else if( spx->prec>spy->prec ){ /* higher precedence wins */ + apy->type = RD_RESOLVED; + }else if( spx->precprec ){ + apx->type = SH_RESOLVED; + }else if( spx->prec==spy->prec && spx->assoc==RIGHT ){ /* Use operator */ + apy->type = RD_RESOLVED; /* associativity */ + }else if( spx->prec==spy->prec && spx->assoc==LEFT ){ /* to break tie */ + apx->type = SH_RESOLVED; + }else{ + assert( spx->prec==spy->prec && spx->assoc==NONE ); + apx->type = ERROR; + } + }else if( apx->type==REDUCE && apy->type==REDUCE ){ + spx = apx->x.rp->precsym; + spy = apy->x.rp->precsym; + if( spx==0 || spy==0 || spx->prec<0 || + spy->prec<0 || spx->prec==spy->prec ){ + apy->type = RRCONFLICT; + errcnt++; + }else if( spx->prec>spy->prec ){ + apy->type = RD_RESOLVED; + }else if( spx->precprec ){ + apx->type = RD_RESOLVED; + } + }else{ + assert( + apx->type==SH_RESOLVED || + apx->type==RD_RESOLVED || + apx->type==SSCONFLICT || + apx->type==SRCONFLICT || + apx->type==RRCONFLICT || + apy->type==SH_RESOLVED || + apy->type==RD_RESOLVED || + apy->type==SSCONFLICT || + apy->type==SRCONFLICT || + apy->type==RRCONFLICT + ); + /* The REDUCE/SHIFT case cannot happen because SHIFTs come before + ** REDUCEs on the list. If we reach this point it must be because + ** the parser conflict had already been resolved. */ + } + return errcnt; +} +/********************* From the file "configlist.c" *************************/ +/* +** Routines to processing a configuration list and building a state +** in the LEMON parser generator. +*/ + +static struct config *freelist = 0; /* List of free configurations */ +static struct config *current = 0; /* Top of list of configurations */ +static struct config **currentend = 0; /* Last on list of configs */ +static struct config *basis = 0; /* Top of list of basis configs */ +static struct config **basisend = 0; /* End of list of basis configs */ + +/* Return a pointer to a new configuration */ +PRIVATE struct config *newconfig(void){ + return (struct config*)lemon_calloc(1, sizeof(struct config)); +} + +/* The configuration "old" is no longer used */ +PRIVATE void deleteconfig(struct config *old) +{ + old->next = freelist; + freelist = old; +} + +/* Initialized the configuration list builder */ +void Configlist_init(void){ + current = 0; + currentend = ¤t; + basis = 0; + basisend = &basis; + Configtable_init(); + return; +} + +/* Initialized the configuration list builder */ +void Configlist_reset(void){ + current = 0; + currentend = ¤t; + basis = 0; + basisend = &basis; + Configtable_clear(0); + return; +} + +/* Add another configuration to the configuration list */ +struct config *Configlist_add( + struct rule *rp, /* The rule */ + int dot /* Index into the RHS of the rule where the dot goes */ +){ + struct config *cfp, model; + + assert( currentend!=0 ); + model.rp = rp; + model.dot = dot; + cfp = Configtable_find(&model); + if( cfp==0 ){ + cfp = newconfig(); + cfp->rp = rp; + cfp->dot = dot; + cfp->fws = SetNew(); + cfp->stp = 0; + cfp->fplp = cfp->bplp = 0; + cfp->next = 0; + cfp->bp = 0; + *currentend = cfp; + currentend = &cfp->next; + Configtable_insert(cfp); + } + return cfp; +} + +/* Add a basis configuration to the configuration list */ +struct config *Configlist_addbasis(struct rule *rp, int dot) +{ + struct config *cfp, model; + + assert( basisend!=0 ); + assert( currentend!=0 ); + model.rp = rp; + model.dot = dot; + cfp = Configtable_find(&model); + if( cfp==0 ){ + cfp = newconfig(); + cfp->rp = rp; + cfp->dot = dot; + cfp->fws = SetNew(); + cfp->stp = 0; + cfp->fplp = cfp->bplp = 0; + cfp->next = 0; + cfp->bp = 0; + *currentend = cfp; + currentend = &cfp->next; + *basisend = cfp; + basisend = &cfp->bp; + Configtable_insert(cfp); + } + return cfp; +} + +/* Compute the closure of the configuration list */ +void Configlist_closure(struct lemon *lemp) +{ + struct config *cfp, *newcfp; + struct rule *rp, *newrp; + struct symbol *sp, *xsp; + int i, dot; + + assert( currentend!=0 ); + for(cfp=current; cfp; cfp=cfp->next){ + rp = cfp->rp; + dot = cfp->dot; + if( dot>=rp->nrhs ) continue; + sp = rp->rhs[dot]; + if( sp->type==NONTERMINAL ){ + if( sp->rule==0 && sp!=lemp->errsym ){ + ErrorMsg(lemp->filename,rp->line,"Nonterminal \"%s\" has no rules.", + sp->name); + lemp->errorcnt++; + } + for(newrp=sp->rule; newrp; newrp=newrp->nextlhs){ + newcfp = Configlist_add(newrp,0); + for(i=dot+1; inrhs; i++){ + xsp = rp->rhs[i]; + if( xsp->type==TERMINAL ){ + SetAdd(newcfp->fws,xsp->index); + break; + }else if( xsp->type==MULTITERMINAL ){ + int k; + for(k=0; knsubsym; k++){ + SetAdd(newcfp->fws, xsp->subsym[k]->index); + } + break; + }else{ + SetUnion(newcfp->fws,xsp->firstset); + if( xsp->lambda==LEMON_FALSE ) break; + } + } + if( i==rp->nrhs ) Plink_add(&cfp->fplp,newcfp); + } + } + } + return; +} + +/* Sort the configuration list */ +void Configlist_sort(void){ + current = (struct config*)msort((char*)current,(char**)&(current->next), + Configcmp); + currentend = 0; + return; +} + +/* Sort the basis configuration list */ +void Configlist_sortbasis(void){ + basis = (struct config*)msort((char*)current,(char**)&(current->bp), + Configcmp); + basisend = 0; + return; +} + +/* Return a pointer to the head of the configuration list and +** reset the list */ +struct config *Configlist_return(void){ + struct config *old; + old = current; + current = 0; + currentend = 0; + return old; +} + +/* Return a pointer to the head of the configuration list and +** reset the list */ +struct config *Configlist_basis(void){ + struct config *old; + old = basis; + basis = 0; + basisend = 0; + return old; +} + +/* Free all elements of the given configuration list */ +void Configlist_eat(struct config *cfp) +{ + struct config *nextcfp; + for(; cfp; cfp=nextcfp){ + nextcfp = cfp->next; + assert( cfp->fplp==0 ); + assert( cfp->bplp==0 ); + if( cfp->fws ) SetFree(cfp->fws); + deleteconfig(cfp); + } + return; +} +/***************** From the file "error.c" *********************************/ +/* +** Code for printing error message. +*/ + +void ErrorMsg(const char *filename, int lineno, const char *format, ...){ + va_list ap; + fprintf(stderr, "%s:%d: ", filename, lineno); + va_start(ap, format); + vfprintf(stderr,format,ap); + va_end(ap); + fprintf(stderr, "\n"); +} +/**************** From the file "main.c" ************************************/ +/* +** Main program file for the LEMON parser generator. +*/ + +/* Report an out-of-memory condition and abort. This function +** is used mostly by the "MemoryCheck" macro in struct.h +*/ +void memory_error(void){ + fprintf(stderr,"Out of memory. Aborting...\n"); + exit(1); +} + +static int nDefine = 0; /* Number of -D options on the command line */ +static int nDefineUsed = 0; /* Number of -D options actually used */ +static char **azDefine = 0; /* Name of the -D macros */ +static char *bDefineUsed = 0; /* True for every -D macro actually used */ + +/* This routine is called with the argument to each -D command-line option. +** Add the macro defined to the azDefine array. +*/ +static void handle_D_option(char *z){ + char **paz; + nDefine++; + azDefine = (char **) lemon_realloc(azDefine, sizeof(azDefine[0])*nDefine); + if( azDefine==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + bDefineUsed = (char*)lemon_realloc(bDefineUsed, nDefine); + if( bDefineUsed==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + bDefineUsed[nDefine-1] = 0; + paz = &azDefine[nDefine-1]; + *paz = (char *) lemon_malloc( lemonStrlen(z)+1 ); + if( *paz==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + lemon_strcpy(*paz, z); + for(z=*paz; *z && *z!='='; z++){} + *z = 0; +} + +/* Rember the name of the output directory +*/ +static char *outputDir = NULL; +static void handle_d_option(char *z){ + outputDir = (char *) lemon_malloc( lemonStrlen(z)+1 ); + if( outputDir==0 ){ + fprintf(stderr,"out of memory\n"); + exit(1); + } + lemon_strcpy(outputDir, z); +} + +static char *user_templatename = NULL; +static void handle_T_option(char *z){ + user_templatename = (char *) lemon_malloc( lemonStrlen(z)+1 ); + if( user_templatename==0 ){ + memory_error(); + } + lemon_strcpy(user_templatename, z); +} + +/* Merge together to lists of rules ordered by rule.iRule */ +static struct rule *Rule_merge(struct rule *pA, struct rule *pB){ + struct rule *pFirst = 0; + struct rule **ppPrev = &pFirst; + while( pA && pB ){ + if( pA->iRuleiRule ){ + *ppPrev = pA; + ppPrev = &pA->next; + pA = pA->next; + }else{ + *ppPrev = pB; + ppPrev = &pB->next; + pB = pB->next; + } + } + if( pA ){ + *ppPrev = pA; + }else{ + *ppPrev = pB; + } + return pFirst; +} + +/* +** Sort a list of rules in order of increasing iRule value +*/ +static struct rule *Rule_sort(struct rule *rp){ + unsigned int i; + struct rule *pNext; + struct rule *x[32]; + memset(x, 0, sizeof(x)); + while( rp ){ + pNext = rp->next; + rp->next = 0; + for(i=0; iindex = i; + qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp); + for(i=0; iindex = i; + while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } + assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); + lem.nsymbol = i - 1; + for(i=1; ISUPPER(lem.symbols[i]->name[0]); i++); + lem.nterminal = i; + + /* Assign sequential rule numbers. Start with 0. Put rules that have no + ** reduce action C-code associated with them last, so that the switch() + ** statement that selects reduction actions will have a smaller jump table. + */ + for(i=0, rp=lem.rule; rp; rp=rp->next){ + rp->iRule = rp->code ? i++ : -1; + } + lem.nruleWithAction = i; + for(rp=lem.rule; rp; rp=rp->next){ + if( rp->iRule<0 ) rp->iRule = i++; + } + lem.startRule = lem.rule; + lem.rule = Rule_sort(lem.rule); + + /* Generate a reprint of the grammar, if requested on the command line */ + if( rpflag ){ + Reprint(&lem); + }else{ + /* Initialize the size for all follow and first sets */ + SetSize(lem.nterminal+1); + + /* Find the precedence for every production rule (that has one) */ + FindRulePrecedences(&lem); + + /* Compute the lambda-nonterminals and the first-sets for every + ** nonterminal */ + FindFirstSets(&lem); + + /* Compute all LR(0) states. Also record follow-set propagation + ** links so that the follow-set can be computed later */ + lem.nstate = 0; + FindStates(&lem); + lem.sorted = State_arrayof(); + + /* Tie up loose ends on the propagation links */ + FindLinks(&lem); + + /* Compute the follow set of every reducible configuration */ + FindFollowSets(&lem); + + /* Compute the action tables */ + FindActions(&lem); + + /* Compress the action tables */ + if( compress==0 ) CompressTables(&lem); + + /* Reorder and renumber the states so that states with fewer choices + ** occur at the end. This is an optimization that helps make the + ** generated parser tables smaller. */ + if( noResort==0 ) ResortStates(&lem); + + /* Generate a report of the parser generated. (the "y.output" file) */ + if( !quiet ) ReportOutput(&lem); + + /* Generate the source code for the parser */ + ReportTable(&lem, mhflag, sqlFlag); + + /* Produce a header file for use by the scanner. (This step is + ** omitted if the "-m" option is used because makeheaders will + ** generate the file for us.) */ + if( !mhflag ) ReportHeader(&lem); + } + if( statistics ){ + printf("Parser statistics:\n"); + stats_line("terminal symbols", lem.nterminal); + stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal); + stats_line("total symbols", lem.nsymbol); + stats_line("rules", lem.nrule); + stats_line("states", lem.nxstate); + stats_line("conflicts", lem.nconflict); + stats_line("action table entries", lem.nactiontab); + stats_line("lookahead table entries", lem.nlookaheadtab); + stats_line("total table size (bytes)", lem.tablesize); + } + if( lem.nconflict > 0 ){ + fprintf(stderr,"%d parsing conflicts.\n",lem.nconflict); + } + + /* return 0 on success, 1 on failure. */ + exitcode = ((lem.errorcnt > 0) || (lem.nconflict > 0)) ? 1 : 0; + lemon_free_all(); + exit(exitcode); + return (exitcode); +} +/******************** From the file "msort.c" *******************************/ +/* +** A generic merge-sort program. +** +** USAGE: +** Let "ptr" be a pointer to some structure which is at the head of +** a null-terminated list. Then to sort the list call: +** +** ptr = msort(ptr,&(ptr->next),cmpfnc); +** +** In the above, "cmpfnc" is a pointer to a function which compares +** two instances of the structure and returns an integer, as in +** strcmp. The second argument is a pointer to the pointer to the +** second element of the linked list. This address is used to compute +** the offset to the "next" field within the structure. The offset to +** the "next" field must be constant for all structures in the list. +** +** The function returns a new pointer which is the head of the list +** after sorting. +** +** ALGORITHM: +** Merge-sort. +*/ + +/* +** Return a pointer to the next structure in the linked list. +*/ +#define NEXT(A) (*(char**)(((char*)A)+offset)) + +/* +** Inputs: +** a: A sorted, null-terminated linked list. (May be null). +** b: A sorted, null-terminated linked list. (May be null). +** cmp: A pointer to the comparison function. +** offset: Offset in the structure to the "next" field. +** +** Return Value: +** A pointer to the head of a sorted list containing the elements +** of both a and b. +** +** Side effects: +** The "next" pointers for elements in the lists a and b are +** changed. +*/ +static char *merge( + char *a, + char *b, + int (*cmp)(const char*,const char*), + int offset +){ + char *ptr, *head; + + if( a==0 ){ + head = b; + }else if( b==0 ){ + head = a; + }else{ + if( (*cmp)(a,b)<=0 ){ + ptr = a; + a = NEXT(a); + }else{ + ptr = b; + b = NEXT(b); + } + head = ptr; + while( a && b ){ + if( (*cmp)(a,b)<=0 ){ + NEXT(ptr) = a; + ptr = a; + a = NEXT(a); + }else{ + NEXT(ptr) = b; + ptr = b; + b = NEXT(b); + } + } + if( a ) NEXT(ptr) = a; + else NEXT(ptr) = b; + } + return head; +} + +/* +** Inputs: +** list: Pointer to a singly-linked list of structures. +** next: Pointer to pointer to the second element of the list. +** cmp: A comparison function. +** +** Return Value: +** A pointer to the head of a sorted list containing the elements +** originally in list. +** +** Side effects: +** The "next" pointers for elements in list are changed. +*/ +#define LISTSIZE 30 +static char *msort( + char *list, + char **next, + int (*cmp)(const char*,const char*) +){ + unsigned long offset; + char *ep; + char *set[LISTSIZE]; + int i; + offset = (unsigned long)((char*)next - (char*)list); + for(i=0; istate = WAITING_FOR_DECL_KEYWORD; + }else if( ISLOWER(x[0]) ){ + psp->lhs = Symbol_new(x); + psp->nrhs = 0; + psp->lhsalias = 0; + psp->state = WAITING_FOR_ARROW; + }else if( x[0]=='{' ){ + if( psp->prevrule==0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "There is no prior rule upon which to attach the code " + "fragment which begins on this line."); + psp->errorcnt++; + }else if( psp->prevrule->code!=0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Code fragment beginning on this line is not the first " + "to follow the previous rule."); + psp->errorcnt++; + }else if( strcmp(x, "{NEVER-REDUCE")==0 ){ + psp->prevrule->neverReduce = 1; + }else{ + psp->prevrule->line = psp->tokenlineno; + psp->prevrule->code = &x[1]; + psp->prevrule->noCode = 0; + } + }else if( x[0]=='[' ){ + psp->state = PRECEDENCE_MARK_1; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Token \"%s\" should be either \"%%\" or a nonterminal name.", + x); + psp->errorcnt++; + } + break; + case PRECEDENCE_MARK_1: + if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "The precedence symbol must be a terminal."); + psp->errorcnt++; + }else if( psp->prevrule==0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "There is no prior rule to assign precedence \"[%s]\".",x); + psp->errorcnt++; + }else if( psp->prevrule->precsym!=0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Precedence mark on this line is not the first " + "to follow the previous rule."); + psp->errorcnt++; + }else{ + psp->prevrule->precsym = Symbol_new(x); + } + psp->state = PRECEDENCE_MARK_2; + break; + case PRECEDENCE_MARK_2: + if( x[0]!=']' ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \"]\" on precedence mark."); + psp->errorcnt++; + } + psp->state = WAITING_FOR_DECL_OR_RULE; + break; + case WAITING_FOR_ARROW: + if( x[0]==':' && x[1]==':' && x[2]=='=' ){ + psp->state = IN_RHS; + }else if( x[0]=='(' ){ + psp->state = LHS_ALIAS_1; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Expected to see a \":\" following the LHS symbol \"%s\".", + psp->lhs->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_1: + if( ISALPHA(x[0]) ){ + psp->lhsalias = x; + psp->state = LHS_ALIAS_2; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "\"%s\" is not a valid alias for the LHS \"%s\"\n", + x,psp->lhs->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_2: + if( x[0]==')' ){ + psp->state = LHS_ALIAS_3; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \")\" following LHS alias name \"%s\".",psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case LHS_ALIAS_3: + if( x[0]==':' && x[1]==':' && x[2]=='=' ){ + psp->state = IN_RHS; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \"->\" following: \"%s(%s)\".", + psp->lhs->name,psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case IN_RHS: + if( x[0]=='.' ){ + struct rule *rp; + rp = (struct rule *)lemon_calloc( sizeof(struct rule) + + sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1); + if( rp==0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Can't allocate enough memory for this rule."); + psp->errorcnt++; + psp->prevrule = 0; + }else{ + int i; + rp->ruleline = psp->tokenlineno; + rp->rhs = (struct symbol**)&rp[1]; + rp->rhsalias = (const char**)&(rp->rhs[psp->nrhs]); + for(i=0; inrhs; i++){ + rp->rhs[i] = psp->rhs[i]; + rp->rhsalias[i] = psp->alias[i]; + if( rp->rhsalias[i]!=0 ){ rp->rhs[i]->bContent = 1; } + } + rp->lhs = psp->lhs; + rp->lhsalias = psp->lhsalias; + rp->nrhs = psp->nrhs; + rp->code = 0; + rp->noCode = 1; + rp->precsym = 0; + rp->index = psp->gp->nrule++; + rp->nextlhs = rp->lhs->rule; + rp->lhs->rule = rp; + rp->next = 0; + if( psp->firstrule==0 ){ + psp->firstrule = psp->lastrule = rp; + }else{ + psp->lastrule->next = rp; + psp->lastrule = rp; + } + psp->prevrule = rp; + } + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( ISALPHA(x[0]) ){ + if( psp->nrhs>=MAXRHS ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Too many symbols on RHS of rule beginning at \"%s\".", + x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + }else{ + psp->rhs[psp->nrhs] = Symbol_new(x); + psp->alias[psp->nrhs] = 0; + psp->nrhs++; + } + }else if( (x[0]=='|' || x[0]=='/') && psp->nrhs>0 && ISUPPER(x[1]) ){ + struct symbol *msp = psp->rhs[psp->nrhs-1]; + if( msp->type!=MULTITERMINAL ){ + struct symbol *origsp = msp; + msp = (struct symbol *) lemon_calloc(1,sizeof(*msp)); + memset(msp, 0, sizeof(*msp)); + msp->type = MULTITERMINAL; + msp->nsubsym = 1; + msp->subsym = (struct symbol**)lemon_calloc(1,sizeof(struct symbol*)); + msp->subsym[0] = origsp; + msp->name = origsp->name; + psp->rhs[psp->nrhs-1] = msp; + } + msp->nsubsym++; + msp->subsym = (struct symbol **) lemon_realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]); + if( ISLOWER(x[1]) || ISLOWER(msp->subsym[0]->name[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Cannot form a compound containing a non-terminal"); + psp->errorcnt++; + } + }else if( x[0]=='(' && psp->nrhs>0 ){ + psp->state = RHS_ALIAS_1; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal character on RHS of rule: \"%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case RHS_ALIAS_1: + if( ISALPHA(x[0]) ){ + psp->alias[psp->nrhs-1] = x; + psp->state = RHS_ALIAS_2; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "\"%s\" is not a valid alias for the RHS symbol \"%s\"\n", + x,psp->rhs[psp->nrhs-1]->name); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case RHS_ALIAS_2: + if( x[0]==')' ){ + psp->state = IN_RHS; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Missing \")\" following LHS alias name \"%s\".",psp->lhsalias); + psp->errorcnt++; + psp->state = RESYNC_AFTER_RULE_ERROR; + } + break; + case WAITING_FOR_DECL_KEYWORD: + if( ISALPHA(x[0]) ){ + psp->declkeyword = x; + psp->declargslot = 0; + psp->decllinenoslot = 0; + psp->insertLineMacro = 1; + psp->state = WAITING_FOR_DECL_ARG; + if( strcmp(x,"name")==0 ){ + psp->declargslot = &(psp->gp->name); + psp->insertLineMacro = 0; + }else if( strcmp(x,"include")==0 ){ + psp->declargslot = &(psp->gp->include); + }else if( strcmp(x,"code")==0 ){ + psp->declargslot = &(psp->gp->extracode); + }else if( strcmp(x,"token_destructor")==0 ){ + psp->declargslot = &psp->gp->tokendest; + }else if( strcmp(x,"default_destructor")==0 ){ + psp->declargslot = &psp->gp->vardest; + }else if( strcmp(x,"token_prefix")==0 ){ + psp->declargslot = &psp->gp->tokenprefix; + psp->insertLineMacro = 0; + }else if( strcmp(x,"syntax_error")==0 ){ + psp->declargslot = &(psp->gp->error); + }else if( strcmp(x,"parse_accept")==0 ){ + psp->declargslot = &(psp->gp->accept); + }else if( strcmp(x,"parse_failure")==0 ){ + psp->declargslot = &(psp->gp->failure); + }else if( strcmp(x,"stack_overflow")==0 ){ + psp->declargslot = &(psp->gp->overflow); + }else if( strcmp(x,"extra_argument")==0 ){ + psp->declargslot = &(psp->gp->arg); + psp->insertLineMacro = 0; + }else if( strcmp(x,"extra_context")==0 ){ + psp->declargslot = &(psp->gp->ctx); + psp->insertLineMacro = 0; + }else if( strcmp(x,"token_type")==0 ){ + psp->declargslot = &(psp->gp->tokentype); + psp->insertLineMacro = 0; + }else if( strcmp(x,"default_type")==0 ){ + psp->declargslot = &(psp->gp->vartype); + psp->insertLineMacro = 0; + }else if( strcmp(x,"realloc")==0 ){ + psp->declargslot = &(psp->gp->reallocFunc); + psp->insertLineMacro = 0; + }else if( strcmp(x,"free")==0 ){ + psp->declargslot = &(psp->gp->freeFunc); + psp->insertLineMacro = 0; + }else if( strcmp(x,"stack_size")==0 ){ + psp->declargslot = &(psp->gp->stacksize); + psp->insertLineMacro = 0; + }else if( strcmp(x,"start_symbol")==0 ){ + psp->declargslot = &(psp->gp->start); + psp->insertLineMacro = 0; + }else if( strcmp(x,"left")==0 ){ + psp->preccounter++; + psp->declassoc = LEFT; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + }else if( strcmp(x,"right")==0 ){ + psp->preccounter++; + psp->declassoc = RIGHT; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + }else if( strcmp(x,"nonassoc")==0 ){ + psp->preccounter++; + psp->declassoc = NONE; + psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; + }else if( strcmp(x,"destructor")==0 ){ + psp->state = WAITING_FOR_DESTRUCTOR_SYMBOL; + }else if( strcmp(x,"type")==0 ){ + psp->state = WAITING_FOR_DATATYPE_SYMBOL; + }else if( strcmp(x,"fallback")==0 ){ + psp->fallback = 0; + psp->state = WAITING_FOR_FALLBACK_ID; + }else if( strcmp(x,"token")==0 ){ + psp->state = WAITING_FOR_TOKEN_NAME; + }else if( strcmp(x,"wildcard")==0 ){ + psp->state = WAITING_FOR_WILDCARD_ID; + }else if( strcmp(x,"token_class")==0 ){ + psp->state = WAITING_FOR_CLASS_ID; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Unknown declaration keyword: \"%%%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal declaration keyword: \"%s\".",x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case WAITING_FOR_DESTRUCTOR_SYMBOL: + if( !ISALPHA(x[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol name missing after %%destructor keyword"); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + struct symbol *sp = Symbol_new(x); + psp->declargslot = &sp->destructor; + psp->decllinenoslot = &sp->destLineno; + psp->insertLineMacro = 1; + psp->state = WAITING_FOR_DECL_ARG; + } + break; + case WAITING_FOR_DATATYPE_SYMBOL: + if( !ISALPHA(x[0]) ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol name missing after %%type keyword"); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + struct symbol *sp = Symbol_find(x); + if((sp) && (sp->datatype)){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol %%type \"%s\" already defined", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + if (!sp){ + sp = Symbol_new(x); + } + psp->declargslot = &sp->datatype; + psp->insertLineMacro = 0; + psp->state = WAITING_FOR_DECL_ARG; + } + } + break; + case WAITING_FOR_PRECEDENCE_SYMBOL: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( ISUPPER(x[0]) ){ + struct symbol *sp; + sp = Symbol_new(x); + if( sp->prec>=0 ){ + ErrorMsg(psp->filename,psp->tokenlineno, + "Symbol \"%s\" has already be given a precedence.",x); + psp->errorcnt++; + }else{ + sp->prec = psp->preccounter; + sp->assoc = psp->declassoc; + } + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Can't assign a precedence to \"%s\".",x); + psp->errorcnt++; + } + break; + case WAITING_FOR_DECL_ARG: + if( x[0]=='{' || x[0]=='\"' || ISALNUM(x[0]) ){ + const char *zOld, *zNew; + char *zBuf, *z; + int nOld, n, nLine = 0, nNew, nBack; + int addLineMacro; + char zLine[50]; + zNew = x; + if( zNew[0]=='"' || zNew[0]=='{' ) zNew++; + nNew = lemonStrlen(zNew); + if( *psp->declargslot ){ + zOld = *psp->declargslot; + }else{ + zOld = ""; + } + nOld = lemonStrlen(zOld); + n = nOld + nNew + 20; + addLineMacro = !psp->gp->nolinenosflag + && psp->insertLineMacro + && psp->tokenlineno>1 + && (psp->decllinenoslot==0 || psp->decllinenoslot[0]!=0); + if( addLineMacro ){ + for(z=psp->filename, nBack=0; *z; z++){ + if( *z=='\\' ) nBack++; + } + lemon_sprintf(zLine, "#line %d ", psp->tokenlineno); + nLine = lemonStrlen(zLine); + n += nLine + lemonStrlen(psp->filename) + nBack; + } + *psp->declargslot = (char *) lemon_realloc(*psp->declargslot, n); + zBuf = *psp->declargslot + nOld; + if( addLineMacro ){ + if( nOld && zBuf[-1]!='\n' ){ + *(zBuf++) = '\n'; + } + memcpy(zBuf, zLine, nLine); + zBuf += nLine; + *(zBuf++) = '"'; + for(z=psp->filename; *z; z++){ + if( *z=='\\' ){ + *(zBuf++) = '\\'; + } + *(zBuf++) = *z; + } + *(zBuf++) = '"'; + *(zBuf++) = '\n'; + } + if( psp->decllinenoslot && psp->decllinenoslot[0]==0 ){ + psp->decllinenoslot[0] = psp->tokenlineno; + } + memcpy(zBuf, zNew, nNew); + zBuf += nNew; + *zBuf = 0; + psp->state = WAITING_FOR_DECL_OR_RULE; + }else{ + ErrorMsg(psp->filename,psp->tokenlineno, + "Illegal argument to %%%s: %s",psp->declkeyword,x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case WAITING_FOR_FALLBACK_ID: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%fallback argument \"%s\" should be a token", x); + psp->errorcnt++; + }else{ + struct symbol *sp = Symbol_new(x); + if( psp->fallback==0 ){ + psp->fallback = sp; + }else if( sp->fallback ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "More than one fallback assigned to token %s", x); + psp->errorcnt++; + }else{ + sp->fallback = psp->fallback; + psp->gp->has_fallback = 1; + } + } + break; + case WAITING_FOR_TOKEN_NAME: + /* Tokens do not have to be declared before use. But they can be + ** in order to control their assigned integer number. The number for + ** each token is assigned when it is first seen. So by including + ** + ** %token ONE TWO THREE. + ** + ** early in the grammar file, that assigns small consecutive values + ** to each of the tokens ONE TWO and THREE. + */ + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token argument \"%s\" should be a token", x); + psp->errorcnt++; + }else{ + (void)Symbol_new(x); + } + break; + case WAITING_FOR_WILDCARD_ID: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%wildcard argument \"%s\" should be a token", x); + psp->errorcnt++; + }else{ + struct symbol *sp = Symbol_new(x); + if( psp->gp->wildcard==0 ){ + psp->gp->wildcard = sp; + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "Extra wildcard to token: %s", x); + psp->errorcnt++; + } + } + break; + case WAITING_FOR_CLASS_ID: + if( !ISLOWER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class must be followed by an identifier: %s", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else if( Symbol_find(x) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "Symbol \"%s\" already used", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + psp->tkclass = Symbol_new(x); + psp->tkclass->type = MULTITERMINAL; + psp->state = WAITING_FOR_CLASS_TOKEN; + } + break; + case WAITING_FOR_CLASS_TOKEN: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( ISUPPER(x[0]) || ((x[0]=='|' || x[0]=='/') && ISUPPER(x[1])) ){ + struct symbol *msp = psp->tkclass; + msp->nsubsym++; + msp->subsym = (struct symbol **) lemon_realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + if( !ISUPPER(x[0]) ) x++; + msp->subsym[msp->nsubsym-1] = Symbol_new(x); + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class argument \"%s\" should be a token", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; + case RESYNC_AFTER_RULE_ERROR: +/* if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; +** break; */ + case RESYNC_AFTER_DECL_ERROR: + if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; + if( x[0]=='%' ) psp->state = WAITING_FOR_DECL_KEYWORD; + break; + } +} + +/* The text in the input is part of the argument to an %ifdef or %ifndef. +** Evaluate the text as a boolean expression. Return true or false. +*/ +static int eval_preprocessor_boolean(char *z, int lineno){ + int neg = 0; + int res = 0; + int okTerm = 1; + int i; + for(i=0; z[i]!=0; i++){ + if( ISSPACE(z[i]) ) continue; + if( z[i]=='!' ){ + if( !okTerm ) goto pp_syntax_error; + neg = !neg; + continue; + } + if( z[i]=='|' && z[i+1]=='|' ){ + if( okTerm ) goto pp_syntax_error; + if( res ) return 1; + i++; + okTerm = 1; + continue; + } + if( z[i]=='&' && z[i+1]=='&' ){ + if( okTerm ) goto pp_syntax_error; + if( !res ) return 0; + i++; + okTerm = 1; + continue; + } + if( z[i]=='(' ){ + int k; + int n = 1; + if( !okTerm ) goto pp_syntax_error; + for(k=i+1; z[k]; k++){ + if( z[k]==')' ){ + n--; + if( n==0 ){ + z[k] = 0; + res = eval_preprocessor_boolean(&z[i+1], -1); + z[k] = ')'; + if( res<0 ){ + i = i-res; + goto pp_syntax_error; + } + i = k; + break; + } + }else if( z[k]=='(' ){ + n++; + }else if( z[k]==0 ){ + i = k; + goto pp_syntax_error; + } + } + if( neg ){ + res = !res; + neg = 0; + } + okTerm = 0; + continue; + } + if( ISALPHA(z[i]) ){ + int j, k, n; + if( !okTerm ) goto pp_syntax_error; + for(k=i+1; ISALNUM(z[k]) || z[k]=='_'; k++){} + n = k - i; + res = 0; + for(j=0; j0 ){ + fprintf(stderr, "%%if syntax error on line %d.\n", lineno); + fprintf(stderr, " %.*s <-- syntax error here\n", i+1, z); + exit(1); + }else{ + return -(i+1); + } +} + +/* Run the preprocessor over the input file text. The global variables +** azDefine[0] through azDefine[nDefine-1] contains the names of all defined +** macros. This routine looks for "%ifdef" and "%ifndef" and "%endif" and +** comments them out. Text in between is also commented out as appropriate. +*/ +static void preprocess_input(char *z){ + int i, j, k; + int exclude = 0; + int start = 0; + int lineno = 1; + int start_lineno = 1; + for(i=0; z[i]; i++){ + if( z[i]=='\n' ) lineno++; + if( z[i]!='%' || (i>0 && z[i-1]!='\n') ) continue; + if( strncmp(&z[i],"%endif",6)==0 && ISSPACE(z[i+6]) ){ + if( exclude ){ + exclude--; + if( exclude==0 ){ + for(j=start; jfilename; + ps.errorcnt = 0; + ps.state = INITIALIZE; + + /* Begin by reading the input file */ + fp = fopen(ps.filename,"rb"); + if( fp==0 ){ + ErrorMsg(ps.filename,0,"Can't open this file for reading."); + gp->errorcnt++; + return; + } + fseek(fp,0,2); + filesize = ftell(fp); + rewind(fp); + filebuf = (char *)lemon_malloc( filesize+1 ); + if( filesize>100000000 || filebuf==0 ){ + ErrorMsg(ps.filename,0,"Input file too large."); + lemon_free(filebuf); + gp->errorcnt++; + fclose(fp); + return; + } + if( fread(filebuf,1,filesize,fp)!=filesize ){ + ErrorMsg(ps.filename,0,"Can't read in all %d bytes of this file.", + filesize); + lemon_free(filebuf); + gp->errorcnt++; + fclose(fp); + return; + } + fclose(fp); + filebuf[filesize] = 0; + + /* Make an initial pass through the file to handle %ifdef and %ifndef */ + preprocess_input(filebuf); + if( gp->printPreprocessed ){ + printf("%s\n", filebuf); + return; + } + + /* Now scan the text of the input file */ + lineno = 1; + for(cp=filebuf; (c= *cp)!=0; ){ + if( c=='\n' ) lineno++; /* Keep track of the line number */ + if( ISSPACE(c) ){ cp++; continue; } /* Skip all white space */ + if( c=='/' && cp[1]=='/' ){ /* Skip C++ style comments */ + cp+=2; + while( (c= *cp)!=0 && c!='\n' ) cp++; + continue; + } + if( c=='/' && cp[1]=='*' ){ /* Skip C style comments */ + cp+=2; + if( (*cp)=='/' ) cp++; + while( (c= *cp)!=0 && (c!='/' || cp[-1]!='*') ){ + if( c=='\n' ) lineno++; + cp++; + } + if( c ) cp++; + continue; + } + ps.tokenstart = cp; /* Mark the beginning of the token */ + ps.tokenlineno = lineno; /* Linenumber on which token begins */ + if( c=='\"' ){ /* String literals */ + cp++; + while( (c= *cp)!=0 && c!='\"' ){ + if( c=='\n' ) lineno++; + cp++; + } + if( c==0 ){ + ErrorMsg(ps.filename,startline, + "String starting on this line is not terminated before " + "the end of the file."); + ps.errorcnt++; + nextcp = cp; + }else{ + nextcp = cp+1; + } + }else if( c=='{' ){ /* A block of C code */ + int level; + cp++; + for(level=1; (c= *cp)!=0 && (level>1 || c!='}'); cp++){ + if( c=='\n' ) lineno++; + else if( c=='{' ) level++; + else if( c=='}' ) level--; + else if( c=='/' && cp[1]=='*' ){ /* Skip comments */ + int prevc; + cp = &cp[2]; + prevc = 0; + while( (c= *cp)!=0 && (c!='/' || prevc!='*') ){ + if( c=='\n' ) lineno++; + prevc = c; + cp++; + } + }else if( c=='/' && cp[1]=='/' ){ /* Skip C++ style comments too */ + cp = &cp[2]; + while( (c= *cp)!=0 && c!='\n' ) cp++; + if( c ) lineno++; + }else if( c=='\'' || c=='\"' ){ /* String a character literals */ + int startchar, prevc; + startchar = c; + prevc = 0; + for(cp++; (c= *cp)!=0 && (c!=startchar || prevc=='\\'); cp++){ + if( c=='\n' ) lineno++; + if( prevc=='\\' ) prevc = 0; + else prevc = c; + } + } + } + if( c==0 ){ + ErrorMsg(ps.filename,ps.tokenlineno, + "C code starting on this line is not terminated before " + "the end of the file."); + ps.errorcnt++; + nextcp = cp; + }else{ + nextcp = cp+1; + } + }else if( ISALNUM(c) ){ /* Identifiers */ + while( (c= *cp)!=0 && (ISALNUM(c) || c=='_') ) cp++; + nextcp = cp; + }else if( c==':' && cp[1]==':' && cp[2]=='=' ){ /* The operator "::=" */ + cp += 3; + nextcp = cp; + }else if( (c=='/' || c=='|') && ISALPHA(cp[1]) ){ + cp += 2; + while( (c = *cp)!=0 && (ISALNUM(c) || c=='_') ) cp++; + nextcp = cp; + }else{ /* All other (one character) operators */ + cp++; + nextcp = cp; + } + c = *cp; + *cp = 0; /* Null terminate the token */ + parseonetoken(&ps); /* Parse the token */ + *cp = (char)c; /* Restore the buffer */ + cp = nextcp; + } + lemon_free(filebuf); /* Release the buffer after parsing */ + gp->rule = ps.firstrule; + gp->errorcnt = ps.errorcnt; +} +/*************************** From the file "plink.c" *********************/ +/* +** Routines processing configuration follow-set propagation links +** in the LEMON parser generator. +*/ +static struct plink *plink_freelist = 0; + +/* Allocate a new plink */ +struct plink *Plink_new(void){ + struct plink *newlink; + + if( plink_freelist==0 ){ + int i; + int amt = 100; + plink_freelist = (struct plink *)lemon_calloc( amt, sizeof(struct plink) ); + if( plink_freelist==0 ){ + fprintf(stderr, + "Unable to allocate memory for a new follow-set propagation link.\n"); + exit(1); + } + for(i=0; inext; + return newlink; +} + +/* Add a plink to a plink list */ +void Plink_add(struct plink **plpp, struct config *cfp) +{ + struct plink *newlink; + newlink = Plink_new(); + newlink->next = *plpp; + *plpp = newlink; + newlink->cfp = cfp; +} + +/* Transfer every plink on the list "from" to the list "to" */ +void Plink_copy(struct plink **to, struct plink *from) +{ + struct plink *nextpl; + while( from ){ + nextpl = from->next; + from->next = *to; + *to = from; + from = nextpl; + } +} + +/* Delete every plink on the list */ +void Plink_delete(struct plink *plp) +{ + struct plink *nextpl; + + while( plp ){ + nextpl = plp->next; + plp->next = plink_freelist; + plink_freelist = plp; + plp = nextpl; + } +} +/*********************** From the file "report.c" **************************/ +/* +** Procedures for generating reports and tables in the LEMON parser generator. +*/ + +/* Generate a filename with the given suffix. +*/ +PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) +{ + char *name; + char *cp; + char *filename = lemp->filename; + int sz; + + if( outputDir ){ + cp = strrchr(filename, '/'); + if( cp ) filename = cp + 1; + } + sz = lemonStrlen(filename); + sz += lemonStrlen(suffix); + if( outputDir ) sz += lemonStrlen(outputDir) + 1; + sz += 5; + name = (char*)lemon_malloc( sz ); + if( name==0 ){ + fprintf(stderr,"Can't allocate space for a filename.\n"); + exit(1); + } + name[0] = 0; + if( outputDir ){ + lemon_strcpy(name, outputDir); + lemon_strcat(name, "/"); + } + lemon_strcat(name,filename); + cp = strrchr(name,'.'); + if( cp ) *cp = 0; + lemon_strcat(name,suffix); + return name; +} + +/* Open a file with a name based on the name of the input file, +** but with a different (specified) suffix, and return a pointer +** to the stream */ +PRIVATE FILE *file_open( + struct lemon *lemp, + const char *suffix, + const char *mode +){ + FILE *fp; + + if( lemp->outname ) lemon_free(lemp->outname); + lemp->outname = file_makename(lemp, suffix); + fp = fopen(lemp->outname,mode); + if( fp==0 && *mode=='w' ){ + fprintf(stderr,"Can't open file \"%s\".\n",lemp->outname); + lemp->errorcnt++; + return 0; + } + return fp; +} + +/* Print the text of a rule +*/ +void rule_print(FILE *out, struct rule *rp){ + int i, j; + fprintf(out, "%s",rp->lhs->name); + /* if( rp->lhsalias ) fprintf(out,"(%s)",rp->lhsalias); */ + fprintf(out," ::="); + for(i=0; inrhs; i++){ + struct symbol *sp = rp->rhs[i]; + if( sp->type==MULTITERMINAL ){ + fprintf(out," %s", sp->subsym[0]->name); + for(j=1; jnsubsym; j++){ + fprintf(out,"|%s", sp->subsym[j]->name); + } + }else{ + fprintf(out," %s", sp->name); + } + /* if( rp->rhsalias[i] ) fprintf(out,"(%s)",rp->rhsalias[i]); */ + } +} + +/* Duplicate the input file without comments and without actions +** on rules */ +void Reprint(struct lemon *lemp) +{ + struct rule *rp; + struct symbol *sp; + int i, j, maxlen, len, ncolumns, skip; + printf("// Reprint of input file \"%s\".\n// Symbols:\n",lemp->filename); + maxlen = 10; + for(i=0; insymbol; i++){ + sp = lemp->symbols[i]; + len = lemonStrlen(sp->name); + if( len>maxlen ) maxlen = len; + } + ncolumns = 76/(maxlen+5); + if( ncolumns<1 ) ncolumns = 1; + skip = (lemp->nsymbol + ncolumns - 1)/ncolumns; + for(i=0; insymbol; j+=skip){ + sp = lemp->symbols[j]; + assert( sp->index==j ); + printf(" %3d %-*.*s",j,maxlen,maxlen,sp->name); + } + printf("\n"); + } + for(rp=lemp->rule; rp; rp=rp->next){ + rule_print(stdout, rp); + printf("."); + if( rp->precsym ) printf(" [%s]",rp->precsym->name); + /* if( rp->code ) printf("\n %s",rp->code); */ + printf("\n"); + } +} + +/* Print a single rule. +*/ +void RulePrint(FILE *fp, struct rule *rp, int iCursor){ + struct symbol *sp; + int i, j; + fprintf(fp,"%s ::=",rp->lhs->name); + for(i=0; i<=rp->nrhs; i++){ + if( i==iCursor ) fprintf(fp," *"); + if( i==rp->nrhs ) break; + sp = rp->rhs[i]; + if( sp->type==MULTITERMINAL ){ + fprintf(fp," %s", sp->subsym[0]->name); + for(j=1; jnsubsym; j++){ + fprintf(fp,"|%s",sp->subsym[j]->name); + } + }else{ + fprintf(fp," %s", sp->name); + } + } +} + +/* Print the rule for a configuration. +*/ +void ConfigPrint(FILE *fp, struct config *cfp){ + RulePrint(fp, cfp->rp, cfp->dot); +} + +/* #define TEST */ +#if 0 +/* Print a set */ +PRIVATE void SetPrint(out,set,lemp) +FILE *out; +char *set; +struct lemon *lemp; +{ + int i; + char *spacer; + spacer = ""; + fprintf(out,"%12s[",""); + for(i=0; interminal; i++){ + if( SetFind(set,i) ){ + fprintf(out,"%s%s",spacer,lemp->symbols[i]->name); + spacer = " "; + } + } + fprintf(out,"]\n"); +} + +/* Print a plink chain */ +PRIVATE void PlinkPrint(out,plp,tag) +FILE *out; +struct plink *plp; +char *tag; +{ + while( plp ){ + fprintf(out,"%12s%s (state %2d) ","",tag,plp->cfp->stp->statenum); + ConfigPrint(out,plp->cfp); + fprintf(out,"\n"); + plp = plp->next; + } +} +#endif + +/* Print an action to the given file descriptor. Return FALSE if +** nothing was actually printed. +*/ +int PrintAction( + struct action *ap, /* The action to print */ + FILE *fp, /* Print the action here */ + int indent /* Indent by this amount */ +){ + int result = 1; + switch( ap->type ){ + case SHIFT: { + struct state *stp = ap->x.stp; + fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); + break; + } + case REDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->iRule); + RulePrint(fp, rp, -1); + break; + } + case SHIFTREDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->iRule); + RulePrint(fp, rp, -1); + break; + } + case ACCEPT: + fprintf(fp,"%*s accept",indent,ap->sp->name); + break; + case ERROR: + fprintf(fp,"%*s error",indent,ap->sp->name); + break; + case SRCONFLICT: + case RRCONFLICT: + fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", + indent,ap->sp->name,ap->x.rp->iRule); + break; + case SSCONFLICT: + fprintf(fp,"%*s shift %-7d ** Parsing conflict **", + indent,ap->sp->name,ap->x.stp->statenum); + break; + case SH_RESOLVED: + if( showPrecedenceConflict ){ + fprintf(fp,"%*s shift %-7d -- dropped by precedence", + indent,ap->sp->name,ap->x.stp->statenum); + }else{ + result = 0; + } + break; + case RD_RESOLVED: + if( showPrecedenceConflict ){ + fprintf(fp,"%*s reduce %-7d -- dropped by precedence", + indent,ap->sp->name,ap->x.rp->iRule); + }else{ + result = 0; + } + break; + case NOT_USED: + result = 0; + break; + } + if( result && ap->spOpt ){ + fprintf(fp," /* because %s==%s */", ap->sp->name, ap->spOpt->name); + } + return result; +} + +/* Generate the "*.out" log file */ +void ReportOutput(struct lemon *lemp) +{ + int i, n; + struct state *stp; + struct config *cfp; + struct action *ap; + struct rule *rp; + FILE *fp; + + fp = file_open(lemp,".out","wb"); + if( fp==0 ) return; + for(i=0; inxstate; i++){ + stp = lemp->sorted[i]; + fprintf(fp,"State %d:\n",stp->statenum); + if( lemp->basisflag ) cfp=stp->bp; + else cfp=stp->cfp; + while( cfp ){ + char buf[20]; + if( cfp->dot==cfp->rp->nrhs ){ + lemon_sprintf(buf,"(%d)",cfp->rp->iRule); + fprintf(fp," %5s ",buf); + }else{ + fprintf(fp," "); + } + ConfigPrint(fp,cfp); + fprintf(fp,"\n"); +#if 0 + SetPrint(fp,cfp->fws,lemp); + PlinkPrint(fp,cfp->fplp,"To "); + PlinkPrint(fp,cfp->bplp,"From"); +#endif + if( lemp->basisflag ) cfp=cfp->bp; + else cfp=cfp->next; + } + fprintf(fp,"\n"); + for(ap=stp->ap; ap; ap=ap->next){ + if( PrintAction(ap,fp,30) ) fprintf(fp,"\n"); + } + fprintf(fp,"\n"); + } + fprintf(fp, "----------------------------------------------------\n"); + fprintf(fp, "Symbols:\n"); + fprintf(fp, "The first-set of non-terminals is shown after the name.\n\n"); + for(i=0; insymbol; i++){ + int j; + struct symbol *sp; + + sp = lemp->symbols[i]; + fprintf(fp, " %3d: %s", i, sp->name); + if( sp->type==NONTERMINAL ){ + fprintf(fp, ":"); + if( sp->lambda ){ + fprintf(fp, " "); + } + for(j=0; jnterminal; j++){ + if( sp->firstset && SetFind(sp->firstset, j) ){ + fprintf(fp, " %s", lemp->symbols[j]->name); + } + } + } + if( sp->prec>=0 ) fprintf(fp," (precedence=%d)", sp->prec); + fprintf(fp, "\n"); + } + fprintf(fp, "----------------------------------------------------\n"); + fprintf(fp, "Syntax-only Symbols:\n"); + fprintf(fp, "The following symbols never carry semantic content.\n\n"); + for(i=n=0; insymbol; i++){ + int w; + struct symbol *sp = lemp->symbols[i]; + if( sp->bContent ) continue; + w = (int)strlen(sp->name); + if( n>0 && n+w>75 ){ + fprintf(fp,"\n"); + n = 0; + } + if( n>0 ){ + fprintf(fp, " "); + n++; + } + fprintf(fp, "%s", sp->name); + n += w; + } + if( n>0 ) fprintf(fp, "\n"); + fprintf(fp, "----------------------------------------------------\n"); + fprintf(fp, "Rules:\n"); + for(rp=lemp->rule; rp; rp=rp->next){ + fprintf(fp, "%4d: ", rp->iRule); + rule_print(fp, rp); + fprintf(fp,"."); + if( rp->precsym ){ + fprintf(fp," [%s precedence=%d]", + rp->precsym->name, rp->precsym->prec); + } + fprintf(fp,"\n"); + } + fclose(fp); + return; +} + +/* Search for the file "name" which is in the same directory as +** the executable */ +PRIVATE char *pathsearch(char *argv0, char *name, int modemask) +{ + const char *pathlist; + char *pathbufptr = 0; + char *pathbuf = 0; + char *path,*cp; + char c; + +#ifdef __WIN32__ + cp = strrchr(argv0,'\\'); +#else + cp = strrchr(argv0,'/'); +#endif + if( cp ){ + c = *cp; + *cp = 0; + path = (char *)lemon_malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 ); + if( path ) lemon_sprintf(path,"%s/%s",argv0,name); + *cp = c; + }else{ + pathlist = getenv("PATH"); + if( pathlist==0 ) pathlist = ".:/bin:/usr/bin"; + pathbuf = (char *) lemon_malloc( lemonStrlen(pathlist) + 1 ); + path = (char *)lemon_malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); + if( (pathbuf != 0) && (path!=0) ){ + pathbufptr = pathbuf; + lemon_strcpy(pathbuf, pathlist); + while( *pathbuf ){ + cp = strchr(pathbuf,':'); + if( cp==0 ) cp = &pathbuf[lemonStrlen(pathbuf)]; + c = *cp; + *cp = 0; + lemon_sprintf(path,"%s/%s",pathbuf,name); + *cp = c; + if( c==0 ) pathbuf[0] = 0; + else pathbuf = &cp[1]; + if( access(path,modemask)==0 ) break; + } + } + lemon_free(pathbufptr); + } + return path; +} + +/* Given an action, compute the integer value for that action +** which is to be put in the action table of the generated machine. +** Return negative if no action should be generated. +*/ +PRIVATE int compute_action(struct lemon *lemp, struct action *ap) +{ + int act; + switch( ap->type ){ + case SHIFT: act = ap->x.stp->statenum; break; + case SHIFTREDUCE: { + /* Since a SHIFT is inherient after a prior REDUCE, convert any + ** SHIFTREDUCE action with a nonterminal on the LHS into a simple + ** REDUCE action: */ + if( ap->sp->index>=lemp->nterminal + && (lemp->errsym==0 || ap->sp->index!=lemp->errsym->index) + ){ + act = lemp->minReduce + ap->x.rp->iRule; + }else{ + act = lemp->minShiftReduce + ap->x.rp->iRule; + } + break; + } + case REDUCE: act = lemp->minReduce + ap->x.rp->iRule; break; + case ERROR: act = lemp->errAction; break; + case ACCEPT: act = lemp->accAction; break; + default: act = -1; break; + } + return act; +} + +#define LINESIZE 1000 +/* The next cluster of routines are for reading the template file +** and writing the results to the generated parser */ +/* The first function transfers data from "in" to "out" until +** a line is seen which begins with "%%". The line number is +** tracked. +** +** if name!=0, then any word that begin with "Parse" is changed to +** begin with *name instead. +*/ +PRIVATE void tplt_xfer(char *name, FILE *in, FILE *out, int *lineno) +{ + int i, iStart; + char line[LINESIZE]; + while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){ + (*lineno)++; + iStart = 0; + if( name ){ + for(i=0; line[i]; i++){ + if( line[i]=='P' && strncmp(&line[i],"Parse",5)==0 + && (i==0 || !ISALPHA(line[i-1])) + ){ + if( i>iStart ) fprintf(out,"%.*s",i-iStart,&line[iStart]); + fprintf(out,"%s",name); + i += 4; + iStart = i+1; + } + } + } + fprintf(out,"%s",&line[iStart]); + } +} + +/* Skip forward past the header of the template file to the first "%%" +*/ +PRIVATE void tplt_skip_header(FILE *in, int *lineno) +{ + char line[LINESIZE]; + while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){ + (*lineno)++; + } +} + +/* The next function finds the template file and opens it, returning +** a pointer to the opened file. */ +PRIVATE FILE *tplt_open(struct lemon *lemp) +{ + static char templatename[] = "lempar.c"; + char buf[1000]; + FILE *in; + char *tpltname; + char *toFree = 0; + char *cp; + + /* first, see if user specified a template filename on the command line. */ + if (user_templatename != 0) { + if( access(user_templatename,004)==-1 ){ + fprintf(stderr,"Can't find the parser driver template file \"%s\".\n", + user_templatename); + lemp->errorcnt++; + return 0; + } + in = fopen(user_templatename,"rb"); + if( in==0 ){ + fprintf(stderr,"Can't open the template file \"%s\".\n", + user_templatename); + lemp->errorcnt++; + return 0; + } + return in; + } + + cp = strrchr(lemp->filename,'.'); + if( cp ){ + lemon_sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); + }else{ + lemon_sprintf(buf,"%s.lt",lemp->filename); + } + if( access(buf,004)==0 ){ + tpltname = buf; + }else if( access(templatename,004)==0 ){ + tpltname = templatename; + }else{ + toFree = tpltname = pathsearch(lemp->argv[0],templatename,0); + } + if( tpltname==0 ){ + fprintf(stderr,"Can't find the parser driver template file \"%s\".\n", + templatename); + lemp->errorcnt++; + return 0; + } + in = fopen(tpltname,"rb"); + if( in==0 ){ + fprintf(stderr,"Can't open the template file \"%s\".\n",tpltname); + lemp->errorcnt++; + } + lemon_free(toFree); + return in; +} + +/* Print a #line directive line to the output file. */ +PRIVATE void tplt_linedir(FILE *out, int lineno, char *filename) +{ + fprintf(out,"#line %d \"",lineno); + while( *filename ){ + if( *filename == '\\' ) putc('\\',out); + putc(*filename,out); + filename++; + } + fprintf(out,"\"\n"); +} + +/* Print a string to the file and keep the linenumber up to date */ +PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) +{ + if( str==0 ) return; + while( *str ){ + putc(*str,out); + if( *str=='\n' ) (*lineno)++; + str++; + } + if( str[-1]!='\n' ){ + putc('\n',out); + (*lineno)++; + } + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + return; +} + +/* +** The following routine emits code for the destructor for the +** symbol sp +*/ +void emit_destructor_code( + FILE *out, + struct symbol *sp, + struct lemon *lemp, + int *lineno +){ + char *cp = 0; + + if( sp->type==TERMINAL ){ + cp = lemp->tokendest; + if( cp==0 ) return; + fprintf(out,"{\n"); (*lineno)++; + }else if( sp->destructor ){ + cp = sp->destructor; + fprintf(out,"{\n"); (*lineno)++; + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,sp->destLineno,lemp->filename); + } + }else if( lemp->vardest ){ + cp = lemp->vardest; + if( cp==0 ) return; + fprintf(out,"{\n"); (*lineno)++; + }else{ + assert( 0 ); /* Cannot happen */ + } + for(; *cp; cp++){ + if( *cp=='$' && cp[1]=='$' ){ + fprintf(out,"(yypminor->yy%d)",sp->dtnum); + cp++; + continue; + } + if( *cp=='\n' ) (*lineno)++; + fputc(*cp,out); + } + fprintf(out,"\n"); (*lineno)++; + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + } + fprintf(out,"}\n"); (*lineno)++; + return; +} + +/* +** Return TRUE (non-zero) if the given symbol has a destructor. +*/ +int has_destructor(struct symbol *sp, struct lemon *lemp) +{ + int ret; + if( sp->type==TERMINAL ){ + ret = lemp->tokendest!=0; + }else{ + ret = lemp->vardest!=0 || sp->destructor!=0; + } + return ret; +} + +/* +** Append text to a dynamically allocated string. If zText is 0 then +** reset the string to be empty again. Always return the complete text +** of the string (which is overwritten with each call). +** +** n bytes of zText are stored. If n==0 then all of zText up to the first +** \000 terminator is stored. zText can contain up to two instances of +** %d. The values of p1 and p2 are written into the first and second +** %d. +** +** If n==-1, then the previous character is overwritten. +*/ +PRIVATE char *append_str(const char *zText, int n, int p1, int p2){ + static char empty[1] = { 0 }; + static char *z = 0; + static int alloced = 0; + static int used = 0; + int c; + char zInt[40]; + if( zText==0 ){ + if( used==0 && z!=0 ) z[0] = 0; + used = 0; + return z; + } + if( n<=0 ){ + if( n<0 ){ + used += n; + assert( used>=0 ); + } + n = lemonStrlen(zText); + } + if( (int) (n+sizeof(zInt)*2+used) >= alloced ){ + alloced = n + sizeof(zInt)*2 + used + 200; + z = (char *) lemon_realloc(z, alloced); + } + if( z==0 ) return empty; + while( n-- > 0 ){ + c = *(zText++); + if( c=='%' && n>0 && zText[0]=='d' ){ + lemon_sprintf(zInt, "%d", p1); + p1 = p2; + lemon_strcpy(&z[used], zInt); + used += lemonStrlen(&z[used]); + zText++; + n--; + }else{ + z[used++] = (char)c; + } + } + z[used] = 0; + return z; +} + +/* +** Write and transform the rp->code string so that symbols are expanded. +** Populate the rp->codePrefix and rp->codeSuffix strings, as appropriate. +** +** Return 1 if the expanded code requires that "yylhsminor" local variable +** to be defined. +*/ +PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ + char *cp, *xp; + int i; + int rc = 0; /* True if yylhsminor is used */ + int dontUseRhs0 = 0; /* If true, use of left-most RHS label is illegal */ + const char *zSkip = 0; /* The zOvwrt comment within rp->code, or NULL */ + char lhsused = 0; /* True if the LHS element has been used */ + char lhsdirect; /* True if LHS writes directly into stack */ + char used[MAXRHS]; /* True for each RHS element which is used */ + char zLhs[50]; /* Convert the LHS symbol into this string */ + char zOvwrt[900]; /* Comment that to allow LHS to overwrite RHS */ + + for(i=0; inrhs; i++) used[i] = 0; + lhsused = 0; + + if( rp->code==0 ){ + static char newlinestr[2] = { '\n', '\0' }; + rp->code = newlinestr; + rp->line = rp->ruleline; + rp->noCode = 1; + }else{ + rp->noCode = 0; + } + + + if( rp->nrhs==0 ){ + /* If there are no RHS symbols, then writing directly to the LHS is ok */ + lhsdirect = 1; + }else if( rp->rhsalias[0]==0 ){ + /* The left-most RHS symbol has no value. LHS direct is ok. But + ** we have to call the destructor on the RHS symbol first. */ + lhsdirect = 1; + if( has_destructor(rp->rhs[0],lemp) ){ + append_str(0,0,0,0); + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[0]->index,1-rp->nrhs); + rp->codePrefix = Strsafe(append_str(0,0,0,0)); + rp->noCode = 0; + } + }else if( rp->lhsalias==0 ){ + /* There is no LHS value symbol. */ + lhsdirect = 1; + }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){ + /* The LHS symbol and the left-most RHS symbol are the same, so + ** direct writing is allowed */ + lhsdirect = 1; + lhsused = 1; + used[0] = 1; + if( rp->lhs->dtnum!=rp->rhs[0]->dtnum ){ + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) and %s(%s) share the same label but have " + "different datatypes.", + rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]); + lemp->errorcnt++; + } + }else{ + lemon_sprintf(zOvwrt, "/*%s-overwrites-%s*/", + rp->lhsalias, rp->rhsalias[0]); + zSkip = strstr(rp->code, zOvwrt); + if( zSkip!=0 ){ + /* The code contains a special comment that indicates that it is safe + ** for the LHS label to overwrite left-most RHS label. */ + lhsdirect = 1; + }else{ + lhsdirect = 0; + } + } + if( lhsdirect ){ + sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); + }else{ + rc = 1; + sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum); + } + + append_str(0,0,0,0); + + /* This const cast is wrong but harmless, if we're careful. */ + for(cp=(char *)rp->code; *cp; cp++){ + if( cp==zSkip ){ + append_str(zOvwrt,0,0,0); + cp += lemonStrlen(zOvwrt)-1; + dontUseRhs0 = 1; + continue; + } + if( ISALPHA(*cp) && (cp==rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ){ + char saved; + for(xp= &cp[1]; ISALNUM(*xp) || *xp=='_'; xp++); + saved = *xp; + *xp = 0; + if( rp->lhsalias && strcmp(cp,rp->lhsalias)==0 ){ + append_str(zLhs,0,0,0); + cp = xp; + lhsused = 1; + }else{ + for(i=0; inrhs; i++){ + if( rp->rhsalias[i] && strcmp(cp,rp->rhsalias[i])==0 ){ + if( i==0 && dontUseRhs0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used after '%s'.", + rp->rhsalias[0], zOvwrt); + lemp->errorcnt++; + }else if( cp!=rp->code && cp[-1]=='@' ){ + /* If the argument is of the form @X then substituted + ** the token number of X, not the value of X */ + append_str("yymsp[%d].major",-1,i-rp->nrhs+1,0); + }else{ + struct symbol *sp = rp->rhs[i]; + int dtnum; + if( sp->type==MULTITERMINAL ){ + dtnum = sp->subsym[0]->dtnum; + }else{ + dtnum = sp->dtnum; + } + append_str("yymsp[%d].minor.yy%d",0,i-rp->nrhs+1, dtnum); + } + cp = xp; + used[i] = 1; + break; + } + } + } + *xp = saved; + } + append_str(cp, 1, 0, 0); + } /* End loop */ + + /* Main code generation completed */ + cp = append_str(0,0,0,0); + if( cp && cp[0] ) rp->code = Strsafe(cp); + append_str(0,0,0,0); + + /* Check to make sure the LHS has been used */ + if( rp->lhsalias && !lhsused ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label \"%s\" for \"%s(%s)\" is never used.", + rp->lhsalias,rp->lhs->name,rp->lhsalias); + lemp->errorcnt++; + } + + /* Generate destructor code for RHS minor values which are not referenced. + ** Generate error messages for unused labels and duplicate labels. + */ + for(i=0; inrhs; i++){ + if( rp->rhsalias[i] ){ + if( i>0 ){ + int j; + if( rp->lhsalias && strcmp(rp->lhsalias,rp->rhsalias[i])==0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "%s(%s) has the same label as the LHS but is not the left-most " + "symbol on the RHS.", + rp->rhs[i]->name, rp->rhsalias[i]); + lemp->errorcnt++; + } + for(j=0; jrhsalias[j] && strcmp(rp->rhsalias[j],rp->rhsalias[i])==0 ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s used for multiple symbols on the RHS of a rule.", + rp->rhsalias[i]); + lemp->errorcnt++; + break; + } + } + } + if( !used[i] ){ + ErrorMsg(lemp->filename,rp->ruleline, + "Label %s for \"%s(%s)\" is never used.", + rp->rhsalias[i],rp->rhs[i]->name,rp->rhsalias[i]); + lemp->errorcnt++; + } + }else if( i>0 && has_destructor(rp->rhs[i],lemp) ){ + append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0, + rp->rhs[i]->index,i-rp->nrhs+1); + } + } + + /* If unable to write LHS values directly into the stack, write the + ** saved LHS value now. */ + if( lhsdirect==0 ){ + append_str(" yymsp[%d].minor.yy%d = ", 0, 1-rp->nrhs, rp->lhs->dtnum); + append_str(zLhs, 0, 0, 0); + append_str(";\n", 0, 0, 0); + } + + /* Suffix code generation complete */ + cp = append_str(0,0,0,0); + if( cp && cp[0] ){ + rp->codeSuffix = Strsafe(cp); + rp->noCode = 0; + } + + return rc; +} + +/* +** Generate code which executes when the rule "rp" is reduced. Write +** the code to "out". Make sure lineno stays up-to-date. +*/ +PRIVATE void emit_code( + FILE *out, + struct rule *rp, + struct lemon *lemp, + int *lineno +){ + const char *cp; + + /* Setup code prior to the #line directive */ + if( rp->codePrefix && rp->codePrefix[0] ){ + fprintf(out, "{%s", rp->codePrefix); + for(cp=rp->codePrefix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + } + + /* Generate code to do the reduce action */ + if( rp->code ){ + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,rp->line,lemp->filename); + } + fprintf(out,"{%s",rp->code); + for(cp=rp->code; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + fprintf(out,"}\n"); (*lineno)++; + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,*lineno,lemp->outname); + } + } + + /* Generate breakdown code that occurs after the #line directive */ + if( rp->codeSuffix && rp->codeSuffix[0] ){ + fprintf(out, "%s", rp->codeSuffix); + for(cp=rp->codeSuffix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } + } + + if( rp->codePrefix ){ + fprintf(out, "}\n"); (*lineno)++; + } + + return; +} + +/* +** Print the definition of the union used for the parser's data stack. +** This union contains fields for every possible data type for tokens +** and nonterminals. In the process of computing and printing this +** union, also set the ".dtnum" field of every terminal and nonterminal +** symbol. +*/ +void print_stack_union( + FILE *out, /* The output stream */ + struct lemon *lemp, /* The main info structure for this parser */ + int *plineno, /* Pointer to the line number */ + int mhflag /* True if generating makeheaders output */ +){ + int lineno; /* The line number of the output */ + char **types; /* A hash table of datatypes */ + int arraysize; /* Size of the "types" array */ + int maxdtlength; /* Maximum length of any ".datatype" field. */ + char *stddt; /* Standardized name for a datatype */ + int i,j; /* Loop counters */ + unsigned hash; /* For hashing the name of a type */ + const char *name; /* Name of the parser */ + + /* Allocate and initialize types[] and allocate stddt[] */ + arraysize = lemp->nsymbol * 2; + types = (char**)lemon_calloc( arraysize, sizeof(char*) ); + if( types==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + for(i=0; ivartype ){ + maxdtlength = lemonStrlen(lemp->vartype); + } + for(i=0; insymbol; i++){ + int len; + struct symbol *sp = lemp->symbols[i]; + if( sp->datatype==0 ) continue; + len = lemonStrlen(sp->datatype); + if( len>maxdtlength ) maxdtlength = len; + } + stddt = (char*)lemon_malloc( maxdtlength*2 + 1 ); + if( stddt==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + + /* Build a hash table of datatypes. The ".dtnum" field of each symbol + ** is filled in with the hash index plus 1. A ".dtnum" value of 0 is + ** used for terminal symbols. If there is no %default_type defined then + ** 0 is also used as the .dtnum value for nonterminals which do not specify + ** a datatype using the %type directive. + */ + for(i=0; insymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + char *cp; + if( sp==lemp->errsym ){ + sp->dtnum = arraysize+1; + continue; + } + if( sp->type!=NONTERMINAL || (sp->datatype==0 && lemp->vartype==0) ){ + sp->dtnum = 0; + continue; + } + cp = sp->datatype; + if( cp==0 ) cp = lemp->vartype; + j = 0; + while( ISSPACE(*cp) ) cp++; + while( *cp ) stddt[j++] = *cp++; + while( j>0 && ISSPACE(stddt[j-1]) ) j--; + stddt[j] = 0; + if( lemp->tokentype && strcmp(stddt, lemp->tokentype)==0 ){ + sp->dtnum = 0; + continue; + } + hash = 0; + for(j=0; stddt[j]; j++){ + hash = hash*53 + stddt[j]; + } + hash = (hash & 0x7fffffff)%arraysize; + while( types[hash] ){ + if( strcmp(types[hash],stddt)==0 ){ + sp->dtnum = hash + 1; + break; + } + hash++; + if( hash>=(unsigned)arraysize ) hash = 0; + } + if( types[hash]==0 ){ + sp->dtnum = hash + 1; + types[hash] = (char*)lemon_malloc( lemonStrlen(stddt)+1 ); + if( types[hash]==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } + lemon_strcpy(types[hash],stddt); + } + } + + /* Print out the definition of YYTOKENTYPE and YYMINORTYPE */ + name = lemp->name ? lemp->name : "Parse"; + lineno = *plineno; + if( mhflag ){ fprintf(out,"#if INTERFACE\n"); lineno++; } + fprintf(out,"#define %sTOKENTYPE %s\n",name, + lemp->tokentype?lemp->tokentype:"void*"); lineno++; + if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } + fprintf(out,"typedef union {\n"); lineno++; + fprintf(out," int yyinit;\n"); lineno++; + fprintf(out," %sTOKENTYPE yy0;\n",name); lineno++; + for(i=0; ierrsym && lemp->errsym->useCnt ){ + fprintf(out," int yy%d;\n",lemp->errsym->dtnum); lineno++; + } + lemon_free(stddt); + lemon_free(types); + fprintf(out,"} YYMINORTYPE;\n"); lineno++; + *plineno = lineno; +} + +/* +** Return the name of a C datatype able to represent values between +** lwr and upr, inclusive. If pnByte!=NULL then also write the sizeof +** for that type (1, 2, or 4) into *pnByte. +*/ +static const char *minimum_size_type(int lwr, int upr, int *pnByte){ + const char *zType = "int"; + int nByte = 4; + if( lwr>=0 ){ + if( upr<=255 ){ + zType = "unsigned char"; + nByte = 1; + }else if( upr<65535 ){ + zType = "unsigned short int"; + nByte = 2; + }else{ + zType = "unsigned int"; + nByte = 4; + } + }else if( lwr>=-127 && upr<=127 ){ + zType = "signed char"; + nByte = 1; + }else if( lwr>=-32767 && upr<32767 ){ + zType = "short"; + nByte = 2; + } + if( pnByte ) *pnByte = nByte; + return zType; +} + +/* +** Each state contains a set of token transaction and a set of +** nonterminal transactions. Each of these sets makes an instance +** of the following structure. An array of these structures is used +** to order the creation of entries in the yy_action[] table. +*/ +struct axset { + struct state *stp; /* A pointer to a state */ + int isTkn; /* True to use tokens. False for non-terminals */ + int nAction; /* Number of actions */ + int iOrder; /* Original order of action sets */ +}; + +/* +** Compare to axset structures for sorting purposes +*/ +static int axset_compare(const void *a, const void *b){ + struct axset *p1 = (struct axset*)a; + struct axset *p2 = (struct axset*)b; + int c; + c = p2->nAction - p1->nAction; + if( c==0 ){ + c = p1->iOrder - p2->iOrder; + } + assert( c!=0 || p1==p2 ); + return c; +} + +/* +** Write text on "out" that describes the rule "rp". +*/ +static void writeRuleText(FILE *out, struct rule *rp){ + int j; + fprintf(out,"%s ::=", rp->lhs->name); + for(j=0; jnrhs; j++){ + struct symbol *sp = rp->rhs[j]; + if( sp->type!=MULTITERMINAL ){ + fprintf(out," %s", sp->name); + }else{ + int k; + fprintf(out," %s", sp->subsym[0]->name); + for(k=1; knsubsym; k++){ + fprintf(out,"|%s",sp->subsym[k]->name); + } + } + } +} + + +/* Generate C source code for the parser */ +void ReportTable( + struct lemon *lemp, + int mhflag, /* Output in makeheaders format if true */ + int sqlFlag /* Generate the *.sql file too */ +){ + FILE *out, *in, *sql; + int lineno; + struct state *stp; + struct action *ap; + struct rule *rp; + struct acttab *pActtab; + int i, j, n, sz, mn, mx; + int nLookAhead; + int szActionType; /* sizeof(YYACTIONTYPE) */ + int szCodeType; /* sizeof(YYCODETYPE) */ + const char *name; + int mnTknOfst, mxTknOfst; + int mnNtOfst, mxNtOfst; + struct axset *ax; + char *prefix; + + lemp->minShiftReduce = lemp->nstate; + lemp->errAction = lemp->minShiftReduce + lemp->nrule; + lemp->accAction = lemp->errAction + 1; + lemp->noAction = lemp->accAction + 1; + lemp->minReduce = lemp->noAction + 1; + lemp->maxAction = lemp->minReduce + lemp->nrule; + + in = tplt_open(lemp); + if( in==0 ) return; + out = file_open(lemp,".c","wb"); + if( out==0 ){ + fclose(in); + return; + } + if( sqlFlag==0 ){ + sql = 0; + }else{ + sql = file_open(lemp, ".sql", "wb"); + if( sql==0 ){ + fclose(in); + fclose(out); + return; + } + fprintf(sql, + "BEGIN;\n" + "CREATE TABLE symbol(\n" + " id INTEGER PRIMARY KEY,\n" + " name TEXT NOT NULL,\n" + " isTerminal BOOLEAN NOT NULL,\n" + " fallback INTEGER REFERENCES symbol" + " DEFERRABLE INITIALLY DEFERRED\n" + ");\n" + ); + for(i=0; insymbol; i++){ + fprintf(sql, + "INSERT INTO symbol(id,name,isTerminal,fallback)" + "VALUES(%d,'%s',%s", + i, lemp->symbols[i]->name, + interminal ? "TRUE" : "FALSE" + ); + if( lemp->symbols[i]->fallback ){ + fprintf(sql, ",%d);\n", lemp->symbols[i]->fallback->index); + }else{ + fprintf(sql, ",NULL);\n"); + } + } + fprintf(sql, + "CREATE TABLE rule(\n" + " ruleid INTEGER PRIMARY KEY,\n" + " lhs INTEGER REFERENCES symbol(id),\n" + " txt TEXT\n" + ");\n" + "CREATE TABLE rulerhs(\n" + " ruleid INTEGER REFERENCES rule(ruleid),\n" + " pos INTEGER,\n" + " sym INTEGER REFERENCES symbol(id)\n" + ");\n" + ); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + assert( i==rp->iRule ); + fprintf(sql, + "INSERT INTO rule(ruleid,lhs,txt)VALUES(%d,%d,'", + rp->iRule, rp->lhs->index + ); + writeRuleText(sql, rp); + fprintf(sql,"');\n"); + for(j=0; jnrhs; j++){ + struct symbol *sp = rp->rhs[j]; + if( sp->type!=MULTITERMINAL ){ + fprintf(sql, + "INSERT INTO rulerhs(ruleid,pos,sym)VALUES(%d,%d,%d);\n", + i,j,sp->index + ); + }else{ + int k; + for(k=0; knsubsym; k++){ + fprintf(sql, + "INSERT INTO rulerhs(ruleid,pos,sym)VALUES(%d,%d,%d);\n", + i,j,sp->subsym[k]->index + ); + } + } + } + } + fprintf(sql, "COMMIT;\n"); + } + lineno = 1; + + fprintf(out, + "/* This file is automatically generated by Lemon from input grammar\n" + "** source file \"%s\"", lemp->filename); lineno++; + if( nDefineUsed==0 ){ + fprintf(out, ".\n*/\n"); lineno += 2; + }else{ + fprintf(out, " with these options:\n**\n"); lineno += 2; + for(i=0; iinclude==0 ) lemp->include = ""; + for(i=0; ISSPACE(lemp->include[i]); i++){ + if( lemp->include[i]=='\n' ){ + lemp->include += i+1; + i = -1; + } + } + if( lemp->include[0]=='/' ){ + tplt_skip_header(in,&lineno); + }else{ + tplt_xfer(lemp->name,in,out,&lineno); + } + + /* Generate the include code, if any */ + tplt_print(out,lemp,lemp->include,&lineno); + if( mhflag ){ + char *incName = file_makename(lemp, ".h"); + fprintf(out,"#include \"%s\"\n", incName); lineno++; + lemon_free(incName); + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate #defines for all tokens */ + if( lemp->tokenprefix ) prefix = lemp->tokenprefix; + else prefix = ""; + if( mhflag ){ + fprintf(out,"#if INTERFACE\n"); lineno++; + }else{ + fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); + } + for(i=1; interminal; i++){ + fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + lineno++; + } + fprintf(out,"#endif\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the defines */ + fprintf(out,"#define YYCODETYPE %s\n", + minimum_size_type(0, lemp->nsymbol, &szCodeType)); lineno++; + fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol); lineno++; + fprintf(out,"#define YYACTIONTYPE %s\n", + minimum_size_type(0,lemp->maxAction,&szActionType)); lineno++; + if( lemp->wildcard ){ + fprintf(out,"#define YYWILDCARD %d\n", + lemp->wildcard->index); lineno++; + } + print_stack_union(out,lemp,&lineno,mhflag); + fprintf(out, "#ifndef YYSTACKDEPTH\n"); lineno++; + if( lemp->stacksize ){ + fprintf(out,"#define YYSTACKDEPTH %s\n",lemp->stacksize); lineno++; + }else{ + fprintf(out,"#define YYSTACKDEPTH 100\n"); lineno++; + } + fprintf(out, "#endif\n"); lineno++; + if( mhflag ){ + fprintf(out,"#if INTERFACE\n"); lineno++; + } + name = lemp->name ? lemp->name : "Parse"; + if( lemp->arg && lemp->arg[0] ){ + i = lemonStrlen(lemp->arg); + while( i>=1 && ISSPACE(lemp->arg[i-1]) ) i--; + while( i>=1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--; + fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++; + fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg); lineno++; + fprintf(out,"#define %sARG_PARAM ,%s\n",name,&lemp->arg[i]); lineno++; + fprintf(out,"#define %sARG_FETCH %s=yypParser->%s;\n", + name,lemp->arg,&lemp->arg[i]); lineno++; + fprintf(out,"#define %sARG_STORE yypParser->%s=%s;\n", + name,&lemp->arg[i],&lemp->arg[i]); lineno++; + }else{ + fprintf(out,"#define %sARG_SDECL\n",name); lineno++; + fprintf(out,"#define %sARG_PDECL\n",name); lineno++; + fprintf(out,"#define %sARG_PARAM\n",name); lineno++; + fprintf(out,"#define %sARG_FETCH\n",name); lineno++; + fprintf(out,"#define %sARG_STORE\n",name); lineno++; + } + if( lemp->reallocFunc ){ + fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++; + }else{ + fprintf(out,"#define YYREALLOC realloc\n"); lineno++; + } + if( lemp->freeFunc ){ + fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++; + }else{ + fprintf(out,"#define YYFREE free\n"); lineno++; + } + if( lemp->reallocFunc && lemp->freeFunc ){ + fprintf(out,"#define YYDYNSTACK 1\n"); lineno++; + }else{ + fprintf(out,"#define YYDYNSTACK 0\n"); lineno++; + } + if( lemp->ctx && lemp->ctx[0] ){ + i = lemonStrlen(lemp->ctx); + while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--; + while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--; + fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++; + fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++; + fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++; + fprintf(out,"#define %sCTX_FETCH %s=yypParser->%s;\n", + name,lemp->ctx,&lemp->ctx[i]); lineno++; + fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n", + name,&lemp->ctx[i],&lemp->ctx[i]); lineno++; + }else{ + fprintf(out,"#define %sCTX_SDECL\n",name); lineno++; + fprintf(out,"#define %sCTX_PDECL\n",name); lineno++; + fprintf(out,"#define %sCTX_PARAM\n",name); lineno++; + fprintf(out,"#define %sCTX_FETCH\n",name); lineno++; + fprintf(out,"#define %sCTX_STORE\n",name); lineno++; + } + if( mhflag ){ + fprintf(out,"#endif\n"); lineno++; + } + if( lemp->errsym && lemp->errsym->useCnt ){ + fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; + fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; + } + if( lemp->has_fallback ){ + fprintf(out,"#define YYFALLBACK 1\n"); lineno++; + } + + /* Compute the action table, but do not output it yet. The action + ** table must be computed before generating the YYNSTATE macro because + ** we need to know how many states can be eliminated. + */ + ax = (struct axset *) lemon_calloc(lemp->nxstate*2, sizeof(ax[0])); + if( ax==0 ){ + fprintf(stderr,"malloc failed\n"); + exit(1); + } + for(i=0; inxstate; i++){ + stp = lemp->sorted[i]; + ax[i*2].stp = stp; + ax[i*2].isTkn = 1; + ax[i*2].nAction = stp->nTknAct; + ax[i*2+1].stp = stp; + ax[i*2+1].isTkn = 0; + ax[i*2+1].nAction = stp->nNtAct; + } + mxTknOfst = mnTknOfst = 0; + mxNtOfst = mnNtOfst = 0; + /* In an effort to minimize the action table size, use the heuristic + ** of placing the largest action sets first */ + for(i=0; inxstate*2; i++) ax[i].iOrder = i; + qsort(ax, lemp->nxstate*2, sizeof(ax[0]), axset_compare); + pActtab = acttab_alloc(lemp->nsymbol, lemp->nterminal); + for(i=0; inxstate*2 && ax[i].nAction>0; i++){ + stp = ax[i].stp; + if( ax[i].isTkn ){ + for(ap=stp->ap; ap; ap=ap->next){ + int action; + if( ap->sp->index>=lemp->nterminal ) continue; + action = compute_action(lemp, ap); + if( action<0 ) continue; + acttab_action(pActtab, ap->sp->index, action); + } + stp->iTknOfst = acttab_insert(pActtab, 1); + if( stp->iTknOfstiTknOfst; + if( stp->iTknOfst>mxTknOfst ) mxTknOfst = stp->iTknOfst; + }else{ + for(ap=stp->ap; ap; ap=ap->next){ + int action; + if( ap->sp->indexnterminal ) continue; + if( ap->sp->index==lemp->nsymbol ) continue; + action = compute_action(lemp, ap); + if( action<0 ) continue; + acttab_action(pActtab, ap->sp->index, action); + } + stp->iNtOfst = acttab_insert(pActtab, 0); + if( stp->iNtOfstiNtOfst; + if( stp->iNtOfst>mxNtOfst ) mxNtOfst = stp->iNtOfst; + } +#if 0 /* Uncomment for a trace of how the yy_action[] table fills out */ + { int jj, nn; + for(jj=nn=0; jjnAction; jj++){ + if( pActtab->aAction[jj].action<0 ) nn++; + } + printf("%4d: State %3d %s n: %2d size: %5d freespace: %d\n", + i, stp->statenum, ax[i].isTkn ? "Token" : "Var ", + ax[i].nAction, pActtab->nAction, nn); + } +#endif + } + lemon_free(ax); + + /* Mark rules that are actually used for reduce actions after all + ** optimizations have been applied + */ + for(rp=lemp->rule; rp; rp=rp->next) rp->doesReduce = LEMON_FALSE; + for(i=0; inxstate; i++){ + for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){ + if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){ + ap->x.rp->doesReduce = 1; + } + } + } + + /* Finish rendering the constants now that the action table has + ** been computed */ + fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; + fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; + fprintf(out,"#define YYNRULE_WITH_ACTION %d\n",lemp->nruleWithAction); + lineno++; + fprintf(out,"#define YYNTOKEN %d\n",lemp->nterminal); lineno++; + fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; + i = lemp->minShiftReduce; + fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",i); lineno++; + i += lemp->nrule; + fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_ERROR_ACTION %d\n", lemp->errAction); lineno++; + fprintf(out,"#define YY_ACCEPT_ACTION %d\n", lemp->accAction); lineno++; + fprintf(out,"#define YY_NO_ACTION %d\n", lemp->noAction); lineno++; + fprintf(out,"#define YY_MIN_REDUCE %d\n", lemp->minReduce); lineno++; + i = lemp->minReduce + lemp->nrule; + fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; + + /* Minimum and maximum token values that have a destructor */ + mn = mx = 0; + for(i=0; insymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + + if( sp && sp->type!=TERMINAL && sp->destructor ){ + if( mn==0 || sp->indexindex; + if( sp->index>mx ) mx = sp->index; + } + } + if( lemp->tokendest ) mn = 0; + if( lemp->vardest ) mx = lemp->nsymbol-1; + fprintf(out,"#define YY_MIN_DSTRCTR %d\n", mn); lineno++; + fprintf(out,"#define YY_MAX_DSTRCTR %d\n", mx); lineno++; + + tplt_xfer(lemp->name,in,out,&lineno); + + /* Now output the action table and its associates: + ** + ** yy_action[] A single table containing all actions. + ** yy_lookahead[] A table containing the lookahead for each entry in + ** yy_action. Used to detect hash collisions. + ** yy_shift_ofst[] For each state, the offset into yy_action for + ** shifting terminals. + ** yy_reduce_ofst[] For each state, the offset into yy_action for + ** shifting non-terminals after a reduce. + ** yy_default[] Default action for each state. + */ + + /* Output the yy_action table */ + lemp->nactiontab = n = acttab_action_size(pActtab); + lemp->tablesize += n*szActionType; + fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++; + fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; + for(i=j=0; inoAction; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", action); + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_lookahead table */ + lemp->nlookaheadtab = n = acttab_lookahead_size(pActtab); + lemp->tablesize += n*szCodeType; + fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++; + for(i=j=0; insymbol; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", la); + if( j==9 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + /* Add extra entries to the end of the yy_lookahead[] table so that + ** yy_shift_ofst[]+iToken will always be a valid index into the array, + ** even for the largest possible value of yy_shift_ofst[] and iToken. */ + nLookAhead = lemp->nterminal + lemp->nactiontab; + while( interminal); + if( j==9 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + i++; + } + if( j>0 ){ fprintf(out, "\n"); lineno++; } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_shift_ofst[] table */ + n = lemp->nxstate; + while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; + fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; + fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; + fprintf(out, "static const %s yy_shift_ofst[] = {\n", + minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz)); + lineno++; + lemp->tablesize += n*sz; + for(i=j=0; isorted[i]; + ofst = stp->iTknOfst; + if( ofst==NO_OFFSET ) ofst = lemp->nactiontab; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", ofst); + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the yy_reduce_ofst[] table */ + n = lemp->nxstate; + while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--; + fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; + fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; + fprintf(out, "#define YY_REDUCE_MAX (%d)\n", mxNtOfst); lineno++; + fprintf(out, "static const %s yy_reduce_ofst[] = {\n", + minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++; + lemp->tablesize += n*sz; + for(i=j=0; isorted[i]; + ofst = stp->iNtOfst; + if( ofst==NO_OFFSET ) ofst = mnNtOfst - 1; + if( j==0 ) fprintf(out," /* %5d */ ", i); + fprintf(out, " %4d,", ofst); + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + + /* Output the default action table */ + fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++; + n = lemp->nxstate; + lemp->tablesize += n*szActionType; + for(i=j=0; isorted[i]; + if( j==0 ) fprintf(out," /* %5d */ ", i); + if( stp->iDfltReduce<0 ){ + fprintf(out, " %4d,", lemp->errAction); + }else{ + fprintf(out, " %4d,", stp->iDfltReduce + lemp->minReduce); + } + if( j==9 || i==n-1 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + } + fprintf(out, "};\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the table of fallback tokens. + */ + if( lemp->has_fallback ){ + mx = lemp->nterminal - 1; + /* 2019-08-28: Generate fallback entries for every token to avoid + ** having to do a range check on the index */ + /* while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } */ + lemp->tablesize += (mx+1)*szCodeType; + for(i=0; i<=mx; i++){ + struct symbol *p = lemp->symbols[i]; + if( p->fallback==0 ){ + fprintf(out, " 0, /* %10s => nothing */\n", p->name); + }else{ + fprintf(out, " %3d, /* %10s => %s */\n", p->fallback->index, + p->name, p->fallback->name); + } + lineno++; + } + } + tplt_xfer(lemp->name, in, out, &lineno); + + /* Generate a table containing the symbolic name of every symbol + */ + for(i=0; insymbol; i++){ + fprintf(out," /* %4d */ \"%s\",\n",i, lemp->symbols[i]->name); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate a table containing a text string that describes every + ** rule in the rule set of the grammar. This information is used + ** when tracing REDUCE actions. + */ + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + assert( rp->iRule==i ); + fprintf(out," /* %3d */ \"", i); + writeRuleText(out, rp); + fprintf(out,"\",\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes every time a symbol is popped from + ** the stack while processing errors or while destroying the parser. + ** (In other words, generate the %destructor actions) + */ + if( lemp->tokendest ){ + int once = 1; + for(i=0; insymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + if( sp==0 || sp->type!=TERMINAL ) continue; + if( once ){ + fprintf(out, " /* TERMINAL Destructor */\n"); lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + } + for(i=0; insymbol && lemp->symbols[i]->type!=TERMINAL; i++); + if( insymbol ){ + emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); + fprintf(out," break;\n"); lineno++; + } + } + if( lemp->vardest ){ + struct symbol *dflt_sp = 0; + int once = 1; + for(i=0; insymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + if( sp==0 || sp->type==TERMINAL || + sp->index<=0 || sp->destructor!=0 ) continue; + if( once ){ + fprintf(out, " /* Default NON-TERMINAL Destructor */\n");lineno++; + once = 0; + } + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + dflt_sp = sp; + } + if( dflt_sp!=0 ){ + emit_destructor_code(out,dflt_sp,lemp,&lineno); + } + fprintf(out," break;\n"); lineno++; + } + for(i=0; insymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + if( sp==0 || sp->type==TERMINAL || sp->destructor==0 ) continue; + if( sp->destLineno<0 ) continue; /* Already emitted */ + fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; + + /* Combine duplicate destructors into a single case */ + for(j=i+1; jnsymbol; j++){ + struct symbol *sp2 = lemp->symbols[j]; + if( sp2 && sp2->type!=TERMINAL && sp2->destructor + && sp2->dtnum==sp->dtnum + && strcmp(sp->destructor,sp2->destructor)==0 ){ + fprintf(out," case %d: /* %s */\n", + sp2->index, sp2->name); lineno++; + sp2->destLineno = -1; /* Avoid emitting this destructor again */ + } + } + + emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); + fprintf(out," break;\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes whenever the parser stack overflows */ + tplt_print(out,lemp,lemp->overflow,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate the tables of rule information. yyRuleInfoLhs[] and + ** yyRuleInfoNRhs[]. + ** + ** Note: This code depends on the fact that rules are number + ** sequentially beginning with 0. + */ + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," %4d, /* (%d) ", rp->lhs->index, i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," %3d, /* (%d) ", -rp->nrhs, i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which execution during each REDUCE action */ + i = 0; + for(rp=lemp->rule; rp; rp=rp->next){ + i += translate_code(lemp, rp); + } + if( i ){ + fprintf(out," YYMINORTYPE yylhsminor;\n"); lineno++; + } + /* First output rules other than the default: rule */ + for(rp=lemp->rule; rp; rp=rp->next){ + struct rule *rp2; /* Other rules with the same action */ + if( rp->codeEmitted ) continue; + if( rp->noCode ){ + /* No C code actions, so this will be part of the "default:" rule */ + continue; + } + fprintf(out," case %d: /* ", rp->iRule); + writeRuleText(out, rp); + fprintf(out, " */\n"); lineno++; + for(rp2=rp->next; rp2; rp2=rp2->next){ + if( rp2->code==rp->code && rp2->codePrefix==rp->codePrefix + && rp2->codeSuffix==rp->codeSuffix ){ + fprintf(out," case %d: /* ", rp2->iRule); + writeRuleText(out, rp2); + fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++; + rp2->codeEmitted = 1; + } + } + emit_code(out,rp,lemp,&lineno); + fprintf(out," break;\n"); lineno++; + rp->codeEmitted = 1; + } + /* Finally, output the default: rule. We choose as the default: all + ** empty actions. */ + fprintf(out," default:\n"); lineno++; + for(rp=lemp->rule; rp; rp=rp->next){ + if( rp->codeEmitted ) continue; + assert( rp->noCode ); + fprintf(out," /* (%d) ", rp->iRule); + writeRuleText(out, rp); + if( rp->neverReduce ){ + fprintf(out, " (NEVER REDUCES) */ assert(yyruleno!=%d);\n", + rp->iRule); lineno++; + }else if( rp->doesReduce ){ + fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; + }else{ + fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n", + rp->iRule); lineno++; + } + } + fprintf(out," break;\n"); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes if a parse fails */ + tplt_print(out,lemp,lemp->failure,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes when a syntax error occurs */ + tplt_print(out,lemp,lemp->error,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Generate code which executes when the parser accepts its input */ + tplt_print(out,lemp,lemp->accept,&lineno); + tplt_xfer(lemp->name,in,out,&lineno); + + /* Append any addition code the user desires */ + tplt_print(out,lemp,lemp->extracode,&lineno); + + acttab_free(pActtab); + fclose(in); + fclose(out); + if( sql ) fclose(sql); + return; +} + +/* Generate a header file for the parser */ +void ReportHeader(struct lemon *lemp) +{ + FILE *out, *in; + const char *prefix; + char line[LINESIZE]; + char pattern[LINESIZE]; + int i; + + if( lemp->tokenprefix ) prefix = lemp->tokenprefix; + else prefix = ""; + in = file_open(lemp,".h","rb"); + if( in ){ + int nextChar; + for(i=1; interminal && fgets(line,LINESIZE,in); i++){ + lemon_sprintf(pattern,"#define %s%-30s %3d\n", + prefix,lemp->symbols[i]->name,i); + if( strcmp(line,pattern) ) break; + } + nextChar = fgetc(in); + fclose(in); + if( i==lemp->nterminal && nextChar==EOF ){ + /* No change in the file. Don't rewrite it. */ + return; + } + } + out = file_open(lemp,".h","wb"); + if( out ){ + for(i=1; interminal; i++){ + fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); + } + fclose(out); + } + return; +} + +/* Reduce the size of the action tables, if possible, by making use +** of defaults. +** +** In this version, we take the most frequent REDUCE action and make +** it the default. Except, there is no default if the wildcard token +** is a possible look-ahead. +*/ +void CompressTables(struct lemon *lemp) +{ + struct state *stp; + struct action *ap, *ap2, *nextap; + struct rule *rp, *rp2, *rbest; + int nbest, n; + int i; + int usesWildcard; + + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + nbest = 0; + rbest = 0; + usesWildcard = 0; + + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==SHIFT && ap->sp==lemp->wildcard ){ + usesWildcard = 1; + } + if( ap->type!=REDUCE ) continue; + rp = ap->x.rp; + if( rp->lhsStart ) continue; + if( rp==rbest ) continue; + n = 1; + for(ap2=ap->next; ap2; ap2=ap2->next){ + if( ap2->type!=REDUCE ) continue; + rp2 = ap2->x.rp; + if( rp2==rbest ) continue; + if( rp2==rp ) n++; + } + if( n>nbest ){ + nbest = n; + rbest = rp; + } + } + + /* Do not make a default if the number of rules to default + ** is not at least 1 or if the wildcard token is a possible + ** lookahead. + */ + if( nbest<1 || usesWildcard ) continue; + + + /* Combine matching REDUCE actions into a single default */ + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==REDUCE && ap->x.rp==rbest ) break; + } + assert( ap ); + ap->sp = Symbol_new("{default}"); + for(ap=ap->next; ap; ap=ap->next){ + if( ap->type==REDUCE && ap->x.rp==rbest ) ap->type = NOT_USED; + } + stp->ap = Action_sort(stp->ap); + + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==SHIFT ) break; + if( ap->type==REDUCE && ap->x.rp!=rbest ) break; + } + if( ap==0 ){ + stp->autoReduce = 1; + stp->pDfltReduce = rbest; + } + } + + /* Make a second pass over all states and actions. Convert + ** every action that is a SHIFT to an autoReduce state into + ** a SHIFTREDUCE action. + */ + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=ap->next){ + struct state *pNextState; + if( ap->type!=SHIFT ) continue; + pNextState = ap->x.stp; + if( pNextState->autoReduce && pNextState->pDfltReduce!=0 ){ + ap->type = SHIFTREDUCE; + ap->x.rp = pNextState->pDfltReduce; + } + } + } + + /* If a SHIFTREDUCE action specifies a rule that has a single RHS term + ** (meaning that the SHIFTREDUCE will land back in the state where it + ** started) and if there is no C-code associated with the reduce action, + ** then we can go ahead and convert the action to be the same as the + ** action for the RHS of the rule. + */ + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=nextap){ + nextap = ap->next; + if( ap->type!=SHIFTREDUCE ) continue; + rp = ap->x.rp; + if( rp->noCode==0 ) continue; + if( rp->nrhs!=1 ) continue; +#if 1 + /* Only apply this optimization to non-terminals. It would be OK to + ** apply it to terminal symbols too, but that makes the parser tables + ** larger. */ + if( ap->sp->indexnterminal ) continue; +#endif + /* If we reach this point, it means the optimization can be applied */ + nextap = ap; + for(ap2=stp->ap; ap2 && (ap2==ap || ap2->sp!=rp->lhs); ap2=ap2->next){} + assert( ap2!=0 ); + ap->spOpt = ap2->sp; + ap->type = ap2->type; + ap->x = ap2->x; + } + } +} + + +/* +** Compare two states for sorting purposes. The smaller state is the +** one with the most non-terminal actions. If they have the same number +** of non-terminal actions, then the smaller is the one with the most +** token actions. +*/ +static int stateResortCompare(const void *a, const void *b){ + const struct state *pA = *(const struct state**)a; + const struct state *pB = *(const struct state**)b; + int n; + + n = pB->nNtAct - pA->nNtAct; + if( n==0 ){ + n = pB->nTknAct - pA->nTknAct; + if( n==0 ){ + n = pB->statenum - pA->statenum; + } + } + assert( n!=0 ); + return n; +} + + +/* +** Renumber and resort states so that states with fewer choices +** occur at the end. Except, keep state 0 as the first state. +*/ +void ResortStates(struct lemon *lemp) +{ + int i; + struct state *stp; + struct action *ap; + + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + stp->nTknAct = stp->nNtAct = 0; + stp->iDfltReduce = -1; /* Init dflt action to "syntax error" */ + stp->iTknOfst = NO_OFFSET; + stp->iNtOfst = NO_OFFSET; + for(ap=stp->ap; ap; ap=ap->next){ + int iAction = compute_action(lemp,ap); + if( iAction>=0 ){ + if( ap->sp->indexnterminal ){ + stp->nTknAct++; + }else if( ap->sp->indexnsymbol ){ + stp->nNtAct++; + }else{ + assert( stp->autoReduce==0 || stp->pDfltReduce==ap->x.rp ); + stp->iDfltReduce = iAction; + } + } + } + } + qsort(&lemp->sorted[1], lemp->nstate-1, sizeof(lemp->sorted[0]), + stateResortCompare); + for(i=0; instate; i++){ + lemp->sorted[i]->statenum = i; + } + lemp->nxstate = lemp->nstate; + while( lemp->nxstate>1 && lemp->sorted[lemp->nxstate-1]->autoReduce ){ + lemp->nxstate--; + } +} + + +/***************** From the file "set.c" ************************************/ +/* +** Set manipulation routines for the LEMON parser generator. +*/ + +static int size = 0; + +/* Set the set size */ +void SetSize(int n) +{ + size = n+1; +} + +/* Allocate a new set */ +char *SetNew(void){ + char *s; + s = (char*)lemon_calloc( size, 1); + if( s==0 ){ + memory_error(); + } + return s; +} + +/* Deallocate a set */ +void SetFree(char *s) +{ + lemon_free(s); +} + +/* Add a new element to the set. Return TRUE if the element was added +** and FALSE if it was already there. */ +int SetAdd(char *s, int e) +{ + int rv; + assert( e>=0 && esize = 1024; + x1a->count = 0; + x1a->tbl = (x1node*)lemon_calloc(1024, sizeof(x1node) + sizeof(x1node*)); + if( x1a->tbl==0 ){ + lemon_free(x1a); + x1a = 0; + }else{ + int i; + x1a->ht = (x1node**)&(x1a->tbl[1024]); + for(i=0; i<1024; i++) x1a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Strsafe_insert(const char *data) +{ + x1node *np; + unsigned h; + unsigned ph; + + if( x1a==0 ) return 0; + ph = strhash(data); + h = ph & (x1a->size-1); + np = x1a->ht[h]; + while( np ){ + if( strcmp(np->data,data)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x1a->count>=x1a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x1 array; + array.size = arrSize = x1a->size*2; + array.count = x1a->count; + array.tbl = (x1node*)lemon_calloc(arrSize, sizeof(x1node)+sizeof(x1node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x1node**)&(array.tbl[arrSize]); + for(i=0; icount; i++){ + x1node *oldnp, *newnp; + oldnp = &(x1a->tbl[i]); + h = strhash(oldnp->data) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + /* lemon_free(x1a->tbl); // This program was originally for 16-bit machines. + ** Don't worry about freeing memory on modern platforms. */ + *x1a = array; + } + /* Insert the new data */ + h = ph & (x1a->size-1); + np = &(x1a->tbl[x1a->count++]); + np->data = data; + if( x1a->ht[h] ) x1a->ht[h]->from = &(np->next); + np->next = x1a->ht[h]; + x1a->ht[h] = np; + np->from = &(x1a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +const char *Strsafe_find(const char *key) +{ + unsigned h; + x1node *np; + + if( x1a==0 ) return 0; + h = strhash(key) & (x1a->size-1); + np = x1a->ht[h]; + while( np ){ + if( strcmp(np->data,key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return a pointer to the (terminal or nonterminal) symbol "x". +** Create a new symbol if this is the first time "x" has been seen. +*/ +struct symbol *Symbol_new(const char *x) +{ + struct symbol *sp; + + sp = Symbol_find(x); + if( sp==0 ){ + sp = (struct symbol *)lemon_calloc(1, sizeof(struct symbol) ); + MemoryCheck(sp); + sp->name = Strsafe(x); + sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL; + sp->rule = 0; + sp->fallback = 0; + sp->prec = -1; + sp->assoc = UNK; + sp->firstset = 0; + sp->lambda = LEMON_FALSE; + sp->destructor = 0; + sp->destLineno = 0; + sp->datatype = 0; + sp->useCnt = 0; + Symbol_insert(sp,sp->name); + } + sp->useCnt++; + return sp; +} + +/* Compare two symbols for sorting purposes. Return negative, +** zero, or positive if a is less then, equal to, or greater +** than b. +** +** Symbols that begin with upper case letters (terminals or tokens) +** must sort before symbols that begin with lower case letters +** (non-terminals). And MULTITERMINAL symbols (created using the +** %token_class directive) must sort at the very end. Other than +** that, the order does not matter. +** +** We find experimentally that leaving the symbols in their original +** order (the order they appeared in the grammar file) gives the +** smallest parser tables in SQLite. +*/ +int Symbolcmpp(const void *_a, const void *_b) +{ + const struct symbol *a = *(const struct symbol **) _a; + const struct symbol *b = *(const struct symbol **) _b; + int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1; + int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1; + return i1==i2 ? a->index - b->index : i1 - i2; +} + +/* There is one instance of the following structure for each +** associative array of type "x2". +*/ +struct s_x2 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x2node *tbl; /* The data stored here */ + struct s_x2node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x2". +*/ +typedef struct s_x2node { + struct symbol *data; /* The data */ + const char *key; /* The key */ + struct s_x2node *next; /* Next entry with the same hash */ + struct s_x2node **from; /* Previous link */ +} x2node; + +/* There is only one instance of the array, which is the following */ +static struct s_x2 *x2a; + +/* Allocate a new associative array */ +void Symbol_init(void){ + if( x2a ) return; + x2a = (struct s_x2*)lemon_malloc( sizeof(struct s_x2) ); + if( x2a ){ + x2a->size = 128; + x2a->count = 0; + x2a->tbl = (x2node*)lemon_calloc(128, sizeof(x2node) + sizeof(x2node*)); + if( x2a->tbl==0 ){ + lemon_free(x2a); + x2a = 0; + }else{ + int i; + x2a->ht = (x2node**)&(x2a->tbl[128]); + for(i=0; i<128; i++) x2a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Symbol_insert(struct symbol *data, const char *key) +{ + x2node *np; + unsigned h; + unsigned ph; + + if( x2a==0 ) return 0; + ph = strhash(key); + h = ph & (x2a->size-1); + np = x2a->ht[h]; + while( np ){ + if( strcmp(np->key,key)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x2a->count>=x2a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x2 array; + array.size = arrSize = x2a->size*2; + array.count = x2a->count; + array.tbl = (x2node*)lemon_calloc(arrSize, sizeof(x2node)+sizeof(x2node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x2node**)&(array.tbl[arrSize]); + for(i=0; icount; i++){ + x2node *oldnp, *newnp; + oldnp = &(x2a->tbl[i]); + h = strhash(oldnp->key) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->key = oldnp->key; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + /* lemon_free(x2a->tbl); // This program was originally written for 16-bit + ** machines. Don't worry about freeing this trivial amount of memory + ** on modern platforms. Just leak it. */ + *x2a = array; + } + /* Insert the new data */ + h = ph & (x2a->size-1); + np = &(x2a->tbl[x2a->count++]); + np->key = key; + np->data = data; + if( x2a->ht[h] ) x2a->ht[h]->from = &(np->next); + np->next = x2a->ht[h]; + x2a->ht[h] = np; + np->from = &(x2a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct symbol *Symbol_find(const char *key) +{ + unsigned h; + x2node *np; + + if( x2a==0 ) return 0; + h = strhash(key) & (x2a->size-1); + np = x2a->ht[h]; + while( np ){ + if( strcmp(np->key,key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return the n-th data. Return NULL if n is out of range. */ +struct symbol *Symbol_Nth(int n) +{ + struct symbol *data; + if( x2a && n>0 && n<=x2a->count ){ + data = x2a->tbl[n-1].data; + }else{ + data = 0; + } + return data; +} + +/* Return the size of the array */ +int Symbol_count() +{ + return x2a ? x2a->count : 0; +} + +/* Return an array of pointers to all data in the table. +** The array is obtained from malloc. Return NULL if memory allocation +** problems, or if the array is empty. */ +struct symbol **Symbol_arrayof() +{ + struct symbol **array; + int i,arrSize; + if( x2a==0 ) return 0; + arrSize = x2a->count; + array = (struct symbol **)lemon_calloc(arrSize, sizeof(struct symbol *)); + if( array ){ + for(i=0; itbl[i].data; + } + return array; +} + +/* Compare two configurations */ +int Configcmp(const char *_a,const char *_b) +{ + const struct config *a = (struct config *) _a; + const struct config *b = (struct config *) _b; + int x; + x = a->rp->index - b->rp->index; + if( x==0 ) x = a->dot - b->dot; + return x; +} + +/* Compare two states */ +PRIVATE int statecmp(struct config *a, struct config *b) +{ + int rc; + for(rc=0; rc==0 && a && b; a=a->bp, b=b->bp){ + rc = a->rp->index - b->rp->index; + if( rc==0 ) rc = a->dot - b->dot; + } + if( rc==0 ){ + if( a ) rc = 1; + if( b ) rc = -1; + } + return rc; +} + +/* Hash a state */ +PRIVATE unsigned statehash(struct config *a) +{ + unsigned h=0; + while( a ){ + h = h*571 + a->rp->index*37 + a->dot; + a = a->bp; + } + return h; +} + +/* Allocate a new state structure */ +struct state *State_new() +{ + struct state *newstate; + newstate = (struct state *)lemon_calloc(1, sizeof(struct state) ); + MemoryCheck(newstate); + return newstate; +} + +/* There is one instance of the following structure for each +** associative array of type "x3". +*/ +struct s_x3 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x3node *tbl; /* The data stored here */ + struct s_x3node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x3". +*/ +typedef struct s_x3node { + struct state *data; /* The data */ + struct config *key; /* The key */ + struct s_x3node *next; /* Next entry with the same hash */ + struct s_x3node **from; /* Previous link */ +} x3node; + +/* There is only one instance of the array, which is the following */ +static struct s_x3 *x3a; + +/* Allocate a new associative array */ +void State_init(void){ + if( x3a ) return; + x3a = (struct s_x3*)lemon_malloc( sizeof(struct s_x3) ); + if( x3a ){ + x3a->size = 128; + x3a->count = 0; + x3a->tbl = (x3node*)lemon_calloc(128, sizeof(x3node) + sizeof(x3node*)); + if( x3a->tbl==0 ){ + lemon_free(x3a); + x3a = 0; + }else{ + int i; + x3a->ht = (x3node**)&(x3a->tbl[128]); + for(i=0; i<128; i++) x3a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int State_insert(struct state *data, struct config *key) +{ + x3node *np; + unsigned h; + unsigned ph; + + if( x3a==0 ) return 0; + ph = statehash(key); + h = ph & (x3a->size-1); + np = x3a->ht[h]; + while( np ){ + if( statecmp(np->key,key)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x3a->count>=x3a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x3 array; + array.size = arrSize = x3a->size*2; + array.count = x3a->count; + array.tbl = (x3node*)lemon_calloc(arrSize, sizeof(x3node)+sizeof(x3node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x3node**)&(array.tbl[arrSize]); + for(i=0; icount; i++){ + x3node *oldnp, *newnp; + oldnp = &(x3a->tbl[i]); + h = statehash(oldnp->key) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->key = oldnp->key; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + lemon_free(x3a->tbl); + *x3a = array; + } + /* Insert the new data */ + h = ph & (x3a->size-1); + np = &(x3a->tbl[x3a->count++]); + np->key = key; + np->data = data; + if( x3a->ht[h] ) x3a->ht[h]->from = &(np->next); + np->next = x3a->ht[h]; + x3a->ht[h] = np; + np->from = &(x3a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct state *State_find(struct config *key) +{ + unsigned h; + x3node *np; + + if( x3a==0 ) return 0; + h = statehash(key) & (x3a->size-1); + np = x3a->ht[h]; + while( np ){ + if( statecmp(np->key,key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Return an array of pointers to all data in the table. +** The array is obtained from malloc. Return NULL if memory allocation +** problems, or if the array is empty. */ +struct state **State_arrayof(void) +{ + struct state **array; + int i,arrSize; + if( x3a==0 ) return 0; + arrSize = x3a->count; + array = (struct state **)lemon_calloc(arrSize, sizeof(struct state *)); + if( array ){ + for(i=0; itbl[i].data; + } + return array; +} + +/* Hash a configuration */ +PRIVATE unsigned confighash(struct config *a) +{ + unsigned h=0; + h = h*571 + a->rp->index*37 + a->dot; + return h; +} + +/* There is one instance of the following structure for each +** associative array of type "x4". +*/ +struct s_x4 { + int size; /* The number of available slots. */ + /* Must be a power of 2 greater than or */ + /* equal to 1 */ + int count; /* Number of currently slots filled */ + struct s_x4node *tbl; /* The data stored here */ + struct s_x4node **ht; /* Hash table for lookups */ +}; + +/* There is one instance of this structure for every data element +** in an associative array of type "x4". +*/ +typedef struct s_x4node { + struct config *data; /* The data */ + struct s_x4node *next; /* Next entry with the same hash */ + struct s_x4node **from; /* Previous link */ +} x4node; + +/* There is only one instance of the array, which is the following */ +static struct s_x4 *x4a; + +/* Allocate a new associative array */ +void Configtable_init(void){ + if( x4a ) return; + x4a = (struct s_x4*)lemon_malloc( sizeof(struct s_x4) ); + if( x4a ){ + x4a->size = 64; + x4a->count = 0; + x4a->tbl = (x4node*)lemon_calloc(64, sizeof(x4node) + sizeof(x4node*)); + if( x4a->tbl==0 ){ + lemon_free(x4a); + x4a = 0; + }else{ + int i; + x4a->ht = (x4node**)&(x4a->tbl[64]); + for(i=0; i<64; i++) x4a->ht[i] = 0; + } + } +} +/* Insert a new record into the array. Return TRUE if successful. +** Prior data with the same key is NOT overwritten */ +int Configtable_insert(struct config *data) +{ + x4node *np; + unsigned h; + unsigned ph; + + if( x4a==0 ) return 0; + ph = confighash(data); + h = ph & (x4a->size-1); + np = x4a->ht[h]; + while( np ){ + if( Configcmp((const char *) np->data,(const char *) data)==0 ){ + /* An existing entry with the same key is found. */ + /* Fail because overwrite is not allows. */ + return 0; + } + np = np->next; + } + if( x4a->count>=x4a->size ){ + /* Need to make the hash table bigger */ + int i,arrSize; + struct s_x4 array; + array.size = arrSize = x4a->size*2; + array.count = x4a->count; + array.tbl = (x4node*)lemon_calloc(arrSize, + sizeof(x4node) + sizeof(x4node*)); + if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ + array.ht = (x4node**)&(array.tbl[arrSize]); + for(i=0; icount; i++){ + x4node *oldnp, *newnp; + oldnp = &(x4a->tbl[i]); + h = confighash(oldnp->data) & (arrSize-1); + newnp = &(array.tbl[i]); + if( array.ht[h] ) array.ht[h]->from = &(newnp->next); + newnp->next = array.ht[h]; + newnp->data = oldnp->data; + newnp->from = &(array.ht[h]); + array.ht[h] = newnp; + } + *x4a = array; + } + /* Insert the new data */ + h = ph & (x4a->size-1); + np = &(x4a->tbl[x4a->count++]); + np->data = data; + if( x4a->ht[h] ) x4a->ht[h]->from = &(np->next); + np->next = x4a->ht[h]; + x4a->ht[h] = np; + np->from = &(x4a->ht[h]); + return 1; +} + +/* Return a pointer to data assigned to the given key. Return NULL +** if no such key. */ +struct config *Configtable_find(struct config *key) +{ + int h; + x4node *np; + + if( x4a==0 ) return 0; + h = confighash(key) & (x4a->size-1); + np = x4a->ht[h]; + while( np ){ + if( Configcmp((const char *) np->data,(const char *) key)==0 ) break; + np = np->next; + } + return np ? np->data : 0; +} + +/* Remove all data from the table. Pass each data to the function "f" +** as it is removed. ("f" may be null to avoid this step.) */ +void Configtable_clear(int(*f)(struct config *)) +{ + int i; + if( x4a==0 || x4a->count==0 ) return; + if( f ) for(i=0; icount; i++) (*f)(x4a->tbl[i].data); + for(i=0; isize; i++) x4a->ht[i] = 0; + x4a->count = 0; + return; +} diff --git a/contrib/lemon/lempar.c b/contrib/lemon/lempar.c new file mode 100644 index 0000000000..851a0e2e54 --- /dev/null +++ b/contrib/lemon/lempar.c @@ -0,0 +1,1086 @@ +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: +*/ +/************ Begin %include sections from the grammar ************************/ +%% +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +%% +/**************** End token definitions ***************************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** ParseTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is ParseTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** ParseARG_SDECL A static variable declaration for the %extra_argument +** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter +** ParseARG_STORE Code to store %extra_argument into yypParser +** ParseARG_FETCH Code to extract %extra_argument from yypParser +** ParseCTX_* As ParseARG_ except for %extra_context +** YYREALLOC Name of the realloc() function to use +** YYFREE Name of the free() function to use +** YYDYNSTACK True if stack space should be extended on heap +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +** YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** YY_MAX_DSTRCTR Maximum symbol value that has a destructor +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +%% +/************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if YYSTACKDEPTH<=0 || YYDYNSTACK +# define YYGROWABLESTACK 1 +#else +# define YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if YYSTACKDEPTH<=0 +# undef YYSTACKDEPTH +# define YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +%% +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { +%% +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + yyStackEntry *yytos; /* Pointer to top element of the stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyhwm; /* High-water mark of the stack */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + ParseARG_SDECL /* A place to hold %extra_argument */ + ParseCTX_SDECL /* A place to hold %extra_context */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ + yyStackEntry *yystack; /* The parser stack */ + yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */ +}; +typedef struct yyParser yyParser; + +#include +#ifndef NDEBUG +#include +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
    +**
  • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
  • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
+** +** Outputs: +** None. +*/ +void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { +%% +}; +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { +%% +}; +#endif /* NDEBUG */ + + +#if YYGROWABLESTACK +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int yyGrowStack(yyParser *p){ + int oldSize = 1 + (int)(p->yystackEnd - p->yystack); + int newSize; + int idx; + yyStackEntry *pNew; + + newSize = oldSize*2 + 100; + idx = (int)(p->yytos - p->yystack); + if( p->yystack==p->yystk0 ){ + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); + }else{ + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + } + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, oldSize, newSize); + } +#endif + p->yystackEnd = &p->yystack[newSize-1]; + return 0; +} +#endif /* YYGROWABLESTACK */ + +#if !YYGROWABLESTACK +/* For builds that do no have a growable stack, yyGrowStack always +** returns an error. +*/ +# define yyGrowStack(X) 1 +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to ParseAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +void ParseInit(void *yypRawParser ParseCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + ParseCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif + yypParser->yystack = yypParser->yystk0; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +} + +#ifndef Parse_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to Parse and ParseFree. +*/ +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + ParseCTX_STORE + ParseInit(yypParser ParseCTX_PARAM); + } + return (void*)yypParser; +} +#endif /* Parse_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + ParseARG_FETCH + ParseCTX_FETCH + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ +%% +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +void ParseFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + + /* In-lined version of calling yy_pop_parser_stack() for each + ** element left in the stack */ + yyStackEntry *yytos = pParser->yytos; + while( yytos>pParser->yystack ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + if( yytos->major>=YY_MIN_DSTRCTR ){ + yy_destructor(pParser, yytos->major, &yytos->minor); + } + yytos--; + } + +#if YYGROWABLESTACK + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); +#endif +} + +#ifndef Parse_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +void ParseFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + ParseFinalize(p); + (*freeProc)(p); +} +#endif /* Parse_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int ParseStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +int ParseCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = yy_shift_ofst[stateno]; + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); + i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + assert( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + assert( i>=0 && i<(int)(sizeof(yy_action)/sizeof(yy_action[0])) ); + return yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ +%% +/******** End %stack_overflow code ********************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument var */ + ParseCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ + if( yyTraceFILE ){ + if( yyNewStateyytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); + } + } +} +#else +# define yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ + ParseTOKENTYPE yyMinor /* The minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yytos++; +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + } +#endif + yytos = yypParser->yytos; + if( yytos>yypParser->yystackEnd ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } + yytos = yypParser->yytos; + assert( yytos <= yypParser->yystackEnd ); + } + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos->stateno = yyNewState; + yytos->major = yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState, "Shift"); +} + +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { +%% +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { +%% +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static YYACTIONTYPE yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + ParseCTX_PDECL /* %extra_context */ +){ + int yygoto; /* The next state */ + YYACTIONTYPE yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + ParseARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ +%% +/********** End reduce actions ************************************************/ + }; + assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + ParseARG_FETCH + ParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +%% +/************ End %parse_failure code *****************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + ParseTOKENTYPE yyminor /* The minor type of the error token */ +){ + ParseARG_FETCH + ParseCTX_FETCH +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ +%% +/************ End %syntax_error code ******************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + ParseARG_FETCH + ParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +%% +/*********** End %parse_accept code *******************************************/ + ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + ParseCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "ParseAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
    +**
  • A pointer to the parser (an opaque structure.) +**
  • The major token number. +**
  • The minor token number. +**
  • An option argument of a grammar-specified type. +**
+** +** Outputs: +** None. +*/ +void Parse( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + ParseTOKENTYPE yyminor /* The value for the token */ + ParseARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + YYACTIONTYPE yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + ParseCTX_FETCH + ParseARG_STORE + + assert( yypParser->yytos!=0 ); +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + + yyact = yypParser->yytos->stateno; +#ifndef NDEBUG + if( yyTraceFILE ){ + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } + } +#endif + + while(1){ /* Exit by "break" */ + assert( yypParser->yytos>=yypParser->yystack ); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */ +#ifndef NDEBUG + assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); + if( yyTraceFILE ){ + int yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == + (int)(yypParser->yytos - yypParser->yystack)); + } +#endif + if( yypParser->yytos>=yypParser->yystackEnd ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + break; + } + } + } + yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor ParseCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; + }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yytos->major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + }else{ + while( yypParser->yytos > yypParser->yystack ){ + yyact = yy_find_reduce_action(yypParser->yytos->stateno, + YYERRORSYMBOL); + if( yyact<=YY_MAX_SHIFTREDUCE ) break; + yy_pop_parser_stack(yypParser); + } + if( yypParser->yytos <= yypParser->yystack || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + break; +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + } + break; +#endif + } + } +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int ParseFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} From 29b0acd0b31754d44cf8d7b096ad31afe9a8f5e6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 13 Nov 2024 16:41:15 +0800 Subject: [PATCH 005/108] fix(stream): set correct error code and open inputq for upstream . --- source/dnode/mnode/impl/src/mndStreamHb.c | 2 +- source/libs/stream/src/streamCheckpoint.c | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 0f903632ed..46445af856 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -498,7 +498,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { continue; } - mInfo("stream:0x%" PRIx64 "checkpointId:%" PRId64 + mInfo("stream:0x%" PRIx64 " checkpointId:%" PRId64 " transId:%d failed issue task-reset trans to reset all tasks status", pInfo->streamUid, pInfo->checkpointId, pInfo->transId); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index cc92df368c..a8a934da98 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -255,8 +255,7 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check "the interrupted checkpoint", id, vgId, pBlock->srcTaskId); - streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) { @@ -264,14 +263,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 " discard", id, vgId, pActiveInfo->activeId, checkpointId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } else { // checkpointId == pActiveInfo->activeId if (pActiveInfo->allUpstreamTriggerRecv == 1) { stDebug( "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "checkpointId:%" PRId64 " transId:%d", id, vgId, checkpointId, transId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { @@ -283,17 +282,17 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check } if (p->upstreamTaskId == pBlock->srcTaskId) { - stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 + stWarn("s-task:%s repeatly recv checkpoint-trigger msg from task:0x%x vgId:%d, checkpointId:%" PRId64 ", prev recvTs:%" PRId64 " discard", pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } } } } } - return 0; + return TSDB_CODE_SUCCESS; } int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { @@ -317,6 +316,9 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock code = doCheckBeforeHandleChkptTrigger(pTask, checkpointId, pBlock, transId); streamMutexUnlock(&pTask->lock); if (code) { + if (taskLevel != TASK_LEVEL__SOURCE) { // the checkpoint-trigger is discard, open the inputQ for upstream tasks + streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); + } streamFreeQitem((SStreamQueueItem*)pBlock); return code; } From 46638731506101d1b19f201451ede454835d1760 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 13 Nov 2024 16:47:05 +0800 Subject: [PATCH 006/108] fix(stream): update the checkpoint report info in SCheckpointReport --- source/dnode/mnode/impl/src/mndStream.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 81db427afd..6336cd6e49 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2434,7 +2434,12 @@ static void doAddReportStreamTask(SArray *pList, int64_t reportChkptId, const SC mDebug("s-task:0x%x expired checkpoint-report msg in checkpoint-report list update from %" PRId64 "->%" PRId64, pReport->taskId, p->checkpointId, pReport->checkpointId); - memcpy(p, pReport, sizeof(STaskChkptInfo)); + // update the checkpoint report info + p->checkpointId = pReport->checkpointId; + p->ts = pReport->checkpointTs; + p->version = pReport->checkpointVer; + p->transId = pReport->transId; + p->dropHTask = pReport->dropHTask; } else { mWarn("taskId:0x%x already in checkpoint-report list", pReport->taskId); } From 141b1c5d5a8917649cdd834e228008dbb8b00fea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 13 Nov 2024 19:11:18 +0800 Subject: [PATCH 007/108] fix(test): update test cases. --- tests/script/tsim/db/basic1.sim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 8eb6dce759..f3239957d3 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -53,6 +53,8 @@ if $rows != 5 then return -1 endi +sleep 500 + print =============== show vgroups2 sql show d2.vgroups if $rows != 2 then @@ -126,13 +128,14 @@ if $data12 != d2 then endi if $data13 != leader then + print expect leader , actual $13 return -1 endi -print $data14 -print $data15 +print $data14 , $data15 if $data16 != 1 then + print expect 1, acutal $data16 return -1 endi From 7cad1db36183bcaf7a29011dd92963dda2065ef9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Nov 2024 19:22:37 +0800 Subject: [PATCH 008/108] fix(stream): remove failed task in hash table and array list. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/libs/stream/src/streamMeta.c | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 74706b5c5e..79d7a231b8 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -652,7 +652,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve streamMetaWUnLock(pMeta); if (code < 0) { - tqError("failed to add s-task:0x%x into vgId:%d meta, existed:%d, code:%s", vgId, taskId, numOfTasks, + tqError("vgId:%d failed to register s-task:0x%x into meta, existed tasks:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); return code; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 86f305df60..a6f87711bf 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -723,8 +723,9 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa pTask->id.refId = refId = taosAddRef(streamTaskRefPool, pTask); code = taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask->id.refId, sizeof(int64_t)); - if (code) { // todo remove it from task list + if (code) { stError("s-task:0x%" PRIx64 " failed to register task into meta-list, code: out of memory", id.taskId); + void* pUnused = taosArrayPop(pMeta->pTaskList); int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret != 0) { @@ -734,6 +735,9 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa } if ((code = streamMetaSaveTask(pMeta, pTask)) != 0) { + int32_t unused = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); + void* pUnused = taosArrayPop(pMeta->pTaskList); + int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret) { stError("vgId:%d remove task refId failed, refId:%" PRId64, pMeta->vgId, refId); @@ -742,6 +746,9 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa } if ((code = streamMetaCommit(pMeta)) != 0) { + int32_t unused = taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); + void* pUnused = taosArrayPop(pMeta->pTaskList); + int32_t ret = taosRemoveRef(streamTaskRefPool, refId); if (ret) { stError("vgId:%d remove task refId failed, refId:%" PRId64, pMeta->vgId, refId); From 448647dff28a27a6ef02441d8bcdcd931a5ecd92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Nov 2024 02:12:15 +0800 Subject: [PATCH 009/108] fix(stream): check failedId before sending checkpoint msg --- include/libs/stream/tstream.h | 2 +- source/libs/stream/src/streamCheckpoint.c | 41 ++++++++++++----------- source/libs/stream/src/streamDispatch.c | 10 +++++- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index caae143cc7..83ecb1114d 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -714,7 +714,7 @@ int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeChec void streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId); bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId); void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal); -int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask); +int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask, int64_t sendingChkptId); void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId); int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, int32_t downstreamNodeId, SRpcHandleInfo* pInfo, int32_t code); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index a8a934da98..996f12c4a9 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -1215,34 +1215,37 @@ void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_ // record the dispatch checkpoint trigger info in the list // memory insufficient may cause the stream computing stopped -int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { +int32_t streamTaskInitTriggerDispatchInfo(SStreamTask* pTask, int64_t sendingChkptId) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int64_t now = taosGetTimestampMs(); int32_t code = 0; streamMutexLock(&pInfo->lock); - pInfo->dispatchTrigger = true; - if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { - STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; + if (sendingChkptId > pInfo->failedId) { + pInfo->dispatchTrigger = true; + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; - STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; - void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); - if (px == NULL) { // pause the stream task, if memory not enough - code = terrno; - } - } else { - for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { - SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i); - if (pVgInfo == NULL) { - continue; - } - - STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; - void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); + STaskTriggerSendInfo p = { + .sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; + void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); if (px == NULL) { // pause the stream task, if memory not enough code = terrno; - break; + } + } else { + for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { + SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i); + if (pVgInfo == NULL) { + continue; + } + + STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; + void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p); + if (px == NULL) { // pause the stream task, if memory not enough + code = terrno; + break; + } } } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e0fa199199..c6ef99e76e 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -845,7 +845,15 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { streamMutexUnlock(&pTask->msgInfo.lock); code = doBuildDispatchMsg(pTask, pBlock); + + int64_t chkptId = 0; if (code == 0) { + if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { + SSDataBlock* p = taosArrayGet(pBlock->blocks, 0); + if (pBlock != NULL) { + chkptId = p->info.version; + } + } destroyStreamDataBlock(pBlock); } else { // todo handle build dispatch msg failed } @@ -862,7 +870,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { continue; } - code = streamTaskInitTriggerDispatchInfo(pTask); + code = streamTaskInitTriggerDispatchInfo(pTask, chkptId); if (code != TSDB_CODE_SUCCESS) { // todo handle error } } From aa12038fc49fe35eb34e851c92591dc9a4a6e455 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 10 Dec 2024 17:58:53 +0800 Subject: [PATCH 010/108] fix(stream): record the failure of dispatch msg, and set the update node id. --- include/libs/stream/tstream.h | 1 - source/libs/stream/inc/streamInt.h | 1 + source/libs/stream/src/streamCheckStatus.c | 16 +++++++--------- source/libs/stream/src/streamDispatch.c | 7 ++++++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 53b8e0e0b9..295aa770a9 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -207,7 +207,6 @@ typedef struct { typedef struct { int32_t nodeId; - SEpSet epset; } SDownstreamTaskEpset; typedef enum { diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 8f9e4a311c..8f68116079 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -201,6 +201,7 @@ void streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo); void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo); int32_t streamTaskResetTimewindowFilter(SStreamTask* pTask); void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo); +int32_t streamTaskAddIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); void streamClearChkptReadyMsg(SActiveCheckpointInfo* pActiveInfo); EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 64b19e4ed9..118cb1cfb6 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -21,7 +21,6 @@ #define CHECK_NOT_RSP_DURATION 10 * 1000 // 10 sec static void processDownstreamReadyRsp(SStreamTask* pTask); -static int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); static void rspMonitorFn(void* param, void* tmrId); static void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs); static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id); @@ -226,13 +225,13 @@ int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* stError("s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%" PRId64 ", current stage:%" PRId64 ", not check wait for downstream task nodeUpdate, and all tasks restart", id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); - code = addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); + code = streamTaskAddIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); } else { stError( "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check " "downstream again, nodeUpdate needed", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); - code = addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); + code = streamTaskAddIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); } streamMetaAddFailedTaskSelf(pTask, now); @@ -373,11 +372,10 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { } } -int32_t addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { +int32_t streamTaskAddIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t vgId = pTask->pMeta->vgId; int32_t code = 0; - ; - bool existed = false; + bool existed = false; streamMutexLock(&pTask->lock); @@ -675,8 +673,8 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { SDownstreamStatusInfo* p = NULL; findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { - code = addIntoNodeUpdateList(pTask, p->vgId); - stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpate list", + code = streamTaskAddIntoNodeUpdateList(pTask, p->vgId); + stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpdate list", id, vgId, p->taskId, p->vgId); } } @@ -717,7 +715,7 @@ void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList) { // the action of add status may incur the restart procedure, which should NEVER be executed in the timer thread. // The restart of all tasks requires that all tasks should not have active timer for now. Therefore, the execution -// of restart in timer thread will result in a dead lock. +// of restart in timer thread will result in a deadlock. int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId) { return streamTaskSchedTask(pMsgCb, vgId, streamId, taskId, STREAM_EXEC_T_ADD_FAILED_TASK); } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index b4fcf1edc9..c1e4850bfe 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1552,7 +1552,6 @@ static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t int32_t* pFailed, const char* id) { int32_t numOfRsp = 0; int32_t numOfFailed = 0; - bool allRsp = false; int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo); @@ -1639,6 +1638,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int32_t notRsp = 0; int32_t numOfFailed = 0; bool triggerDispatchRsp = false; + bool addFailure = false; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int64_t tmpCheckpointId = -1; int32_t tmpTranId = -1; @@ -1698,6 +1698,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } else { if (pRsp->inputStatus == TASK_INPUT_STATUS__REFUSED) { // todo handle the role-changed during checkpoint generation, add test case + addFailure = true; stError( "s-task:%s downstream task:0x%x(vgId:%d) refused the dispatch msg, downstream may become follower or " "restart already, treat it as success", @@ -1745,6 +1746,10 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code)); } + if (addFailure) { // add failure downstream node id, and start the nodeEp update procedure + streamTaskAddIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); + } + // all msg rsp already, continue // we need to re-try send dispatch msg to downstream tasks if (allRsp && (numOfFailed == 0)) { From 98075a7c5a40c233f00b0987518e0a61bcb63b30 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Dec 2024 15:37:20 +0800 Subject: [PATCH 011/108] test: for ostime --- source/os/src/osTime.c | 29 ++++++------ source/os/src/osTimezone.c | 29 ++++++------ source/os/test/CMakeLists.txt | 81 +++++++++++++++++----------------- source/os/test/osTimeTests.cpp | 53 +++++++++++++++++++++- 4 files changed, 121 insertions(+), 71 deletions(-) diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 29cbcaeb2c..75cb2b91a2 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -31,7 +31,7 @@ #include #include #include -//#define TM_YEAR_BASE 1970 //origin +// #define TM_YEAR_BASE 1970 //origin #define TM_YEAR_BASE 1900 // slguan // This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) @@ -345,8 +345,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { #endif } -size_t -taosStrfTime(char *s, size_t maxsize, char const *format, struct tm const *t){ +size_t taosStrfTime(char *s, size_t maxsize, char const *format, struct tm const *t) { if (!s || !format || !t) return 0; return strftime(s, maxsize, format, t); } @@ -379,7 +378,7 @@ int32_t taosTime(time_t *t) { if (t == NULL) { return TSDB_CODE_INVALID_PARA; } - time_t r = time(t); + time_t r = time(t); if (r == (time_t)-1) { return TAOS_SYSTEM_ERROR(errno); } @@ -433,15 +432,15 @@ time_t taosMktime(struct tm *timep, timezone_t tz) { return result; } int64_t tzw = 0; - #ifdef _MSC_VER - #if _MSC_VER >= 1900 - tzw = _timezone; - #endif - #endif +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + tzw = _timezone; +#endif +#endif return user_mktime64(timep->tm_year + 1900, timep->tm_mon + 1, timep->tm_mday, timep->tm_hour, timep->tm_min, timep->tm_sec, tzw); #else - time_t r = tz != NULL ? mktime_z(tz, timep) : mktime(timep); + time_t r = (tz != NULL ? mktime_z(tz, timep) : mktime(timep)); if (r == (time_t)-1) { terrno = TAOS_SYSTEM_ERROR(errno); } @@ -450,7 +449,7 @@ time_t taosMktime(struct tm *timep, timezone_t tz) { #endif } -struct tm *taosGmTimeR(const time_t *timep, struct tm *result){ +struct tm *taosGmTimeR(const time_t *timep, struct tm *result) { if (timep == NULL || result == NULL) { return NULL; } @@ -461,7 +460,7 @@ struct tm *taosGmTimeR(const time_t *timep, struct tm *result){ #endif } -time_t taosTimeGm(struct tm *tmp){ +time_t taosTimeGm(struct tm *tmp) { if (tmp == NULL) { return -1; } @@ -530,7 +529,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int3 } return result; #else - res = tz != NULL ? localtime_rz(tz, timep, result): localtime_r(timep, result); + res = (tz != NULL ? localtime_rz(tz, timep, result) : localtime_r(timep, result)); if (res == NULL && buf != NULL) { (void)snprintf(buf, bufSize, "NaN"); } @@ -544,8 +543,8 @@ int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { int32_t code = 0; #ifdef WINDOWS - LARGE_INTEGER t; - FILETIME f; + LARGE_INTEGER t; + FILETIME f; GetSystemTimeAsFileTime(&f); t.QuadPart = f.dwHighDateTime; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 395c1cbb82..cad3f426f2 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -750,13 +750,14 @@ int32_t taosSetGlobalTimezone(const char *tz) { int32_t code = TSDB_CODE_SUCCESS; uDebug("[tz]set timezone to %s", tz) #ifdef WINDOWS - char winStr[TD_TIMEZONE_LEN * 2] = {0}; + char winStr[TD_TIMEZONE_LEN * 2] = {0}; for (size_t i = 0; i < W_TZ_CITY_NUM; i++) { if (strcmp(tz_win[i][0], tz) == 0) { char keyPath[256] = {0}; char keyValue[100] = {0}; DWORD keyValueSize = sizeof(keyValue); - snprintf(keyPath, sizeof(keyPath), "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", tz_win[i][1]); + snprintf(keyPath, sizeof(keyPath), "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", + tz_win[i][1]); RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&keyValue, &keyValueSize); if (keyValueSize > 0) { keyValue[4] = (keyValue[4] == '+' ? '-' : '+'); @@ -770,7 +771,7 @@ int32_t taosSetGlobalTimezone(const char *tz) { _putenv(winStr); _tzset(); #else - code = setenv("TZ", tz, 1); + code = setenv("TZ", tz, 1); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; @@ -779,7 +780,7 @@ int32_t taosSetGlobalTimezone(const char *tz) { tzset(); #endif - time_t tx1 = taosGetTimestampSec(); + time_t tx1 = taosGetTimestampSec(); return taosFormatTimezoneStr(tx1, tz, NULL, tsTimezoneStr); } @@ -797,7 +798,7 @@ int32_t taosGetLocalTimezoneOffset() { #endif } -int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *outTimezoneStr){ +int32_t taosFormatTimezoneStr(time_t t, const char *tz, timezone_t sp, char *outTimezoneStr) { struct tm tm1; if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) { uError("%s failed to get local time: code:%d", __FUNCTION__, errno); @@ -813,16 +814,17 @@ int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *out */ char str1[TD_TIMEZONE_LEN] = {0}; - if (taosStrfTime(str1, sizeof(str1), "%Z", &tm1) == 0){ + if (taosStrfTime(str1, sizeof(str1), "%Z", &tm1) == 0) { uError("failed to get timezone name"); return TSDB_CODE_TIME_ERROR; } char str2[TD_TIMEZONE_LEN] = {0}; - if (taosStrfTime(str2, sizeof(str2), "%z", &tm1) == 0){ + if (taosStrfTime(str2, sizeof(str2), "%z", &tm1) == 0) { uError("failed to get timezone offset"); return TSDB_CODE_TIME_ERROR; } + (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s)", tz, str1, str2); uDebug("[tz] system timezone:%s", outTimezoneStr); return 0; @@ -847,7 +849,6 @@ void getTimezoneStr(char *tz) { goto END; } while (0); - TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); if (pFile == NULL) { uWarn("[tz] failed to open /etc/timezone, reason:%s", strerror(errno)); @@ -876,8 +877,8 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr) { char value[100] = {0}; char keyPath[100] = {0}; DWORD bufferSize = sizeof(value); - LONG result = RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", - RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + LONG result = RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", + "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } @@ -891,9 +892,9 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr) { if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } - if (bufferSize > 0) { // value like (UTC+05:30) Chennai, Kolkata, Mumbai, New Delhi - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, - value[4], value[5], value[6], value[8], value[9]); + if (bufferSize > 0) { // value like (UTC+05:30) Chennai, Kolkata, Mumbai, New Delhi + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], + value[6], value[8], value[9]); } break; } @@ -903,7 +904,7 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr) { #else char tz[TD_TIMEZONE_LEN] = {0}; getTimezoneStr(tz); - time_t tx1 = taosGetTimestampSec(); + time_t tx1 = taosGetTimestampSec(); return taosFormatTimezoneStr(tx1, tz, NULL, outTimezoneStr); #endif } \ No newline at end of file diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index 13fea463f7..c8aa86c852 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -14,20 +14,30 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) -# osTests -add_executable(osTests "osTests.cpp") -target_link_libraries(osTests os util gtest_main) +add_executable(osAtomicTests "osAtomicTests.cpp") +target_link_libraries(osAtomicTests os util gtest_main) add_test( - NAME osTests - COMMAND osTests + NAME osAtomicTests + COMMAND osAtomicTests ) -add_executable(osSystemTests "osSystemTests.cpp") -target_link_libraries(osSystemTests os util gtest_main) +if(TD_LINUX) +add_executable(osDirTests "osDirTests.cpp") +target_link_libraries(osDirTests os util gtest_main) add_test( - NAME osSystemTests - COMMAND osSystemTests + NAME osDirTests + COMMAND osDirTests ) +endif() + +if(TD_LINUX) +add_executable(osEnvTests "osEnvTests.cpp") +target_link_libraries(osEnvTests os util gtest_main) +add_test( + NAME osEnvTests + COMMAND osEnvTests +) +endif() add_executable(osMathTests "osMathTests.cpp") target_link_libraries(osMathTests os util gtest_main) @@ -36,6 +46,13 @@ add_test( COMMAND osMathTests ) +add_executable(osSemaphoreTests "osSemaphoreTests.cpp") +target_link_libraries(osSemaphoreTests os util gtest_main) +add_test( + NAME osSemaphoreTests + COMMAND osSemaphoreTests +) + add_executable(osSignalTests "osSignalTests.cpp") target_link_libraries(osSignalTests os util gtest_main) add_test( @@ -57,6 +74,20 @@ add_test( COMMAND osStringTests ) +add_executable(osTests "osTests.cpp") +target_link_libraries(osTests os util gtest_main) +add_test( + NAME osTests + COMMAND osTests +) + +add_executable(osSystemTests "osSystemTests.cpp") +target_link_libraries(osSystemTests os util gtest_main) +add_test( + NAME osSystemTests + COMMAND osSystemTests +) + add_executable(osThreadTests "osThreadTests.cpp") target_link_libraries(osThreadTests os util gtest_main) add_test( @@ -71,35 +102,3 @@ add_test( COMMAND osTimeTests ) - -if(TD_LINUX) - -add_executable(osAtomicTests "osAtomicTests.cpp") -target_link_libraries(osAtomicTests os util gtest_main) -add_test( - NAME osAtomicTests - COMMAND osAtomicTests -) - -add_executable(osDirTests "osDirTests.cpp") -target_link_libraries(osDirTests os util gtest_main) -add_test( - NAME osDirTests - COMMAND osDirTests -) - -add_executable(osEnvTests "osEnvTests.cpp") -target_link_libraries(osEnvTests os util gtest_main) -add_test( - NAME osEnvTests - COMMAND osEnvTests -) - -endif() - -add_executable(osSemaphoreTests "osSemaphoreTests.cpp") -target_link_libraries(osSemaphoreTests os util gtest_main) -add_test( - NAME osSemaphoreTests - COMMAND osSemaphoreTests -) diff --git a/source/os/test/osTimeTests.cpp b/source/os/test/osTimeTests.cpp index 5c8c837dca..1d34587ad8 100644 --- a/source/os/test/osTimeTests.cpp +++ b/source/os/test/osTimeTests.cpp @@ -33,7 +33,7 @@ TEST(osTimeTests, taosLocalTime) { // Test 1: Test when both timep and result are not NULL time_t timep = 1617531000; // 2021-04-04 18:10:00 struct tm result; - struct tm* local_time = taosLocalTime(&timep, &result, NULL, 0, NULL); + struct tm *local_time = taosLocalTime(&timep, &result, NULL, 0, NULL); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 121); ASSERT_EQ(local_time->tm_mon, 3); @@ -92,4 +92,55 @@ TEST(osTimeTests, taosLocalTime) { local_time = taosLocalTime(&neg_timep3, &result, NULL, 0, NULL); ASSERT_EQ(local_time, nullptr); #endif +} + +TEST(osTimeTests, invalidParameter) { + void *retp = NULL; + int32_t reti = 0; + char buf[1024] = {0}; + char fmt[1024] = {0}; + struct tm tm = {0}; + struct timeval tv = {0}; + + retp = taosStrpTime(buf, fmt, NULL); + EXPECT_EQ(retp, nullptr); + retp = taosStrpTime(NULL, fmt, &tm); + EXPECT_EQ(retp, nullptr); + retp = taosStrpTime(buf, NULL, &tm); + EXPECT_EQ(retp, nullptr); + + reti = taosGetTimeOfDay(NULL); + EXPECT_NE(reti, 0); + + reti = taosTime(NULL); + EXPECT_NE(reti, 0); + + tm.tm_year = 2024; + tm.tm_mon = 10; + tm.tm_mday = 23; + tm.tm_hour = 12; + tm.tm_min = 1; + tm.tm_sec = 0; + tm.tm_isdst = -1; + time_t rett = taosMktime(&tm, NULL); + EXPECT_NE(rett, 0); + + retp = taosLocalTime(NULL, &tm, NULL, 0, NULL); + EXPECT_EQ(retp, nullptr); + + retp = taosLocalTime(&rett, NULL, NULL, 0, NULL); + EXPECT_EQ(retp, nullptr); + + reti = taosSetGlobalTimezone(NULL); + EXPECT_NE(reti, 0); +} + +TEST(osTimeTests, user_mktime64) { + int64_t reti = 0; + + reti = user_mktime64(2024, 10, 23, 12, 3, 2, 1); + EXPECT_NE(reti, 0); + + reti = user_mktime64(2024, 1, 23, 12, 3, 2, 1); + EXPECT_NE(reti, 0); } \ No newline at end of file From ed59e85dab4dba5a97d10ac48268a7a63ffe36d6 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 11 Dec 2024 16:02:25 +0800 Subject: [PATCH 012/108] fix: minus operator resType --- source/libs/scalar/src/scalar.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b3610d035f..74ede393d8 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1673,8 +1673,8 @@ static int32_t sclGetMinusOperatorResType(SOperatorNode *pOp) { if (!IS_MATHABLE_TYPE(((SExprNode *)(pOp->pLeft))->resType.type)) { return TSDB_CODE_TSC_INVALID_OPERATION; } - pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + pOp->node.resType.type = ((SExprNode *)(pOp->pLeft))->resType.type; + pOp->node.resType.bytes = tDataTypes[((SExprNode *)(pOp->pLeft))->resType.type].bytes; return TSDB_CODE_SUCCESS; } From 58a605de47740fcc6e8cab7fc3d80c37f74c5b36 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 11 Dec 2024 17:29:37 +0800 Subject: [PATCH 013/108] fix: in operator only has one overflow contition --- source/libs/scalar/src/filter.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d8622d93ee..45260b69cc 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1305,6 +1305,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { out.columnData->info.type = type; out.columnData->info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; // reserved space for simple_copy + int32_t overflowCount = 0; for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; if (valueNode->node.resType.type != type) { @@ -1317,6 +1318,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { if (overflow) { cell = cell->pNext; + ++overflowCount; continue; } @@ -1357,6 +1359,9 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { cell = cell->pNext; } + if(overflowCount == listNode->pNodeList->length) { + FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); + } colDataDestroy(out.columnData); taosMemoryFree(out.columnData); FLT_ERR_RET(code); From 34a73b96ed032a60ae7889beaa73dc2a6013de82 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Dec 2024 18:04:02 +0800 Subject: [PATCH 014/108] test: ut for osthread --- source/os/src/osThread.c | 187 +++++++++++-------------------- source/os/test/osThreadTests.cpp | 164 ++++++++++++++++++++++++++- 2 files changed, 225 insertions(+), 126 deletions(-) diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index f888835d95..2810c5e196 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -23,8 +23,7 @@ int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start) int32_t code = pthread_create(tid, attr, start, arg); if (code) { taosThreadClear(tid); - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -33,8 +32,7 @@ int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -44,8 +42,7 @@ int32_t taosThreadAttrGetDetachState(const TdThreadAttr *attr, int32_t *detachst OS_PARAM_CHECK(detachstate); int32_t code = pthread_attr_getdetachstate(attr, detachstate); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -55,8 +52,7 @@ int32_t taosThreadAttrGetInheritSched(const TdThreadAttr *attr, int32_t *inherit OS_PARAM_CHECK(inheritsched); int32_t code = pthread_attr_getinheritsched(attr, inheritsched); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -66,8 +62,7 @@ int32_t taosThreadAttrGetSchedParam(const TdThreadAttr *attr, struct sched_param OS_PARAM_CHECK(param); int32_t code = pthread_attr_getschedparam(attr, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -77,8 +72,7 @@ int32_t taosThreadAttrGetSchedPolicy(const TdThreadAttr *attr, int32_t *policy) OS_PARAM_CHECK(policy); int32_t code = pthread_attr_getschedpolicy(attr, policy); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -88,8 +82,7 @@ int32_t taosThreadAttrGetScope(const TdThreadAttr *attr, int32_t *contentionscop OS_PARAM_CHECK(contentionscope); int32_t code = pthread_attr_getscope(attr, contentionscope); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -99,8 +92,7 @@ int32_t taosThreadAttrGetStackSize(const TdThreadAttr *attr, size_t *stacksize) OS_PARAM_CHECK(stacksize); int32_t code = pthread_attr_getstacksize(attr, stacksize); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -109,8 +101,7 @@ int32_t taosThreadAttrInit(TdThreadAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -119,8 +110,7 @@ int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachstate) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setdetachstate(attr, detachstate); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -129,8 +119,7 @@ int32_t taosThreadAttrSetInheritSched(TdThreadAttr *attr, int32_t inheritsched) OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setinheritsched(attr, inheritsched); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -139,8 +128,7 @@ int32_t taosThreadAttrSetSchedParam(TdThreadAttr *attr, const struct sched_param OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setschedparam(attr, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -149,8 +137,7 @@ int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr *attr, int32_t policy) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setschedpolicy(attr, policy); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -159,8 +146,7 @@ int32_t taosThreadAttrSetScope(TdThreadAttr *attr, int32_t contentionscope) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setscope(attr, contentionscope); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -169,8 +155,7 @@ int32_t taosThreadAttrSetStackSize(TdThreadAttr *attr, size_t stacksize) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setstacksize(attr, stacksize); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -178,8 +163,7 @@ int32_t taosThreadAttrSetStackSize(TdThreadAttr *attr, size_t stacksize) { int32_t taosThreadCancel(TdThread thread) { int32_t code = pthread_cancel(thread); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -191,8 +175,7 @@ int32_t taosThreadCondDestroy(TdThreadCond *cond) { #else int32_t code = pthread_cond_destroy(cond); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -206,8 +189,7 @@ int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) { #else int32_t code = pthread_cond_init(cond, attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -221,8 +203,7 @@ int32_t taosThreadCondSignal(TdThreadCond *cond) { #else int32_t code = pthread_cond_signal(cond); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -236,8 +217,7 @@ int32_t taosThreadCondBroadcast(TdThreadCond *cond) { #else int32_t code = pthread_cond_broadcast(cond); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -254,8 +234,7 @@ int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { #else int32_t code = pthread_cond_wait(cond, mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -291,14 +270,14 @@ int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif } int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *pshared) { + OS_PARAM_CHECK(attr); OS_PARAM_CHECK(pshared); #ifdef __USE_WIN_THREAD if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE; @@ -307,8 +286,7 @@ int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *psha OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_getpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -321,8 +299,7 @@ int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -337,8 +314,7 @@ int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId) { OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_setclock(attr, clockId); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -351,8 +327,7 @@ int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) { #else int32_t code = pthread_condattr_setpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -361,8 +336,7 @@ int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) { int32_t taosThreadDetach(TdThread thread) { int32_t code = pthread_detach(thread); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -380,8 +354,7 @@ int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_p OS_PARAM_CHECK(param); int32_t code = pthread_getschedparam(thread, policy, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -393,8 +366,7 @@ void *taosThreadGetSpecific(TdThreadKey key) { int32_t taosThreadJoin(TdThread thread, void **valuePtr) { int32_t code = pthread_join(thread, valuePtr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -403,8 +375,7 @@ int32_t taosThreadKeyCreate(TdThreadKey *key, void (*destructor)(void *)) { OS_PARAM_CHECK(key); int32_t code = pthread_key_create(key, destructor); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -412,8 +383,7 @@ int32_t taosThreadKeyCreate(TdThreadKey *key, void (*destructor)(void *)) { int32_t taosThreadKeyDelete(TdThreadKey key) { int32_t code = pthread_key_delete(key); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -421,8 +391,7 @@ int32_t taosThreadKeyDelete(TdThreadKey key) { int32_t taosThreadKill(TdThread thread, int32_t sig) { int32_t code = pthread_kill(thread, sig); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -439,8 +408,7 @@ int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) { #else int32_t code = pthread_mutex_destroy(mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -459,8 +427,7 @@ int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) #else int32_t code = pthread_mutex_init(mutex, attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -474,8 +441,7 @@ int32_t taosThreadMutexLock(TdThreadMutex *mutex) { #else int32_t code = pthread_mutex_lock(mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -507,8 +473,7 @@ int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { #else int32_t code = pthread_mutex_unlock(mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -521,8 +486,7 @@ int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -537,8 +501,7 @@ int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr *attr, int32_t *ps OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_getpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -557,8 +520,7 @@ int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr *attr, int32_t *kind) OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_gettype(attr, kind); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -571,8 +533,7 @@ int32_t taosThreadMutexAttrInit(TdThreadMutexAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -585,8 +546,7 @@ int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr *attr, int32_t pshared) OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_setpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -603,8 +563,7 @@ int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr *attr, int32_t kind) { OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_settype(attr, kind); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -613,8 +572,7 @@ int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr *attr, int32_t kind) { int32_t taosThreadOnce(TdThreadOnce *onceControl, void (*initRoutine)(void)) { int32_t code = pthread_once(onceControl, initRoutine); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -629,8 +587,7 @@ int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { OS_PARAM_CHECK(rwlock); int32_t code = pthread_rwlock_destroy(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -645,8 +602,7 @@ int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *a #else int32_t code = pthread_rwlock_init(rwlock, attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -660,8 +616,7 @@ int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_rdlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -683,8 +638,7 @@ int32_t taosThreadRwlockTryRdlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_tryrdlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -699,8 +653,7 @@ int32_t taosThreadRwlockTryWrlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_trywrlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -718,8 +671,7 @@ int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_unlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -734,8 +686,7 @@ int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_wrlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -748,8 +699,7 @@ int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_rwlockattr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -763,8 +713,7 @@ int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr *attr, int32_t * #else int32_t code = pthread_rwlockattr_getpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -777,8 +726,7 @@ int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_rwlockattr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -791,8 +739,7 @@ int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr *attr, int32_t pshared OS_PARAM_CHECK(attr); int32_t code = pthread_rwlockattr_setpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -803,8 +750,7 @@ TdThread taosThreadSelf(void) { return pthread_self(); } int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate) { int32_t code = pthread_setcancelstate(state, oldstate); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -812,8 +758,7 @@ int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate) { int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype) { int32_t code = pthread_setcanceltype(type, oldtype); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -822,8 +767,7 @@ int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sc OS_PARAM_CHECK(param); int32_t code = pthread_setschedparam(thread, policy, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -832,8 +776,7 @@ int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { OS_PARAM_CHECK(value); int32_t code = pthread_setspecific(key, value); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -845,8 +788,7 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { #else int32_t code = pthread_spin_destroy((pthread_spinlock_t *)lock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -860,8 +802,7 @@ int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { #else int32_t code = pthread_spin_init((pthread_spinlock_t *)lock, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -874,8 +815,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { #else int32_t code = pthread_spin_lock((pthread_spinlock_t *)lock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -901,8 +841,7 @@ int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { #else int32_t code = pthread_spin_unlock((pthread_spinlock_t *)lock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index e7fc4f1356..622dcd38d7 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -29,6 +29,166 @@ #include "os.h" #include "tlog.h" -TEST(osThreadTests, osThreadTests1) { - +static void *funcPtr0(void *param) { return NULL; } +static void *funcPtr1(void *param) { + taosMsleep(1000); + return NULL; +} + +TEST(osThreadTests, invalidParameter) { + TdThread tid1 = {0}; + TdThread tid2 = {0}; + int32_t reti = 0; + TdThreadAttr attr = {0}; + TdThreadAttr attr2 = {0}; + int32_t param = 0; + (void)taosThreadAttrInit(&attr); + + reti = taosThreadCreate(NULL, NULL, funcPtr0, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCreate(&tid1, NULL, NULL, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCreate(&tid1, NULL, funcPtr1, NULL); + EXPECT_EQ(reti, 0); + reti = taosThreadCancel(tid1); + EXPECT_EQ(reti, 0); + reti = taosThreadCreate(&tid2, NULL, funcPtr0, NULL); + EXPECT_EQ(reti, 0); + taosMsleep(1000); + reti = taosThreadCancel(tid2); + EXPECT_EQ(reti, 0); + + reti = taosThreadAttrDestroy(NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_JOINABLE); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetDetachState(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetDetachState(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetDetachState(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetDetachState(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetDetachState(&attr, ¶m); + EXPECT_EQ(reti, 0); + + reti = taosThreadAttrSetInheritSched(&attr, PTHREAD_INHERIT_SCHED); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetInheritSched(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetInheritSched(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetInheritSched(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetInheritSched(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetInheritSched(&attr, ¶m); + EXPECT_EQ(reti, 0); + + struct sched_param schedparam = {0}; + reti = taosThreadAttrGetSchedParam(&attr, &schedparam); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetSchedParam(&attr, &schedparam); + EXPECT_EQ(reti, 0); + schedparam.sched_priority = -1; + reti = taosThreadAttrSetSchedParam(&attr, &schedparam); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetSchedParam(NULL, &schedparam); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedParam(NULL, &schedparam); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedParam(&attr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadAttrSetSchedPolicy(&attr, SCHED_FIFO); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetSchedPolicy(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetSchedPolicy(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedPolicy(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedPolicy(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedPolicy(&attr, ¶m); + EXPECT_EQ(reti, 0); + + reti = taosThreadAttrSetScope(&attr, PTHREAD_SCOPE_SYSTEM); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetScope(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetScope(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetScope(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetScope(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetScope(&attr, ¶m); + EXPECT_EQ(reti, 0); + + size_t stacksize; + reti = taosThreadAttrGetStackSize(&attr, &stacksize); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetStackSize(&attr, stacksize); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetStackSize(&attr, 2048); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetStackSize(NULL, stacksize); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetStackSize(NULL, &stacksize); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetStackSize(&attr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadAttrInit(NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadCondDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondInit(NULL, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondSignal(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondBroadcast(NULL); + EXPECT_NE(reti, 0); + + TdThreadCond cond; + TdThreadMutex mutex; + reti = taosThreadCondWait(&cond, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondWait(NULL, &mutex); + EXPECT_NE(reti, 0); + + struct timespec abstime = {0}; + reti = taosThreadCondTimedWait(&cond, NULL, &abstime); + EXPECT_NE(reti, 0); + reti = taosThreadCondTimedWait(NULL, &mutex, &abstime); + EXPECT_NE(reti, 0); + reti = taosThreadCondTimedWait(&cond, &mutex, NULL); + EXPECT_NE(reti, 0); + + TdThreadCondAttr condattr; + (void)taosThreadCondAttrInit(&condattr); + reti = taosThreadCondAttrInit(NULL); + EXPECT_NE(reti, 0); + int32_t pshared; + reti = taosThreadCondAttrGetPshared(&condattr, &pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadCondAttrSetPshared(&condattr, pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadCondAttrSetPshared(&condattr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrSetPshared(NULL, pshared); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrGetPshared(NULL, &pshared); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrGetPshared(&condattr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadCondAttrDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrDestroy(&condattr); + EXPECT_EQ(reti, 0); } From 13ccfdc3141248636aeaf0e8a66b92222eb424c9 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 12 Dec 2024 08:31:47 +0800 Subject: [PATCH 015/108] fix: in contition --- source/libs/scalar/inc/filterInt.h | 1 + source/libs/scalar/src/filter.c | 35 +++++--- tests/system-test/2-query/operator.py | 114 ++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 tests/system-test/2-query/operator.py diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 9adc9ee99c..ed07592c73 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -236,6 +236,7 @@ typedef struct SFltBuildGroupCtx { SFilterInfo *info; SArray *group; int32_t code; + bool ignore; } SFltBuildGroupCtx; typedef struct { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 45260b69cc..10850aab9a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1284,7 +1284,8 @@ static void filterFreeGroup(void *pItem) { taosMemoryFreeClear(p->unitFlags); } -int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { +int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree, SArray *group) { + SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext; SOperatorNode *node = (SOperatorNode *)tree; int32_t ret = TSDB_CODE_SUCCESS; SFilterFieldId left = {0}, right = {0}; @@ -1360,7 +1361,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { cell = cell->pNext; } if(overflowCount == listNode->pNodeList->length) { - FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); + ctx->ignore = true; } colDataDestroy(out.columnData); taosMemoryFree(out.columnData); @@ -1697,10 +1698,17 @@ EDealRes fltTreeToGroup(SNode *pNode, void *pContext) { FLT_ERR_RET(terrno); } - SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; + SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup, .ignore = false}; nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); - + if(tctx.ignore) { + ctx->ignore = true; + taosArrayDestroyEx(newGroup, filterFreeGroup); + newGroup = NULL; + taosArrayDestroyEx(resGroup, filterFreeGroup); + resGroup = NULL; + break; + } FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); taosArrayDestroyEx(newGroup, filterFreeGroup); @@ -1712,9 +1720,10 @@ EDealRes fltTreeToGroup(SNode *pNode, void *pContext) { cell = cell->pNext; } - - if (NULL == taosArrayAddAll(ctx->group, preGroup)) { - FLT_ERR_JRET(terrno); + if (!ctx->ignore) { + if (NULL == taosArrayAddAll(ctx->group, preGroup)) { + FLT_ERR_JRET(terrno); + } } taosArrayDestroy(preGroup); @@ -1726,6 +1735,9 @@ EDealRes fltTreeToGroup(SNode *pNode, void *pContext) { SListCell *cell = node->pParameterList->pHead; for (int32_t i = 0; i < node->pParameterList->length; ++i) { nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)pContext); + if(ctx->ignore) { + ctx->ignore = false; + } FLT_ERR_JRET(ctx->code); cell = cell->pNext; @@ -1740,7 +1752,7 @@ EDealRes fltTreeToGroup(SNode *pNode, void *pContext) { } if (QUERY_NODE_OPERATOR == nType) { - FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group)); + FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx, ctx->info, pNode, ctx->group)); return DEAL_RES_IGNORE_CHILD; } @@ -3836,13 +3848,16 @@ int32_t fltInitFromNode(SNode *tree, SFilterInfo *info, uint32_t options) { goto _return; } - SFltBuildGroupCtx tctx = {.info = info, .group = group}; + SFltBuildGroupCtx tctx = {.info = info, .group = group, .ignore = false}; nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx); if (TSDB_CODE_SUCCESS != tctx.code) { taosArrayDestroyEx(group, filterFreeGroup); code = tctx.code; goto _return; } + if (tctx.ignore) { + FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); + } code = filterConvertGroupFromArray(info, group); if (TSDB_CODE_SUCCESS != code) { taosArrayDestroyEx(group, filterFreeGroup); @@ -3876,7 +3891,7 @@ int32_t fltInitFromNode(SNode *tree, SFilterInfo *info, uint32_t options) { _return: if (code) { - qInfo("init from node failed, code:%d", code); + qInfo("init from node failed, code:%d, %s", code, tstrerror(code)); } return code; } diff --git a/tests/system-test/2-query/operator.py b/tests/system-test/2-query/operator.py new file mode 100644 index 0000000000..edfd023b7c --- /dev/null +++ b/tests/system-test/2-query/operator.py @@ -0,0 +1,114 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +from util.common import tdCom +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.dbname = "db" + self.rowNum = 10 + self.ts = 1537146000000 + + # test in/not in contidion with invalid value + def ts5757(self): + + tdSql.execute(f"create database if not exists {self.dbname}") + + tdSql.execute(f"DROP STABLE IF EXISTS {self.dbname}.super_t1;") + tdSql.execute(f"DROP TABLE IF EXISTS {self.dbname}.t1;") + tdSql.execute(f"CREATE STABLE IF NOT EXISTS {self.dbname}.super_t1(time TIMESTAMP, c0 BIGINT UNSIGNED) TAGS (location BINARY(64))") + tdSql.execute(f"CREATE TABLE {self.dbname}.t1 USING {self.dbname}.super_t1 TAGS ('ek')") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c0) VALUES (1641024000000, 1);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c0) VALUES (1641024005000, 2);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c0) VALUES (1641024010000, NULL);") + + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL AND c0 IN (-1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL AND c0 IN (-1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL AND c0 IN (-1, 1);") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL AND c0 IN (2, -1, 1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL AND c0 NOT IN (-1);") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL AND c0 NOT IN (-1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL AND c0 NOT IN (3);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL AND c0 NOT IN (-1, 1);") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL AND c0 NOT IN (2, -1, 1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE (c0 IS NULL AND c0 IN (-1)) or c0 in(1)") + tdSql.checkRows(1) + + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL OR c0 IN (-1);") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL OR c0 IN (-1);") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL OR c0 IN (-1, 1);") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL OR c0 IN (2, -1, 1);") + tdSql.checkRows(3) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL OR c0 NOT IN (-1);") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL OR c0 NOT IN (-1);") + tdSql.checkRows(3) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL OR c0 NOT IN (3);") + tdSql.checkRows(3) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL OR c0 NOT IN (-1, 1);") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL OR c0 NOT IN (-1);") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NULL OR c0 NOT IN (2, -1, 1);") + tdSql.checkRows(1) + + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE ((c0 is NULL) AND (c0 in (-1)) )") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE ((c0 in (-1)) AND (c0 is NULL) )") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE ((c0 in (-1)) AND (c0 is NULL) ) OR c0 in(1)") + tdSql.checkRows(1) + + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (-1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IS NOT NULL;") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (-1) or c0 in(1);") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (1) or c0 in(-1);") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (-1) or c0 in(-1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (-1) and c0 in(1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (1) and c0 in(-1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (-1) and c0 in(-1);") + tdSql.checkRows(0) + + def run(self): + dbname = "db" + tdSql.prepare() + tdSql.execute(f"create database if not exists {self.dbname}") + + self.ts5757() + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) + +tdCases.addLinux(__file__, TDTestCase()) From aa187f1ee3ba93838a5f4783f9b0e00c57709a73 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 12 Dec 2024 09:35:35 +0800 Subject: [PATCH 016/108] fix(stream): check return value. --- source/libs/stream/src/streamDispatch.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index c1e4850bfe..968744a0c5 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1747,7 +1747,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } if (addFailure) { // add failure downstream node id, and start the nodeEp update procedure - streamTaskAddIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); + // ignore the return error and continue + int32_t unused = streamTaskAddIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); } // all msg rsp already, continue From 448c6d0310eca2aca26b3a0fe703fefe94bf76ca Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 10:42:07 +0800 Subject: [PATCH 017/108] test:add local coverage script --- tests/run_local_coverage.sh | 367 ++++++++++++++++++++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 tests/run_local_coverage.sh diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh new file mode 100644 index 0000000000..f5ec7e41ba --- /dev/null +++ b/tests/run_local_coverage.sh @@ -0,0 +1,367 @@ +#!/bin/bash + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function print_color() { + local color="$1" + local message="$2" + echo -e "${color}${message}${NC}" +} + +# Initialization parameter +TDENGINE_DIR="/root/TDinternal/community" +BRANCH="" +TDENGINE_GCDA_DIR="/root/TDinternal/community/debug/" + +# Parse command line parameters +while getopts "hd:b:f:c:u:" arg; do + case $arg in + d) + TDENGINE_DIR=$OPTARG + ;; + b) + BRANCH=$OPTARG + ;; + f) + TDENGINE_GCDA_DIR=$OPTARG + ;; + c) + TEST_CASE=$OPTARG + ;; + u) + UNIT_TEST_CASE=$OPTARG + ;; + h) + echo "Usage: $(basename $0) -d [TDengine dir] -b [branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Branch] [default local branch; eg:cover/3.0] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test single case/all cases] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 0 + ;; + ?) + echo "Usage: ./$(basename $0) -h" + exit 1 + ;; + esac +done + +# Check if the command name is provided +if [ -z "$TDENGINE_DIR" ]; then + echo "Error: TDengine dir is required." + echo "Usage: $(basename $0) -d [TDengine dir] -b [branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Branch] [default local branch; eg:cover/3.0] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test casingle case/all casesse] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 1 +fi + + +echo "TDENGINE_DIR = $TDENGINE_DIR" +today=`date +"%Y%m%d"` +TDENGINE_ALLCI_REPORT="$TDENGINE_DIR/tests/all-ci-report-$today.log" + + + +function buildTDengine() { + print_color "$GREEN" "TDengine build start" + + # pull parent code + cd "$TDENGINE_DIR/../" + print_color "$GREEN" "git pull parent code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -dfx + git pull + + # pull tdengine code + cd $TDENGINE_DIR + print_color "$GREEN" "git pull tdengine code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -dfx + git pull + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + print_color "$GREEN" "rebuild.." + LOCAL_COMMIT=`git rev-parse --short @` + + makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + #$makecmd + + make -j 8 install + + print_color "$GREEN" "TDengine build end" +} + + +# Check and get the branch name +if [ -n "$BRANCH" ]; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test！" + buildTDengine +else + print_color "$GREEN" "Build is not required for this test！" +fi + +function runCasesOneByOne () { + while read -r line; do + if [[ "$line" != "#"* ]]; then + cmd=`echo $line | cut -d',' -f 5` + if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then + echo $cmd + case=`echo $cmd | cut -d' ' -f 3` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + + elif [[ "$line" == *"$2"* ]]; then + echo $cmd + if [[ "$cmd" == *"pytest.sh"* ]]; then + cmd=`echo $cmd | cut -d' ' -f 2-20` + fi + case=`echo $cmd | cut -d' ' -f 4-20` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + fi + fi + done < $1 +} + +function runUnitTest() { + print_color "$GREEN" "=== Run unit test case ===" + print_color "$GREEN" " $TDENGINE_DIR/debug" + cd $TDENGINE_DIR/debug + ctest -j12 + print_color "$GREEN" "3.0 unit test done" +} + +function runSimCases() { + print_color "$GREEN" "=== Run sim cases ===" + + cd $TDENGINE_DIR/tests/script + runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/longtimeruning_cases.task sim + + totalSuccess=`grep 'sim success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'sim failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + +function runPythonCases() { + print_color "$GREEN" "=== Run python cases ===" + + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' longtimeruning_cases.task + + # army + cd $TDENGINE_DIR/tests/army + runCasesOneByOne ../parallel_test/longtimeruning_cases.task army + + # system-test + cd $TDENGINE_DIR/tests/system-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task system-test + + # develop-test + cd $TDENGINE_DIR/tests/develop-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task develop-test + + totalSuccess=`grep 'py success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'py failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + + +function runTest_all() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + runUnitTest + runSimCases + runPythonCases + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + + +function runTest() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + if [ -n "$TEST_CASE" ] && [ "$TEST_CASE" != "all" ] && [ "$TEST_CASE" != "task" ]; then + TEST_CASE="$TEST_CASE" + print_color "$GREEN" "Test case: $TEST_CASE " + cd $TDENGINE_DIR/tests/script/ && $TEST_CASE + cd $TDENGINE_DIR/tests/army/ && $TEST_CASE + cd $TDENGINE_DIR/tests/system-test/ && $TEST_CASE + cd $TDENGINE_DIR/tests/develop-test/ && $TEST_CASE + elif [ "$TEST_CASE" == "all" ]; then + print_color "$GREEN" "Test case is : parallel_test/longtimeruning_cases.task and all unit cases" + runTest_all + elif [ "$TEST_CASE" == "task" ]; then + print_color "$GREEN" "Test case is only: parallel_test/longtimeruning_cases.task " + runSimCases + runPythonCases + elif [ -n "$UNIT_TEST_CASE" ]; then + UNIT_TEST_CASE="$UNIT_TEST_CASE" + cd $TDENGINE_DIR/debug/build/bin/ && $UNIT_TEST_CASE + else + print_color "$GREEN" "Test case is null" + fi + + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + if [ -n "$TDENGINE_GCDA_DIR" ]; then + TDENGINE_GCDA_DIR="$TDENGINE_GCDA_DIR" + print_color "$GREEN" "Test gcda file dir: $TDENGINE_GCDA_DIR " + else + print_color "$GREEN" "Test gcda file dir is default: /root/TDinternal/community/debug" + fi + + # collect data + lcov -d "$TDENGINE_GCDA_DIR" -capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + lcov --remove coverage.info \ + '*/contrib/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c' '/*/enterprise/*' '*/docs/*' '*/sim/*'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/examples/*' '*/tidpool.c' '*/tmempool.c'\ + '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.cc'\ + '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ + '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function stopTaosd { + print_color "$GREEN" "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasod end" +} + +function stopTaosadapter { + print_color "$GREEN" "Stop taosadapter" + systemctl stop taosadapter.service + PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosadapter + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasoadapter end" + +} + +WORK_DIR=/root + +date >> $WORK_DIR/date.log +print_color "$GREEN" "Run local coverage test cases" | tee -a $WORK_DIR/date.log + +stopTaosd + +runTest + +lcovFunc + + +date >> $WORK_DIR/date.log +print_color "$GREEN" "End of local coverage test cases" | tee -a $WORK_DIR/date.log + + +# Define coverage information files and output directories +COVERAGE_INFO="$TDENGINE_DIR/coverage.info" +OUTPUT_DIR="$WORK_DIR/coverage_report" + +# Check whether the coverage information file exists +if [ ! -f "$COVERAGE_INFO" ]; then + echo "Error: $COVERAGE_INFO not found!" + exit 1 +fi + +# Generate local HTML reports +genhtml "$COVERAGE_INFO" --output-directory "$OUTPUT_DIR" + +# Check whether the report was generated successfully +if [ $? -eq 0 ]; then + echo "HTML coverage report generated successfully in $OUTPUT_DIR" + echo "For more details : " + echo "http://192.168.1.61:7000/" +else + echo "Error generating HTML coverage report" + exit 1 +fi + From 2b46344b5490638e2661fa8609c9f43266bceb6c Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 10:54:34 +0800 Subject: [PATCH 018/108] test:add local coverage script --- tests/run_local_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index f5ec7e41ba..af4ed36f6e 100644 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -104,7 +104,7 @@ function buildTDengine() { makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" print_color "$GREEN" "$makecmd" - #$makecmd + $makecmd make -j 8 install From 49ca1448d0ab5f997028490312ccc1f0f44b1d37 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 10:55:12 +0800 Subject: [PATCH 019/108] test:add local coverage script --- tests/run_local_coverage.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/run_local_coverage.sh diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh old mode 100644 new mode 100755 From f8168564e0a3a07fad140855be5a9cec81dcdd43 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 11:04:24 +0800 Subject: [PATCH 020/108] test:add local coverage script --- tests/run_local_coverage.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index af4ed36f6e..f1029a02d5 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -99,13 +99,13 @@ function buildTDengine() { [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug cd $TDENGINE_DIR/debug - print_color "$GREEN" "rebuild.." - LOCAL_COMMIT=`git rev-parse --short @` + # print_color "$GREEN" "rebuild.." - makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" - print_color "$GREEN" "$makecmd" - $makecmd + # makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + # print_color "$GREEN" "$makecmd" + # $makecmd + cd $TDENGINE_DIR/debug make -j 8 install print_color "$GREEN" "TDengine build end" From 7354d57867541b04c16540eba6137fe88408c837 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 11:21:55 +0800 Subject: [PATCH 021/108] test:add local coverage script --- tests/run_local_coverage.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index f1029a02d5..1e43ce681c 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -36,10 +36,14 @@ while getopts "hd:b:f:c:u:" arg; do u) UNIT_TEST_CASE=$OPTARG ;; + build) + BRANCH_BUILD=$OPTARG + ;; h) - echo "Usage: $(basename $0) -d [TDengine dir] -b [branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -build [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " - echo " -b [Branch] [default local branch; eg:cover/3.0] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -build [Build test branch] [default no:not build, but still install ;yes:will build and install ] " echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " echo " -c [Test single case/all cases] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " @@ -55,9 +59,10 @@ done # Check if the command name is provided if [ -z "$TDENGINE_DIR" ]; then echo "Error: TDengine dir is required." - echo "Usage: $(basename $0) -d [TDengine dir] -b [branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -build [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " - echo " -b [Branch] [default local branch; eg:cover/3.0] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -build [Build test branch] [default no:not build, but still install ;yes:will build and install ] " echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " echo " -c [Test casingle case/all casesse] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " @@ -99,14 +104,17 @@ function buildTDengine() { [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug cd $TDENGINE_DIR/debug - # print_color "$GREEN" "rebuild.." - - # makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" - # print_color "$GREEN" "$makecmd" - # $makecmd - - cd $TDENGINE_DIR/debug - make -j 8 install + if [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then + print_color "$GREEN" "rebuild.." + makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + $makecmd + make -j 8 install + else + print_color "$GREEN" "not build,only install" + cd $TDENGINE_DIR/debug + make -j 8 install + fi print_color "$GREEN" "TDengine build end" } From 635d1b0df5c743fe8e0b812fe9b3240a00c5dfbc Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 11:26:40 +0800 Subject: [PATCH 022/108] test:add local coverage script --- tests/run_local_coverage.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index 1e43ce681c..df91313de4 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -19,7 +19,7 @@ BRANCH="" TDENGINE_GCDA_DIR="/root/TDinternal/community/debug/" # Parse command line parameters -while getopts "hd:b:f:c:u:" arg; do +while getopts "hd:b:f:c:u:i:" arg; do case $arg in d) TDENGINE_DIR=$OPTARG @@ -36,14 +36,14 @@ while getopts "hd:b:f:c:u:" arg; do u) UNIT_TEST_CASE=$OPTARG ;; - build) + i) BRANCH_BUILD=$OPTARG ;; h) - echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -build [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " echo " -b [Test branch] [default local branch; eg:cover/3.0] " - echo " -build [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " echo " -c [Test single case/all cases] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " @@ -59,10 +59,10 @@ done # Check if the command name is provided if [ -z "$TDENGINE_DIR" ]; then echo "Error: TDengine dir is required." - echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -build [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " echo " -b [Test branch] [default local branch; eg:cover/3.0] " - echo " -build [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " echo " -c [Test casingle case/all casesse] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " From dfcb0cb4f6a9560e3c29bd9926830d869d9e1b44 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 11:30:53 +0800 Subject: [PATCH 023/108] test:add local coverage script --- tests/run_local_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index df91313de4..fa9de355a3 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -110,7 +110,7 @@ function buildTDengine() { print_color "$GREEN" "$makecmd" $makecmd make -j 8 install - else + elif [ "$BRANCH_BUILD" == "no" ] ; then print_color "$GREEN" "not build,only install" cd $TDENGINE_DIR/debug make -j 8 install From 0a7f86b60202fd7084d9657a9766f811dd917c10 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 11:37:11 +0800 Subject: [PATCH 024/108] test:add local coverage script --- tests/run_local_coverage.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index fa9de355a3..b8411b2dad 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -110,9 +110,15 @@ function buildTDengine() { print_color "$GREEN" "$makecmd" $makecmd make -j 8 install - elif [ "$BRANCH_BUILD" == "no" ] ; then + elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then print_color "$GREEN" "not build,only install" cd $TDENGINE_DIR/debug + make -j 8 install + elif [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then + print_color "$GREEN" "rebuild.." + makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + $makecmd make -j 8 install fi From 17fa4ee0360e434d836d1cdb1094cf6ac2a40fd9 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 12:01:03 +0800 Subject: [PATCH 025/108] test:add local coverage script --- tests/run_local_coverage.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index b8411b2dad..3d0296d3d0 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -106,6 +106,7 @@ function buildTDengine() { if [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then print_color "$GREEN" "rebuild.." + rm -rf * makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" print_color "$GREEN" "$makecmd" $makecmd @@ -116,6 +117,7 @@ function buildTDengine() { make -j 8 install elif [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then print_color "$GREEN" "rebuild.." + rm -rf * makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" print_color "$GREEN" "$makecmd" $makecmd From 940d7f856198a43de471baed30490399de2761bd Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 14:04:55 +0800 Subject: [PATCH 026/108] test:add local coverage script --- tests/run_local_coverage.sh | 91 +++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index 3d0296d3d0..7590d9fc6a 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -74,10 +74,8 @@ echo "TDENGINE_DIR = $TDENGINE_DIR" today=`date +"%Y%m%d"` TDENGINE_ALLCI_REPORT="$TDENGINE_DIR/tests/all-ci-report-$today.log" - - -function buildTDengine() { - print_color "$GREEN" "TDengine build start" +function pullTDengine() { + print_color "$GREEN" "TDengine pull start" # pull parent code cd "$TDENGINE_DIR/../" @@ -101,41 +99,76 @@ function buildTDengine() { git clean -dfx git pull + print_color "$GREEN" "TDengine pull end" +} + +function buildTDengine() { + print_color "$GREEN" "TDengine build start" + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug cd $TDENGINE_DIR/debug - if [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then - print_color "$GREEN" "rebuild.." - rm -rf * - makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" - print_color "$GREEN" "$makecmd" - $makecmd - make -j 8 install - elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then - print_color "$GREEN" "not build,only install" - cd $TDENGINE_DIR/debug - make -j 8 install - elif [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then - print_color "$GREEN" "rebuild.." - rm -rf * - makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" - print_color "$GREEN" "$makecmd" - $makecmd - make -j 8 install - fi - - print_color "$GREEN" "TDengine build end" + print_color "$GREEN" "rebuild.." + rm -rf * + makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + $makecmd + make -j 8 install } +# function buildTDengineold() { +# print_color "$GREEN" "TDengine build start" -# Check and get the branch name -if [ -n "$BRANCH" ]; then +# [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug +# cd $TDENGINE_DIR/debug + +# if [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then +# print_color "$GREEN" "rebuild.." +# rm -rf * +# makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" +# print_color "$GREEN" "$makecmd" +# $makecmd +# make -j 8 install +# elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then +# print_color "$GREEN" "rebuild.." +# #rm -rf * +# makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" +# print_color "$GREEN" "$makecmd" +# $makecmd +# make -j 8 install +# elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then +# print_color "$GREEN" "not build,only install" +# cd $TDENGINE_DIR/debug +# make -j 8 install +# fi + +# print_color "$GREEN" "TDengine build end" +# } + + +# Check and get the branch name and build branch +if [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then branch="$BRANCH" print_color "$GREEN" "Testing branch: $branch " - print_color "$GREEN" "Build is required for this test！" + print_color "$GREEN" "Build is required for this test!" + pullTDengine buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + pullTDengine + buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "not build,only install!" + pullTDengine + cd $TDENGINE_DIR/debug + make -j 8 install else - print_color "$GREEN" "Build is not required for this test！" + print_color "$GREEN" "Build is not required for this test!" + pullTDengine fi function runCasesOneByOne () { From 414b1c306194891625e2c2634657eb8cb6ebfb30 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 14:25:58 +0800 Subject: [PATCH 027/108] test:add local coverage script --- tests/run_local_coverage.sh | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index 7590d9fc6a..c5ad72ce14 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -116,36 +116,6 @@ function buildTDengine() { make -j 8 install } -# function buildTDengineold() { -# print_color "$GREEN" "TDengine build start" - -# [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug -# cd $TDENGINE_DIR/debug - -# if [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then -# print_color "$GREEN" "rebuild.." -# rm -rf * -# makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" -# print_color "$GREEN" "$makecmd" -# $makecmd -# make -j 8 install -# elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then -# print_color "$GREEN" "rebuild.." -# #rm -rf * -# makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" -# print_color "$GREEN" "$makecmd" -# $makecmd -# make -j 8 install -# elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then -# print_color "$GREEN" "not build,only install" -# cd $TDENGINE_DIR/debug -# make -j 8 install -# fi - -# print_color "$GREEN" "TDengine build end" -# } - - # Check and get the branch name and build branch if [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then branch="$BRANCH" @@ -163,12 +133,14 @@ elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then branch="$BRANCH" print_color "$GREEN" "Testing branch: $branch " print_color "$GREEN" "not build,only install!" - pullTDengine + cd "$TDENGINE_DIR/../" + git pull + cd "$TDENGINE_DIR/" + git pull cd $TDENGINE_DIR/debug make -j 8 install else print_color "$GREEN" "Build is not required for this test!" - pullTDengine fi function runCasesOneByOne () { From 1db40d873c4965c136816e9c910f53db285de36f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 12 Dec 2024 14:36:44 +0800 Subject: [PATCH 028/108] fix:[TS-5761] error for in operator --- source/libs/scalar/src/scalar.c | 3 ++ tests/parallel_test/cases.task | 2 + .../system-test/2-query/ts-5761-scalemode.py | 53 +++++++++++++++++++ tests/system-test/2-query/ts-5761.py | 52 ++++++++++++++++++ 4 files changed, 110 insertions(+) create mode 100644 tests/system-test/2-query/ts-5761-scalemode.py create mode 100644 tests/system-test/2-query/ts-5761.py diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b3610d035f..347bb3c7cd 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1718,6 +1718,9 @@ static int32_t sclGetCompOperatorResType(SOperatorNode *pOp) { return TSDB_CODE_TSC_INVALID_OPERATION; } ((SExprNode *)(pOp->pRight))->resType = ldt; + if (IS_NUMERIC_TYPE(ldt.type)){ + ((SExprNode *)(pOp->pRight))->resType.type = TSDB_DATA_TYPE_DOUBLE; + } } else if (nodesIsRegularOp(pOp)) { if (pOp->pRight == NULL) { return TSDB_CODE_TSC_INVALID_OPERATION; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 879d93ab3a..6ceb100169 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -209,6 +209,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5761.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5761-scalemode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5712.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 2 diff --git a/tests/system-test/2-query/ts-5761-scalemode.py b/tests/system-test/2-query/ts-5761-scalemode.py new file mode 100644 index 0000000000..24eafc6cc7 --- /dev/null +++ b/tests/system-test/2-query/ts-5761-scalemode.py @@ -0,0 +1,53 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + updatecfgDict = {'filterScalarMode':1} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + self.dbname = 'db' + self.stbname = 'st' + + def prepareData(self): + # db + tdSql.execute(f"create database db;") + tdSql.execute(f"use db") + + # super tableUNSIGNED + tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool);") + + # create index for all tags + tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1)") + + def check(self): + tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7)") + tdSql.checkRows(1) + + def run(self): + self.prepareData() + self.check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/ts-5761.py b/tests/system-test/2-query/ts-5761.py new file mode 100644 index 0000000000..565d2753d0 --- /dev/null +++ b/tests/system-test/2-query/ts-5761.py @@ -0,0 +1,52 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + self.dbname = 'db' + self.stbname = 'st' + + def prepareData(self): + # db + tdSql.execute(f"create database db;") + tdSql.execute(f"use db") + + # super tableUNSIGNED + tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool);") + + # create index for all tags + tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1)") + + def check(self): + tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7)") + tdSql.checkRows(1) + + def run(self): + self.prepareData() + self.check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From ce7c3f8d5fd7b8571fe02f6796d09ec38fda3b56 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 16:59:01 +0800 Subject: [PATCH 029/108] test:add local coverage script --- tests/run_local_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index c5ad72ce14..26cc6deb4d 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -311,7 +311,7 @@ function lcovFunc { '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.cc'\ '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ - '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c' '*/debug/*' '*/tests/*'\ --rc lcov_branch_coverage=1 -o coverage.info # generate result From d25901636a1a8f765a2f1a657c376024e44b0c12 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 12 Dec 2024 19:08:22 +0800 Subject: [PATCH 030/108] test:add local coverage script --- tests/run_local_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index 26cc6deb4d..30b0df3c1c 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -374,7 +374,7 @@ if [ ! -f "$COVERAGE_INFO" ]; then fi # Generate local HTML reports -genhtml "$COVERAGE_INFO" --output-directory "$OUTPUT_DIR" +genhtml "$COVERAGE_INFO" --branch-coverage --function-coverage --output-directory "$OUTPUT_DIR" # Check whether the report was generated successfully if [ $? -eq 0 ]; then From b5f2d4d38247c3542c423f33960c99d5384e39a3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 12 Dec 2024 19:37:34 +0800 Subject: [PATCH 031/108] test: for tsim --- source/os/test/osThreadTests.cpp | 61 +++++++++++++++++++++++- utils/tsim/CMakeLists.txt | 27 ++++++++--- utils/tsim/inc/simInt.h | 17 +++++-- utils/tsim/src/simEntry.c | 64 ++++++++++++++++++++++++++ utils/tsim/src/{simExe.c => simExec.c} | 4 +- utils/tsim/src/simMain.c | 55 +--------------------- utils/tsim/src/simSystem.c | 3 +- utils/tsim/test/CMakeLists.txt | 23 +++++++++ utils/tsim/test/simTests.cpp | 61 ++++++++++++++++++++++++ 9 files changed, 248 insertions(+), 67 deletions(-) create mode 100644 utils/tsim/src/simEntry.c rename utils/tsim/src/{simExe.c => simExec.c} (100%) create mode 100644 utils/tsim/test/CMakeLists.txt create mode 100644 utils/tsim/test/simTests.cpp diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index 622dcd38d7..076bf6427d 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -35,7 +35,7 @@ static void *funcPtr1(void *param) { return NULL; } -TEST(osThreadTests, invalidParameter) { +TEST(osThreadTests, para1) { TdThread tid1 = {0}; TdThread tid2 = {0}; int32_t reti = 0; @@ -192,3 +192,62 @@ TEST(osThreadTests, invalidParameter) { reti = taosThreadCondAttrDestroy(&condattr); EXPECT_EQ(reti, 0); } + +TEST(osThreadTests, option) { + int32_t reti = 0; + + TdThreadSpinlock lock = {0}; + reti = taosThreadSpinInit(&lock, -1); + EXPECT_NE(reti, 0); + reti = taosThreadSpinLock(&lock); + EXPECT_NE(reti, 0); + reti = taosThreadSpinTrylock(&lock); + EXPECT_NE(reti, 0); + reti = taosThreadSpinUnlock(&lock); + EXPECT_NE(reti, 0); + reti = taosThreadSpinDestroy(&lock); + EXPECT_NE(reti, 0); + + reti = taosThreadSpinInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadSpinLock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinTrylock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinDestroy(NULL); + EXPECT_NE(reti, 0); +} + +TEST(osThreadTests, splilock) { + int32_t reti = 0; + + TdThreadSpinlock lock = {0}; + reti = taosThreadSpinInit(&lock, -1); + EXPECT_NE(reti, 0); + reti = taosThreadSpinLock(&lock); + EXPECT_NE(reti, 0); + reti = taosThreadSpinTrylock(&lock); + EXPECT_NE(reti, 0); + reti = taosThreadSpinUnlock(&lock); + EXPECT_NE(reti, 0); + reti = taosThreadSpinDestroy(&lock); + EXPECT_NE(reti, 0); + + reti = taosThreadSpinInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadSpinLock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinTrylock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinDestroy(NULL); + EXPECT_NE(reti, 0); +} + +TEST(osThreadTests, others) { + taosThreadTestCancel(); + taosThreadClear(NULL); +} \ No newline at end of file diff --git a/utils/tsim/CMakeLists.txt b/utils/tsim/CMakeLists.txt index 209982c659..b725ed919a 100644 --- a/utils/tsim/CMakeLists.txt +++ b/utils/tsim/CMakeLists.txt @@ -1,14 +1,29 @@ -aux_source_directory(src TSIM_SRC) -add_executable(tsim ${TSIM_SRC}) -target_link_libraries( - tsim +LIST(APPEND TSIM_SRC src/simEntry.c) +LIST(APPEND TSIM_SRC src/simExec.c) +LIST(APPEND TSIM_SRC src/simParse.c) +LIST(APPEND TSIM_SRC src/simSystem.c) + +ADD_LIBRARY(tsim_static STATIC ${TSIM_SRC}) +TARGET_INCLUDE_DIRECTORIES( + tsim_static + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +TARGET_LINK_LIBRARIES( + tsim_static PUBLIC ${TAOS_LIB} PUBLIC util PUBLIC common PUBLIC os PUBLIC cjson ) -target_include_directories( + +LIST(APPEND TSIM_EXE_SRC src/simMain.c) +ADD_EXECUTABLE(tsim ${TSIM_EXE_SRC}) +TARGET_LINK_LIBRARIES( tsim - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC tsim_static ) + +IF(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +ENDIF(${BUILD_TEST}) diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h index f2360277e0..0ad0bfdea8 100644 --- a/utils/tsim/inc/simInt.h +++ b/utils/tsim/inc/simInt.h @@ -16,6 +16,10 @@ #ifndef _TD_SIM_INT_H_ #define _TD_SIM_INT_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" #include "cJSON.h" @@ -161,8 +165,8 @@ typedef struct _script_t { int32_t type; bool killed; void *taos; - char rows[12]; // number of rows data retrieved - char cols[12]; // number of columns data retrieved + char rows[12]; // number of rows data retrieved + char cols[12]; // number of columns data retrieved char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results char system_exit_code[12]; char system_ret_content[MAX_SYSTEM_RESULT_LEN]; @@ -192,7 +196,7 @@ SScript *simParseScript(char *fileName); SScript *simProcessCallOver(SScript *script); void *simExecuteScript(void *script); void simInitsimCmdList(); -bool simSystemInit(); +void simSystemInit(); void simSystemCleanUp(); char *simGetVariable(SScript *script, char *varName, int32_t varLen); bool simExecuteExpCmd(SScript *script, char *option); @@ -214,4 +218,11 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *option); bool simExecuteSetBIModeCmd(SScript *script, char *option); void simVisuallizeOption(SScript *script, char *src, char *dst); +int32_t simEntry(int32_t argc, char **argv); +void simHandleSignal(int32_t signo, void *sigInfo, void *context); + +#ifdef __cplusplus +} +#endif + #endif /*_TD_SIM_INT_H_*/ diff --git a/utils/tsim/src/simEntry.c b/utils/tsim/src/simEntry.c new file mode 100644 index 0000000000..dd11c21af0 --- /dev/null +++ b/utils/tsim/src/simEntry.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" + +bool simExecSuccess = false; +bool abortExecution = false; +bool useValgrind = false; + +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { + simSystemCleanUp(); + abortExecution = true; +} + +int32_t simEntry(int32_t argc, char **argv) { + char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; + + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { + tstrncpy(configDir, argv[++i], 128); + } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { + tstrncpy(scriptFile, argv[++i], MAX_FILE_NAME_LEN); + } else if (strcmp(argv[i], "-v") == 0) { + useValgrind = true; + } else { + printf("usage: %s [options] \n", argv[0]); + printf(" [-c config]: config directory, default is: %s\n", configDir); + printf(" [-f script]: script filename\n"); + return 0; + } + } + + simInfo("simulator is running ..."); + + simSystemInit(); + taosSetSignal(SIGINT, simHandleSignal); + + SScript *script = simParseScript(scriptFile); + if (script == NULL) { + simError("parse script file:%s failed", scriptFile); + return -1; + } + + simScriptList[++simScriptPos] = script; + simExecuteScript(script); + + int32_t ret = simExecSuccess ? 0 : -1; + simInfo("execute result %d", ret); + + return ret; +} diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExec.c similarity index 100% rename from utils/tsim/src/simExe.c rename to utils/tsim/src/simExec.c index a9772addbb..82ce852b1e 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExec.c @@ -382,8 +382,8 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } -void simReplaceDirSep(char *buf) { #ifdef WINDOWS +void simReplaceDirSep(char *buf) { int i = 0; while (buf[i] != '\0') { if (buf[i] == '/') { @@ -391,8 +391,8 @@ void simReplaceDirSep(char *buf) { } i++; } -#endif } +#endif bool simReplaceStr(char *buf, char *src, char *dst) { bool replaced = false; diff --git a/utils/tsim/src/simMain.c b/utils/tsim/src/simMain.c index cd4a5117b2..0af1e2eec2 100644 --- a/utils/tsim/src/simMain.c +++ b/utils/tsim/src/simMain.c @@ -16,58 +16,7 @@ #define _DEFAULT_SOURCE #include "simInt.h" -bool simExecSuccess = false; -bool abortExecution = false; -bool useValgrind = false; - -void simHandleSignal(int32_t signo, void *sigInfo, void *context) { - simSystemCleanUp(); - abortExecution = true; -} - int32_t main(int32_t argc, char *argv[]) { - char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; - - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { - tstrncpy(configDir, argv[++i], 128); - } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { - tstrncpy(scriptFile, argv[++i], MAX_FILE_NAME_LEN); - } else if (strcmp(argv[i], "-v") == 0) { - useValgrind = true; - } else { - printf("usage: %s [options] \n", argv[0]); - printf(" [-c config]: config directory, default is: %s\n", configDir); - printf(" [-f script]: script filename\n"); - return 0; - } - } - - if (!simSystemInit()) { - simError("failed to initialize the system"); - simSystemCleanUp(); - return -1; - } - - simInfo("simulator is running ..."); - taosSetSignal(SIGINT, simHandleSignal); - - SScript *script = simParseScript(scriptFile); - if (script == NULL) { - simError("parse script file:%s failed", scriptFile); - return -1; - } - - if (abortExecution) { - simError("execute abort"); - return -1; - } - - simScriptList[++simScriptPos] = script; - simExecuteScript(script); - - int32_t ret = simExecSuccess ? 0 : -1; - simInfo("execute result %d", ret); - - return ret; + // entry function used for unit testing. + return simEntry(argc, argv); } diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index dcf5d6ab12..e1dab0cd6c 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -35,11 +35,10 @@ int32_t simInitCfg() { return 0; } -bool simSystemInit() { +void simSystemInit() { simInitCfg(); simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); - return true; } void simSystemCleanUp() {} diff --git a/utils/tsim/test/CMakeLists.txt b/utils/tsim/test/CMakeLists.txt new file mode 100644 index 0000000000..5df85bf903 --- /dev/null +++ b/utils/tsim/test/CMakeLists.txt @@ -0,0 +1,23 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) + +IF(HEADER_GTEST_INCLUDE_DIR AND(LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) + MESSAGE(STATUS "gTest library found, build os test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) +ENDIF() + +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) + +ADD_EXECUTABLE(simTests "simTests.cpp") +TARGET_LINK_LIBRARIES(simTests os util tsim_static gtest_main) + +ADD_TEST( + NAME simTests + COMMAND simTests +) \ No newline at end of file diff --git a/utils/tsim/test/simTests.cpp b/utils/tsim/test/simTests.cpp new file mode 100644 index 0000000000..e728a1d4cd --- /dev/null +++ b/utils/tsim/test/simTests.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "simInt.h" + +void simHandleSignal(int32_t signo, void *sigInfo, void *context); + +TEST(simTests, parameters) { + int32_t ret = 0; + int32_t argc = 2; + char *argv[4] = {0}; + + simSystemCleanUp(); + // argv[1] = "-c"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // argv[1] = "-f"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // argv[1] = "-v"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // argv[1] = "-h"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // simHandleSignal(0, NULL, NULL); + + // simDebugFlag = 0; + // argc = 1; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, -1); +} From 58582d0910178834953a27708150236d08e8ff5e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 12 Dec 2024 19:46:54 +0800 Subject: [PATCH 032/108] fix:[TS-5761] error for in operator --- source/libs/scalar/src/filter.c | 15 ++++++++++++- source/libs/scalar/src/scalar.c | 3 --- .../system-test/2-query/ts-5761-scalemode.py | 22 +++++++++++-------- tests/system-test/2-query/ts-5761.py | 22 +++++++++++-------- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d8622d93ee..fa03f4ccd1 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4848,6 +4848,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { } SColumnNode *refNode = (SColumnNode *)node->pLeft; + int32_t type = refNode->node.resType.type; SExprNode *exprNode = NULL; if (OP_TYPE_IN != node->opType) { SValueNode *valueNode = (SValueNode *)node->pRight; @@ -4856,6 +4857,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { valueNode->node.resType.type = TSDB_DATA_TYPE_BIGINT; } exprNode = &valueNode->node; + type = vectorGetConvertType(refNode->node.resType.type, exprNode->resType.type); } else { SNodeListNode *listNode = (SNodeListNode *)node->pRight; if (LIST_LENGTH(listNode->pNodeList) > 10) { @@ -4863,8 +4865,19 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { return DEAL_RES_CONTINUE; } exprNode = &listNode->node; + SListCell *cell = listNode->pNodeList->pHead; + for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + cell = cell->pNext; + if (IS_NUMERIC_TYPE(valueNode->node.resType.type) && IS_NUMERIC_TYPE(type)) { + int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type); + if (tmp != 0){ + type = tmp; + } + } + } + exprNode->resType.type = type; } - int32_t type = vectorGetConvertType(refNode->node.resType.type, exprNode->resType.type); if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; return DEAL_RES_CONTINUE; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 347bb3c7cd..b3610d035f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1718,9 +1718,6 @@ static int32_t sclGetCompOperatorResType(SOperatorNode *pOp) { return TSDB_CODE_TSC_INVALID_OPERATION; } ((SExprNode *)(pOp->pRight))->resType = ldt; - if (IS_NUMERIC_TYPE(ldt.type)){ - ((SExprNode *)(pOp->pRight))->resType.type = TSDB_DATA_TYPE_DOUBLE; - } } else if (nodesIsRegularOp(pOp)) { if (pOp->pRight == NULL) { return TSDB_CODE_TSC_INVALID_OPERATION; diff --git a/tests/system-test/2-query/ts-5761-scalemode.py b/tests/system-test/2-query/ts-5761-scalemode.py index 24eafc6cc7..b47a43588e 100644 --- a/tests/system-test/2-query/ts-5761-scalemode.py +++ b/tests/system-test/2-query/ts-5761-scalemode.py @@ -1,4 +1,3 @@ - import taos from util.log import * @@ -22,21 +21,26 @@ class TDTestCase: tdSql.execute(f"use db") # super tableUNSIGNED - tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool);") + tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32));") # create index for all tags - tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1)") + tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1')") def check(self): - tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7, 2)") + tdSql.checkRows(1) + + tdSql.query(f"SELECT * FROM t1 WHERE c6 in (1.7, 2)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in (1, 2)") tdSql.checkRows(1) def run(self): @@ -50,4 +54,4 @@ class TDTestCase: tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/ts-5761.py b/tests/system-test/2-query/ts-5761.py index 565d2753d0..c833661c76 100644 --- a/tests/system-test/2-query/ts-5761.py +++ b/tests/system-test/2-query/ts-5761.py @@ -1,4 +1,3 @@ - import taos from util.log import * @@ -21,21 +20,26 @@ class TDTestCase: tdSql.execute(f"use db") # super tableUNSIGNED - tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool);") + tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32));") # create index for all tags - tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1)") + tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1')") def check(self): - tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7)") + tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7, 2)") + tdSql.checkRows(1) + + tdSql.query(f"SELECT * FROM t1 WHERE c6 in (1.7, 2)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in (1, 2)") tdSql.checkRows(1) def run(self): @@ -49,4 +53,4 @@ class TDTestCase: tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 1727e4fd3e52d4796baf7c2596391423e7d0b464 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 12 Dec 2024 23:33:05 +0800 Subject: [PATCH 033/108] fix:[TS-5761] error for in operator --- source/util/src/thashutil.c | 40 ++----------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index b466e1b351..5499a651c1 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -143,38 +143,6 @@ uint64_t MurmurHash3_64(const char *key, uint32_t len) { uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; } uint32_t taosIntHash_16(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint16_t *)key; } uint32_t taosIntHash_8(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint8_t *)key; } -uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { - float f = GET_FLOAT_VAL(key); - if (isnan(f)) { - return 0x7fc00000; - } - - if (FLT_EQUAL(f, 0.0)) { - return 0; - } - if (fabs(f) < FLT_MAX / BASE - DLT) { - int32_t t = (int32_t)(round(BASE * (f + DLT))); - return (uint32_t)t; - } else { - return 0x7fc00000; - } -} -uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { - double f = GET_DOUBLE_VAL(key); - if (isnan(f)) { - return 0x7fc00000; - } - - if (FLT_EQUAL(f, 0.0)) { - return 0; - } - if (fabs(f) < DBL_MAX / BASE - DLT) { - int32_t t = (int32_t)(round(BASE * (f + DLT))); - return (uint32_t)t; - } else { - return 0x7fc00000; - } -} uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) { uint64_t val = *(uint64_t *)key; @@ -190,6 +158,7 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_DOUBLE: fn = taosIntHash_64; break; case TSDB_DATA_TYPE_BINARY: @@ -200,6 +169,7 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { break; case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_FLOAT: fn = taosIntHash_32; break; case TSDB_DATA_TYPE_SMALLINT: @@ -211,12 +181,6 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break; - case TSDB_DATA_TYPE_FLOAT: - fn = taosFloatHash; - break; - case TSDB_DATA_TYPE_DOUBLE: - fn = taosDoubleHash; - break; default: fn = taosIntHash_32; break; From 99c352f175f2ed3cc1b754315e1ec449578a5369 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 13 Dec 2024 09:50:39 +0800 Subject: [PATCH 034/108] fix: timerange --- source/libs/scalar/src/filter.c | 34 ++++++- tests/system-test/2-query/operator.py | 127 ++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 10850aab9a..aaf05618ca 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ #include +#include "nodes.h" #include "os.h" #include "tglobal.h" #include "thash.h" @@ -4551,6 +4552,29 @@ static int32_t fltSclGetTimeStampDatum(SFltSclPoint *point, SFltSclDatum *d) { return TSDB_CODE_SUCCESS; } +typedef struct SRewriteGetInOperContext { + bool hasInOper; +} SRewriteGetInOperContext; + +static EDealRes rewriteInOperForTimerange(SNode **ppNode, void *pContext) { + SRewriteGetInOperContext *pCxt = pContext; + if (nodeType(*ppNode) == QUERY_NODE_OPERATOR && ((SOperatorNode *)(*ppNode))->opType == OP_TYPE_BIT_OR) { + return DEAL_RES_IGNORE_CHILD; + } + if (nodeType(*ppNode) == QUERY_NODE_OPERATOR && ((SOperatorNode *)(*ppNode))->opType == OP_TYPE_IN) { + pCxt->hasInOper = true; + return DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +bool hasAndTypeInOperator(SNode *pNode) { + SRewriteGetInOperContext cxt = {.hasInOper = false}; + nodesRewriteExpr(&pNode, rewriteInOperForTimerange, &cxt); + + return cxt.hasInOper; +} + int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFilterInfo *info = NULL; int32_t code = 0; @@ -4581,8 +4605,11 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { FLT_ERR_JRET(fltSclGetTimeStampDatum(endPt, &end)); win->skey = start.i; win->ekey = end.i; - if(optNode->opType == OP_TYPE_IN) *isStrict = false; - else *isStrict = true; + if (optNode->opType == OP_TYPE_IN || hasAndTypeInOperator(info->sclCtx.node)) { + *isStrict = false; + } else { + *isStrict = true; + } goto _return; } else if (taosArrayGetSize(points) == 0) { *win = TSWINDOW_DESC_INITIALIZER; @@ -5105,6 +5132,9 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(colRange->points); taosArrayDestroy(points); colRange->points = merged; + if(merged->size == 0) { + break; + } } else { taosArrayDestroy(colRange->points); colRange->points = points; diff --git a/tests/system-test/2-query/operator.py b/tests/system-test/2-query/operator.py index edfd023b7c..c0235d081d 100644 --- a/tests/system-test/2-query/operator.py +++ b/tests/system-test/2-query/operator.py @@ -95,12 +95,139 @@ class TDTestCase: tdSql.query(f"SELECT * FROM {self.dbname}.t1 WHERE c0 IN (-1) and c0 in(-1);") tdSql.checkRows(0) + def ts5760(self): + tdSql.execute(f"create database if not exists {self.dbname}") + + tdSql.execute(f"DROP TABLE IF EXISTS {self.dbname}.t1;") + tdSql.execute(f"CREATE TABLE {self.dbname}.t1( time TIMESTAMP, c0 INT);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c0) VALUES (1641024000000, 1);") + + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time - c0) > 0;") + tdSql.checkRows(1) + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + c0) > 0;") + tdSql.checkRows(1) + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (-(- c0)) > 0;") + tdSql.checkRows(1) + tdSql.query(f"SELECT time, c0 FROM t1 WHERE -(- c0) > 0;") + tdSql.checkRows(1) + tdSql.query(f"SELECT time, c0 FROM t1 WHERE -(- c0) < 0;") + tdSql.checkRows(0) + + tdSql.query(f"SELECT time, c0 FROM t1 WHERE -(- c0) = 0;") + tdSql.checkRows(0) + + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (- c0) > 0;") + tdSql.checkRows(0) + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (- c0) < 0;") + tdSql.checkRows(1) + + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + (- c0)) > 0;") + tdSql.checkRows(1) + + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + (- c0)) > 0;") + tdSql.checkRows(1) + + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time - (- (- c0)) ) > 0;") + tdSql.checkRows(1) + + tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + (-(- c0))) > 0;") + tdSql.checkRows(1) + + def ts5758(self): + tdSql.execute(f"create database if not exists {self.dbname}") + + tdSql.execute(f"DROP TABLE IF EXISTS {self.dbname}.t1;") + tdSql.execute(f"CREATE TABLE {self.dbname}.t1( time TIMESTAMP, c1 BIGINT);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000000, 0);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000001, 1);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000002, 2);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000003, 3);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000004, 4);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000005, 5);") + + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1) AND time BETWEEN (1741024000000) AND (1741024000000);") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time BETWEEN (1741024000000) AND (1741024000000) AND time IN (1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1) and time BETWEEN (1741024000000) AND (1741024000000) AND time IN (1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1641024000000) and time BETWEEN (1741024000000) AND (1741024000000) AND time IN (1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1) AND time BETWEEN (1641024000000) AND (1741024000000);") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1641024000001) AND time BETWEEN (1641024000000) AND (1741024000000);") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1641024000001, 1641024000002, 1641024000003) AND time BETWEEN (1641024000000) AND (1741024000000);") + tdSql.checkRows(3) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1641024000001, 1641024000002, 1641024000005) AND time BETWEEN (1641024000000) AND (1641024000004);") + tdSql.checkRows(2) + + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1) OR time = 1741024000000;") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time = 1741024000000 OR time IN (1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1, 2, 3) OR time BETWEEN (1641024000000) and (1741024000000);") + tdSql.checkRows(6) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1, 2, 3) OR time = 1641024000000;") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time = 1641024000001 OR time BETWEEN (1641024000000) and (1641024000002);") + tdSql.checkRows(3) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time = 1641024000004 OR time BETWEEN (1641024000000) and (1641024000002);") + tdSql.checkRows(4) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time = 1641024000001 OR time = 1741024000000;") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1641024000001, 1641024000002) OR time = 1741024000000;") + tdSql.checkRows(2) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE time IN (1641024000001, 1641024000002) OR time BETWEEN (1641024000000) and (1741024000000);") + tdSql.checkRows(6) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time = 1641024000004 OR time BETWEEN (1641024000000) and (1641024000002)) and time in(1);") + tdSql.checkRows(0) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time = 1641024000004 OR time BETWEEN (1641024000000) and (1641024000002)) and time in(1641024000004, 1641024000002);") + tdSql.checkRows(2) + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time = 1641024000004 OR time BETWEEN (1641024000000) and (1641024000002)) or time in(1);") + tdSql.checkRows(4) + + + def ts5759(self): + tdSql.execute(f"create database if not exists {self.dbname}") + + tdSql.execute(f"DROP TABLE IF EXISTS {self.dbname}.t1;") + tdSql.execute(f"CREATE TABLE {self.dbname}.t1( time TIMESTAMP, c1 BIGINT);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000000, 0);") + + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001)") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 < 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (3 < 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 < 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 > 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 > 2)") + tdSql.checkRows(0) + + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000001, 1);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000002, 2);") + + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 < 2)") + tdSql.checkRows(3) + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 > 2)") + tdSql.checkRows(2) + tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 < 2)") + tdSql.checkRows(2) + + def run(self): dbname = "db" tdSql.prepare() tdSql.execute(f"create database if not exists {self.dbname}") self.ts5757() + # self.ts5760() + self.ts5758() + self.ts5759() From b5fb388fec13d8d223c60e8c8d8788dcef089187 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 13 Dec 2024 09:52:23 +0800 Subject: [PATCH 035/108] revert minus restype --- source/libs/scalar/src/scalar.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 74ede393d8..b3610d035f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1673,8 +1673,8 @@ static int32_t sclGetMinusOperatorResType(SOperatorNode *pOp) { if (!IS_MATHABLE_TYPE(((SExprNode *)(pOp->pLeft))->resType.type)) { return TSDB_CODE_TSC_INVALID_OPERATION; } - pOp->node.resType.type = ((SExprNode *)(pOp->pLeft))->resType.type; - pOp->node.resType.bytes = tDataTypes[((SExprNode *)(pOp->pLeft))->resType.type].bytes; + pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; return TSDB_CODE_SUCCESS; } From 1e98daa25b86d543f514855185caa60242934f48 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Fri, 13 Dec 2024 14:44:15 +0800 Subject: [PATCH 036/108] coverage only branch --- tests/run_local_coverage_only_branch.sh | 388 ++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100755 tests/run_local_coverage_only_branch.sh diff --git a/tests/run_local_coverage_only_branch.sh b/tests/run_local_coverage_only_branch.sh new file mode 100755 index 0000000000..b889ecf528 --- /dev/null +++ b/tests/run_local_coverage_only_branch.sh @@ -0,0 +1,388 @@ +#!/bin/bash + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function print_color() { + local color="$1" + local message="$2" + echo -e "${color}${message}${NC}" +} + +# Initialization parameter +TDENGINE_DIR="/root/TDinternal/community" +BRANCH="" +TDENGINE_GCDA_DIR="/root/TDinternal/community/debug/" + +# Parse command line parameters +while getopts "hd:b:f:c:u:i:" arg; do + case $arg in + d) + TDENGINE_DIR=$OPTARG + ;; + b) + BRANCH=$OPTARG + ;; + f) + TDENGINE_GCDA_DIR=$OPTARG + ;; + c) + TEST_CASE=$OPTARG + ;; + u) + UNIT_TEST_CASE=$OPTARG + ;; + i) + BRANCH_BUILD=$OPTARG + ;; + h) + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test single case/all cases] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 0 + ;; + ?) + echo "Usage: ./$(basename $0) -h" + exit 1 + ;; + esac +done + +# Check if the command name is provided +if [ -z "$TDENGINE_DIR" ]; then + echo "Error: TDengine dir is required." + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test casingle case/all casesse] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 1 +fi + + +echo "TDENGINE_DIR = $TDENGINE_DIR" +today=`date +"%Y%m%d"` +TDENGINE_ALLCI_REPORT="$TDENGINE_DIR/tests/all-ci-report-$today.log" + +function pullTDengine() { + print_color "$GREEN" "TDengine pull start" + + # pull parent code + cd "$TDENGINE_DIR/../" + print_color "$GREEN" "git pull parent code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -dfx + git pull + + # pull tdengine code + cd $TDENGINE_DIR + print_color "$GREEN" "git pull tdengine code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -dfx + git pull + + print_color "$GREEN" "TDengine pull end" +} + +function buildTDengine() { + print_color "$GREEN" "TDengine build start" + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + print_color "$GREEN" "rebuild.." + rm -rf * + makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + $makecmd + make -j 8 install +} + +# Check and get the branch name and build branch +if [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + pullTDengine + buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + pullTDengine + buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "not build,only install!" + cd "$TDENGINE_DIR/../" + git pull + cd "$TDENGINE_DIR/" + git pull + cd $TDENGINE_DIR/debug + make -j 8 install +else + print_color "$GREEN" "Build is not required for this test!" +fi + +function runCasesOneByOne () { + while read -r line; do + if [[ "$line" != "#"* ]]; then + cmd=`echo $line | cut -d',' -f 5` + if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then + echo $cmd + case=`echo $cmd | cut -d' ' -f 3` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + + elif [[ "$line" == *"$2"* ]]; then + echo $cmd + if [[ "$cmd" == *"pytest.sh"* ]]; then + cmd=`echo $cmd | cut -d' ' -f 2-20` + fi + case=`echo $cmd | cut -d' ' -f 4-20` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + fi + fi + done < $1 +} + +function runUnitTest() { + print_color "$GREEN" "=== Run unit test case ===" + print_color "$GREEN" " $TDENGINE_DIR/debug" + cd $TDENGINE_DIR/debug + ctest -j12 + print_color "$GREEN" "3.0 unit test done" +} + +function runSimCases() { + print_color "$GREEN" "=== Run sim cases ===" + + cd $TDENGINE_DIR/tests/script + runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/longtimeruning_cases.task sim + + totalSuccess=`grep 'sim success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'sim failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + +function runPythonCases() { + print_color "$GREEN" "=== Run python cases ===" + + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' longtimeruning_cases.task + + # army + cd $TDENGINE_DIR/tests/army + runCasesOneByOne ../parallel_test/longtimeruning_cases.task army + + # system-test + cd $TDENGINE_DIR/tests/system-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task system-test + + # develop-test + cd $TDENGINE_DIR/tests/develop-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task develop-test + + totalSuccess=`grep 'py success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'py failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + + +function runTest_all() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + runUnitTest + runSimCases + runPythonCases + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + + +function runTest() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + if [ -n "$TEST_CASE" ] && [ "$TEST_CASE" != "all" ] && [ "$TEST_CASE" != "task" ]; then + TEST_CASE="$TEST_CASE" + print_color "$GREEN" "Test case: $TEST_CASE " + cd $TDENGINE_DIR/tests/script/ && $TEST_CASE + cd $TDENGINE_DIR/tests/army/ && $TEST_CASE + cd $TDENGINE_DIR/tests/system-test/ && $TEST_CASE + cd $TDENGINE_DIR/tests/develop-test/ && $TEST_CASE + elif [ "$TEST_CASE" == "all" ]; then + print_color "$GREEN" "Test case is : parallel_test/longtimeruning_cases.task and all unit cases" + runTest_all + elif [ "$TEST_CASE" == "task" ]; then + print_color "$GREEN" "Test case is only: parallel_test/longtimeruning_cases.task " + runSimCases + runPythonCases + elif [ -n "$UNIT_TEST_CASE" ]; then + UNIT_TEST_CASE="$UNIT_TEST_CASE" + cd $TDENGINE_DIR/debug/build/bin/ && $UNIT_TEST_CASE + else + print_color "$GREEN" "Test case is null" + fi + + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + if [ -n "$TDENGINE_GCDA_DIR" ]; then + TDENGINE_GCDA_DIR="$TDENGINE_GCDA_DIR" + print_color "$GREEN" "Test gcda file dir: $TDENGINE_GCDA_DIR " + else + print_color "$GREEN" "Test gcda file dir is default: /root/TDinternal/community/debug" + fi + + # collect data + lcov -d "$TDENGINE_GCDA_DIR" -capture --rc lcov_branch_coverage=0 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + lcov --remove coverage.info \ + '*/contrib/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c' '/*/enterprise/*' '*/docs/*' '*/sim/*'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/examples/*' '*/tidpool.c' '*/tmempool.c'\ + '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.cc' '*/branch/*'\ + '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ + '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c' '*/debug/*' '*/tests/*'\ + --rc lcov_branch_coverage=1 -o coverage.info + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function stopTaosd { + print_color "$GREEN" "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasod end" +} + +function stopTaosadapter { + print_color "$GREEN" "Stop taosadapter" + systemctl stop taosadapter.service + PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosadapter + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasoadapter end" + +} + +WORK_DIR=/root + +date >> $WORK_DIR/date.log +print_color "$GREEN" "Run local coverage test cases" | tee -a $WORK_DIR/date.log + +stopTaosd + +runTest + +lcovFunc + + +date >> $WORK_DIR/date.log +print_color "$GREEN" "End of local coverage test cases" | tee -a $WORK_DIR/date.log + + +# Define coverage information files and output directories +COVERAGE_INFO="$TDENGINE_DIR/coverage.info" +OUTPUT_DIR="$WORK_DIR/coverage_report" + +# Check whether the coverage information file exists +if [ ! -f "$COVERAGE_INFO" ]; then + echo "Error: $COVERAGE_INFO not found!" + exit 1 +fi + +# Generate local HTML reports +genhtml "$COVERAGE_INFO" --branch-coverage --function-coverage --output-directory "$OUTPUT_DIR" + +# Check whether the report was generated successfully +if [ $? -eq 0 ]; then + echo "HTML coverage report generated successfully in $OUTPUT_DIR" + echo "For more details : " + echo "http://192.168.1.61:7000/" +else + echo "Error generating HTML coverage report" + exit 1 +fi + From 3737ce4fd24dfc114875785e8f2c3f3be5f67b9e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 13 Dec 2024 14:56:57 +0800 Subject: [PATCH 037/108] enh: ut for os threads --- source/os/test/osThreadTests.cpp | 125 +++++++++++++++++++------------ 1 file changed, 76 insertions(+), 49 deletions(-) diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index 076bf6427d..5c0e40c3ca 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -29,38 +29,91 @@ #include "os.h" #include "tlog.h" -static void *funcPtr0(void *param) { return NULL; } -static void *funcPtr1(void *param) { - taosMsleep(1000); +static int32_t globalVar = 0; + +static void *funcPtr100(void *param) { + taosMsleep(100); return NULL; } -TEST(osThreadTests, para1) { - TdThread tid1 = {0}; - TdThread tid2 = {0}; - int32_t reti = 0; - TdThreadAttr attr = {0}; - TdThreadAttr attr2 = {0}; - int32_t param = 0; - (void)taosThreadAttrInit(&attr); +static void *funcPtr500(void *param) { + taosMsleep(500); + return NULL; +} - reti = taosThreadCreate(NULL, NULL, funcPtr0, NULL); +static void *funcPtrExit1(void *param) { + taosThreadExit(NULL); + return NULL; +} + +static void *funcPtrExit2(void *param) { + taosThreadExit(&globalVar); + return NULL; +} + +static void funcPtrKey(void *param) { taosMsleep(100); } + +TEST(osThreadTests, thread) { + TdThread tid1 = {0}; + TdThread tid2 = {0}; + int32_t reti = 0; + + reti = taosThreadCreate(NULL, NULL, funcPtr100, NULL); EXPECT_NE(reti, 0); reti = taosThreadCreate(&tid1, NULL, NULL, NULL); EXPECT_NE(reti, 0); - reti = taosThreadCreate(&tid1, NULL, funcPtr1, NULL); + reti = taosThreadCreate(&tid1, NULL, funcPtr100, NULL); EXPECT_EQ(reti, 0); reti = taosThreadCancel(tid1); + EXPECT_EQ(reti, 0); - reti = taosThreadCreate(&tid2, NULL, funcPtr0, NULL); + reti = taosThreadCreate(&tid2, NULL, funcPtr500, NULL); EXPECT_EQ(reti, 0); taosMsleep(1000); reti = taosThreadCancel(tid2); EXPECT_EQ(reti, 0); + taosThreadDetach(tid1); + reti = taosThreadCreate(&tid2, NULL, funcPtr500, NULL); + EXPECT_EQ(reti, 0); + reti = taosThreadDetach(tid2); + + reti = taosThreadEqual(tid1, tid2); + EXPECT_NE(reti, 0); + + reti = taosThreadCreate(&tid2, NULL, funcPtrExit1, NULL); + EXPECT_EQ(reti, 0); + reti = taosThreadCreate(&tid2, NULL, funcPtrExit2, NULL); + EXPECT_EQ(reti, 0); + + reti = taosThreadCreate(&tid2, NULL, funcPtr500, NULL); + EXPECT_EQ(reti, 0); + taosThreadKill(tid2, SIGINT); + + int32_t policy; + struct sched_param param; + taosThreadGetSchedParam(tid2, &policy, ¶m); + taosThreadGetSchedParam(tid2, NULL, ¶m); + taosThreadGetSchedParam(tid2, &policy, NULL); + + TdThreadKey key; + taosThreadKeyCreate(&key, funcPtrKey); + taosThreadGetSpecific(key); + taosThreadKeyDelete(key); +} + +TEST(osThreadTests, attr) { + int32_t reti = 0; + TdThreadAttr attr = {0}; + int32_t param = 0; + + reti = taosThreadAttrInit(NULL); + EXPECT_NE(reti, 0); reti = taosThreadAttrDestroy(NULL); EXPECT_NE(reti, 0); + (void)taosThreadAttrInit(&attr); + reti = taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_JOINABLE); EXPECT_EQ(reti, 0); reti = taosThreadAttrSetDetachState(&attr, -1); @@ -141,21 +194,22 @@ TEST(osThreadTests, para1) { EXPECT_NE(reti, 0); reti = taosThreadAttrGetStackSize(&attr, NULL); EXPECT_NE(reti, 0); +} - reti = taosThreadAttrInit(NULL); - EXPECT_NE(reti, 0); +TEST(osThreadTests, cond) { + int32_t reti = 0; - reti = taosThreadCondDestroy(NULL); - EXPECT_NE(reti, 0); reti = taosThreadCondInit(NULL, NULL); EXPECT_NE(reti, 0); + reti = taosThreadCondDestroy(NULL); + EXPECT_NE(reti, 0); reti = taosThreadCondSignal(NULL); EXPECT_NE(reti, 0); reti = taosThreadCondBroadcast(NULL); EXPECT_NE(reti, 0); - TdThreadCond cond; - TdThreadMutex mutex; + TdThreadCond cond{0}; + TdThreadMutex mutex = {0}; reti = taosThreadCondWait(&cond, NULL); EXPECT_NE(reti, 0); reti = taosThreadCondWait(NULL, &mutex); @@ -169,7 +223,7 @@ TEST(osThreadTests, para1) { reti = taosThreadCondTimedWait(&cond, &mutex, NULL); EXPECT_NE(reti, 0); - TdThreadCondAttr condattr; + TdThreadCondAttr condattr = {0}; (void)taosThreadCondAttrInit(&condattr); reti = taosThreadCondAttrInit(NULL); EXPECT_NE(reti, 0); @@ -193,34 +247,7 @@ TEST(osThreadTests, para1) { EXPECT_EQ(reti, 0); } -TEST(osThreadTests, option) { - int32_t reti = 0; - - TdThreadSpinlock lock = {0}; - reti = taosThreadSpinInit(&lock, -1); - EXPECT_NE(reti, 0); - reti = taosThreadSpinLock(&lock); - EXPECT_NE(reti, 0); - reti = taosThreadSpinTrylock(&lock); - EXPECT_NE(reti, 0); - reti = taosThreadSpinUnlock(&lock); - EXPECT_NE(reti, 0); - reti = taosThreadSpinDestroy(&lock); - EXPECT_NE(reti, 0); - - reti = taosThreadSpinInit(NULL, 0); - EXPECT_NE(reti, 0); - reti = taosThreadSpinLock(NULL); - EXPECT_NE(reti, 0); - reti = taosThreadSpinTrylock(NULL); - EXPECT_NE(reti, 0); - reti = taosThreadSpinUnlock(NULL); - EXPECT_NE(reti, 0); - reti = taosThreadSpinDestroy(NULL); - EXPECT_NE(reti, 0); -} - -TEST(osThreadTests, splilock) { +TEST(osThreadTests, spinlock) { int32_t reti = 0; TdThreadSpinlock lock = {0}; From bc4bd8d0591a8dad2ad50e82f818486205201984 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 13 Dec 2024 15:10:25 +0800 Subject: [PATCH 038/108] fix: oper on time type --- source/libs/scalar/src/scalar.c | 4 +- source/libs/scalar/src/sclvector.c | 6 +- tests/system-test/2-query/operator.py | 79 +++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b3610d035f..370ee4d6e9 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1688,8 +1688,8 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || TSDB_DATA_TYPE_VARBINARY == ldt.type || TSDB_DATA_TYPE_VARBINARY == rdt.type || - (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) { + (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type))) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type)))) { return TSDB_CODE_TSC_INVALID_OPERATION; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c6c8333392..0b232de70e 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1398,9 +1398,11 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BIGINT) || + if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) || + (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) || + (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BOOL) || (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && - GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BIGINT)) { // timestamp minus duration + GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BOOL)) { // timestamp minus duration int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; diff --git a/tests/system-test/2-query/operator.py b/tests/system-test/2-query/operator.py index c0235d081d..9b2589536a 100644 --- a/tests/system-test/2-query/operator.py +++ b/tests/system-test/2-query/operator.py @@ -218,16 +218,87 @@ class TDTestCase: tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 < 2)") tdSql.checkRows(2) + def operOnTime(self): + tdSql.execute(f"create database if not exists {self.dbname}") + + tdSql.execute(f"DROP TABLE IF EXISTS {self.dbname}.t1;") + tdSql.execute(f"CREATE TABLE {self.dbname}.t1( ts TIMESTAMP, c0 INT, c1 INT UNSIGNED, \ + c2 BIGINT, c3 BIGINT UNSIGNED, c4 SMALLINT, c5 SMALLINT UNSIGNED, c6 TINYINT, c7 TINYINT UNSIGNED);") + tdSql.execute(f"INSERT INTO {self.dbname}.t1 VALUES (1641024000001, 1, 1, 1, 1, 1, 1, 1, 1);") + + columns = ["c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7"] + for col in columns: + tdLog.debug(f"oper on time test, {col} start ...") + tdSql.query(f"SELECT ts, ts+1, ts+{col}, ts+(-{col}) FROM {self.dbname}.t1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000002) + tdSql.checkData(0, 2, 1641024000002) + tdSql.checkData(0, 3, 1641024000000) + + tdSql.query(f"SELECT ts, ts+1, ts+{col}, ts+(-{col}) FROM {self.dbname}.t1 where (ts-(-{col})) > 0") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000002) + tdSql.checkData(0, 2, 1641024000002) + tdSql.checkData(0, 3, 1641024000000) + + tdSql.query(f"SELECT ts, ts-1, ts-{col}, ts-(-{col}) FROM {self.dbname}.t1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000000) + tdSql.checkData(0, 2, 1641024000000) + tdSql.checkData(0, 3, 1641024000002) + + tdSql.query(f"SELECT ts, ts+true, ts-true, ts-false, ts+false FROM {self.dbname}.t1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000002) + tdSql.checkData(0, 2, 1641024000000) + tdSql.checkData(0, 3, 1641024000001) + tdSql.checkData(0, 4, 1641024000001) + + tdSql.execute(f"DROP TABLE IF EXISTS {self.dbname}.t2;") + tdSql.execute(f"CREATE TABLE {self.dbname}.t2( ts TIMESTAMP, c1 float, c2 double);") + tdSql.execute(f"INSERT INTO {self.dbname}.t2(ts, c1, c2) VALUES (1641024000001, 1.0, 1.0);") + + columns = ["c1", "c2"] + for col in columns: + tdSql.query(f"SELECT ts, ts+{col}, ts+(-{col}), ts-{col}, ts-(-{col}) FROM {self.dbname}.t2") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000002) + tdSql.checkData(0, 2, 1641024000000) + tdSql.checkData(0, 3, 1641024000000) + tdSql.checkData(0, 4, 1641024000002) + + tdSql.query(f"SELECT ts, ts+{col}, ts+(-{col}), ts-{col}, ts-(-{col}) FROM {self.dbname}.t2 where (ts-(-{col})) > 0") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000002) + tdSql.checkData(0, 2, 1641024000000) + tdSql.checkData(0, 3, 1641024000000) + tdSql.checkData(0, 4, 1641024000002) + + tdSql.query(f"SELECT ts, cast(ts+{col} as bigint), cast(ts+(-{col}) as bigint), cast(ts-{col} as bigint),\ + cast(ts-(-{col}) as bigint) FROM {self.dbname}.t2") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000002) + tdSql.checkData(0, 2, 1641024000000) + tdSql.checkData(0, 3, 1641024000000) + tdSql.checkData(0, 4, 1641024000002) def run(self): dbname = "db" tdSql.prepare() tdSql.execute(f"create database if not exists {self.dbname}") - self.ts5757() - # self.ts5760() - self.ts5758() - self.ts5759() + #self.ts5757() + #self.ts5760() + #self.ts5758() + #self.ts5759() + self.operOnTime() From f6b9bc44c78c608347cf023aa24d72bff672e3ab Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Wed, 11 Dec 2024 15:12:19 +0800 Subject: [PATCH 039/108] refactor stmt2_get_fields --- include/client/taos.h | 10 ++--- include/libs/parser/parser.h | 2 +- source/client/inc/clientStmt2.h | 2 +- source/client/src/clientMain.c | 50 ++++++++++----------- source/client/src/clientStmt2.c | 4 +- source/libs/parser/src/parInsertStmt.c | 14 +++--- tests/script/api/stmt2-example.c | 6 ++- tests/script/api/stmt2-get-fields.c | 12 ++--- tests/script/api/stmt2-nchar.c | 6 +-- tests/script/api/stmt2-nohole.c | 4 +- tests/script/api/stmt2-test.c | 61 ++++++++++++++------------ tests/script/api/stmt2.c | 4 +- 12 files changed, 89 insertions(+), 86 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index e59ea9d418..17f97d3d3d 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -111,14 +111,14 @@ typedef struct TAOS_FIELD_E { int32_t bytes; } TAOS_FIELD_E; -typedef struct TAOS_FIELD_STB { +typedef struct TAOS_FIELD_ALL { char name[65]; int8_t type; uint8_t precision; uint8_t scale; int32_t bytes; uint8_t field_type; -} TAOS_FIELD_STB; +} TAOS_FIELD_ALL; #ifdef WINDOWS #define DLL_EXPORT __declspec(dllexport) @@ -250,10 +250,8 @@ DLL_EXPORT int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows); DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); -DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt2_get_stb_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_STB **fields); -DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields); -DLL_EXPORT void taos_stmt2_free_stb_fields(TAOS_STMT2 *stmt, TAOS_FIELD_STB *fields); +DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields); +DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields); DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt); DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 82985457a8..95f522f504 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -150,7 +150,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bi int32_t colIdx, int32_t rowNum, void* charsetCxt); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtStbColFields(void* pBlock, void* boundTags, bool hasCtbName, int32_t* fieldNum, - TAOS_FIELD_STB** fields); + TAOS_FIELD_ALL** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt); diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 64abf31bc1..c42de739c6 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -222,7 +222,7 @@ int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags); int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx); int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields); -int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_STB **fields); +int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields); int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index d80c7df2e4..c98b89dad4 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2249,29 +2249,29 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) { return stmtIsInsert2(stmt, insert); } -int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) { - if (stmt == NULL || count == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } +// int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) { +// if (stmt == NULL || count == NULL) { +// tscError("NULL parameter for %s", __FUNCTION__); +// terrno = TSDB_CODE_INVALID_PARA; +// return terrno; +// } - if (field_type == TAOS_FIELD_COL) { - return stmtGetColFields2(stmt, count, fields); - } else if (field_type == TAOS_FIELD_TAG) { - return stmtGetTagFields2(stmt, count, fields); - } else if (field_type == TAOS_FIELD_QUERY) { - return stmtGetParamNum2(stmt, count); - } else if (field_type == TAOS_FIELD_TBNAME) { - return stmtGetParamTbName(stmt, count); - } else { - tscError("invalid parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -} +// if (field_type == TAOS_FIELD_COL) { +// return stmtGetColFields2(stmt, count, fields); +// } else if (field_type == TAOS_FIELD_TAG) { +// return stmtGetTagFields2(stmt, count, fields); +// } else if (field_type == TAOS_FIELD_QUERY) { +// return stmtGetParamNum2(stmt, count); +// } else if (field_type == TAOS_FIELD_TBNAME) { +// return stmtGetParamTbName(stmt, count); +// } else { +// tscError("invalid parameter for %s", __FUNCTION__); +// terrno = TSDB_CODE_INVALID_PARA; +// return terrno; +// } +// } -int taos_stmt2_get_stb_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_STB **fields) { +int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields) { if (stmt == NULL || count == NULL) { tscError("NULL parameter for %s", __FUNCTION__); terrno = TSDB_CODE_INVALID_PARA; @@ -2293,13 +2293,7 @@ int taos_stmt2_get_stb_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_STB **fie return stmtGetStbColFields2(stmt, count, fields); } -void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) { - (void)stmt; - if (!fields) return; - taosMemoryFree(fields); -} - -DLL_EXPORT void taos_stmt2_free_stb_fields(TAOS_STMT2 *stmt, TAOS_FIELD_STB *fields) { +DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields) { (void)stmt; if (!fields) return; taosMemoryFree(fields); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index f03879f199..bae9c6da88 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1070,7 +1070,7 @@ static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E return TSDB_CODE_SUCCESS; } -static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_STB** fields) { +static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_ALL** fields) { if (pStmt->errCode != TSDB_CODE_SUCCESS) { return pStmt->errCode; } @@ -1894,7 +1894,7 @@ int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) { return stmtFetchColFields2(stmt, nums, fields); } -int stmtGetStbColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_STB** fields) { +int stmtGetStbColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_ALL** fields) { int32_t code = stmtParseColFields2(stmt); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 9e6abe1517..fae5caaff0 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -938,13 +938,17 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc return TSDB_CODE_SUCCESS; } -int32_t buildStbBoundFields(SBoundColInfo boundColsInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_STB** fields, +int32_t buildStbBoundFields(SBoundColInfo boundColsInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_ALL** fields, STableMeta* pMeta, void* boundTags, bool preCtbname) { SBoundColInfo* tags = (SBoundColInfo*)boundTags; - int32_t numOfBound = boundColsInfo.numOfBound + (tags->mixTagsCols ? 0 : tags->numOfBound) + (preCtbname ? 1 : 0); + bool hastag = tags != NULL; + int32_t numOfBound = boundColsInfo.numOfBound + (preCtbname ? 1 : 0); + if (hastag) { + numOfBound += tags->mixTagsCols ? 0 : tags->numOfBound; + } int32_t idx = 0; if (fields != NULL) { - *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_STB)); + *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_ALL)); if (NULL == *fields) { return terrno; } @@ -957,7 +961,7 @@ int32_t buildStbBoundFields(SBoundColInfo boundColsInfo, SSchema* pSchema, int32 idx++; } - if (tags->numOfBound > 0 && !tags->mixTagsCols) { + if (hastag && tags->numOfBound > 0 && !tags->mixTagsCols) { SSchema* tagSchema = getTableTagSchema(pMeta); for (int32_t i = 0; i < tags->numOfBound; ++i) { @@ -1054,7 +1058,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel } int32_t qBuildStmtStbColFields(void* pBlock, void* boundTags, bool preCtbname, int32_t* fieldNum, - TAOS_FIELD_STB** fields) { + TAOS_FIELD_ALL** fields) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); if (pDataBlock->boundColsInfo.numOfBound <= 0) { diff --git a/tests/script/api/stmt2-example.c b/tests/script/api/stmt2-example.c index 692ff90a06..611de1ebc3 100644 --- a/tests/script/api/stmt2-example.c +++ b/tests/script/api/stmt2-example.c @@ -58,8 +58,8 @@ void do_stmt(TAOS* taos) { } int fieldNum = 0; - TAOS_FIELD_STB* pFields = NULL; - code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); + TAOS_FIELD_ALL* pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); if (code != 0) { printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); } else { @@ -70,6 +70,7 @@ void do_stmt(TAOS* taos) { } } + int64_t ts = 1591060628000; for (int i = 0; i < 2; ++i) { v.ts[i] = ts++; @@ -88,6 +89,7 @@ void do_stmt(TAOS* taos) { return; } + taos_stmt2_free_fields(stmt, pFields); taos_stmt2_close(stmt); } diff --git a/tests/script/api/stmt2-get-fields.c b/tests/script/api/stmt2-get-fields.c index df677eba1a..e14c0e2265 100755 --- a/tests/script/api/stmt2-get-fields.c +++ b/tests/script/api/stmt2-get-fields.c @@ -16,8 +16,8 @@ void getFields(TAOS *taos, const char *sql) { return; } int fieldNum = 0; - TAOS_FIELD_STB *pFields = NULL; - code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); + TAOS_FIELD_ALL *pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); if (code != 0) { printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); } else { @@ -28,7 +28,7 @@ void getFields(TAOS *taos, const char *sql) { } } printf("====================================\n"); - taos_stmt2_free_stb_fields(stmt, pFields); + taos_stmt2_free_fields(stmt, pFields); taos_stmt2_close(stmt); } @@ -42,15 +42,15 @@ void getQueryFields(TAOS *taos, const char *sql) { return; } int fieldNum = 0; - TAOS_FIELD_STB *pFields = NULL; - code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); + TAOS_FIELD_ALL *pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); if (code != 0) { printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); } else { printf("bind nums:%d\n", fieldNum); } printf("====================================\n"); - taos_stmt2_free_stb_fields(stmt, pFields); + taos_stmt2_free_fields(stmt, pFields); taos_stmt2_close(stmt); } diff --git a/tests/script/api/stmt2-nchar.c b/tests/script/api/stmt2-nchar.c index 3952bd5898..0f402517bf 100644 --- a/tests/script/api/stmt2-nchar.c +++ b/tests/script/api/stmt2-nchar.c @@ -205,11 +205,11 @@ void veriry_stmt(TAOS* taos) { return; } printf("param_count: %d\n", param_count); - */ - TAOS_FIELD_E* fields = NULL; + */ + TAOS_FIELD_ALL* fields = NULL; int field_count = -1; start = clock(); - code = taos_stmt2_get_fields(stmt, TAOS_FIELD_COL, &field_count, NULL); + code = taos_stmt2_get_fields(stmt, &field_count, &fields); end = clock(); printf("get fields time:%f\n", (double)(end - start) / CLOCKS_PER_SEC); if (code != 0) { diff --git a/tests/script/api/stmt2-nohole.c b/tests/script/api/stmt2-nohole.c index 5954f3660b..4bf0aae959 100644 --- a/tests/script/api/stmt2-nohole.c +++ b/tests/script/api/stmt2-nohole.c @@ -292,10 +292,10 @@ _bind_again: } printf("param_count: %d\n", param_count); */ - TAOS_FIELD_E* fields = NULL; + TAOS_FIELD_ALL* fields = NULL; int field_count = -1; start = clock(); - code = taos_stmt2_get_fields(stmt, TAOS_FIELD_TBNAME, &field_count, NULL); + code = taos_stmt2_get_fields(stmt, &field_count, &fields); end = clock(); printf("get fields time:%f\n", (double)(end - start) / CLOCKS_PER_SEC); if (code != 0) { diff --git a/tests/script/api/stmt2-test.c b/tests/script/api/stmt2-test.c index 2bbceec92e..d4e617ef66 100644 --- a/tests/script/api/stmt2-test.c +++ b/tests/script/api/stmt2-test.c @@ -35,8 +35,8 @@ typedef struct { } FuncInfo; typedef enum { - BP_BIND_TAG = 1, - BP_BIND_COL, + BP_BIND_COL = 1, + BP_BIND_TAG, } BP_BIND_TYPE; #define BP_BIND_TYPE_STR(t) (((t) == BP_BIND_COL) ? "column" : "tag") @@ -385,7 +385,7 @@ void *taosMemoryMalloc(uint64_t size) { return malloc(size); } void *taosMemoryCalloc(int32_t num, int32_t size) { return calloc(num, size); } void taosMemoryFree(const void *ptr) { - if (ptr == NULL) return; + if (ptr == NULL) return; return free((void *)ptr); } @@ -1303,23 +1303,23 @@ static int st_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows) { return taos_stm static int st_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) { return taos_stmt2_is_insert(stmt, insert); } -static int st_stmt2_num_params(TAOS_STMT2 *stmt, int *count) { - return taos_stmt2_get_fields(stmt, TAOS_FIELD_QUERY, count, NULL); -} +// static int st_stmt2_num_params(TAOS_STMT2 *stmt, int *count) { +// return taos_stmt2_get_fields(stmt, count, NULL); +// } -static int st_stmt2_get_tag_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields) { - return taos_stmt2_get_fields(stmt, TAOS_FIELD_TAG, count, fields); -} +// static int st_stmt2_get_tag_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields) { +// return taos_stmt2_get_fields(stmt, count, fields); +// } -static int st_stmt2_get_col_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields) { - return taos_stmt2_get_fields(stmt, TAOS_FIELD_COL, count, fields); -} +// static int st_stmt2_get_col_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_E **fields) { +// return taos_stmt2_get_fields(stmt, TAOS_FIELD_COL, count, fields); +// } static int st_stmt2_get_param(TAOS_STMT2 *stmt, int idx, int *type, int *bytes) { - int32_t code = 0, nums = 0; - TAOS_FIELD_E *fields = NULL; + int32_t code = 0, nums = 0; + TAOS_FIELD_ALL *fields = NULL; - code = taos_stmt2_get_fields(stmt, TAOS_FIELD_COL, &nums, &fields); + code = taos_stmt2_get_fields(stmt, &nums, &fields); if (code) { return code; } @@ -1398,8 +1398,8 @@ void bpCheckIsInsert(TAOS_STMT2 *stmt, int32_t insert) { void bpCheckParamNum(TAOS_STMT2 *stmt) { int32_t num = 0; - if (st_stmt2_num_params(stmt, &num)) { - printf("!!!st_stmt2_num_params error:%s\n", st_stmt2_error(stmt)); + if (taos_stmt2_get_fields(stmt, &num, NULL)) { + printf("!!!taos_stmt2_get_fields error:%s\n", st_stmt2_error(stmt)); exit(1); } @@ -1468,7 +1468,7 @@ void bpCheckQueryResult(TAOS_STMT2 *stmt, TAOS *taos, char *stmtSql, TAOS_STMT2_ printf("***sql res num match stmt res num %d\n", stmtResNum); } -void bpCheckColTagFields(TAOS_STMT2 *stmt, int32_t fieldNum, TAOS_FIELD_E *pFields, int32_t expecteNum, +void bpCheckColTagFields(TAOS_STMT2 *stmt, int32_t fieldNum, TAOS_FIELD_ALL *pFields, int32_t expecteNum, TAOS_STMT2_BIND *pBind, BP_BIND_TYPE type) { int32_t code = 0; @@ -1485,6 +1485,11 @@ void bpCheckColTagFields(TAOS_STMT2 *stmt, int32_t fieldNum, TAOS_FIELD_E *pFiel } for (int32_t i = 0; i < fieldNum; ++i) { + if (pFields[i].field_type != type) { + printf("!!!%s %dth field type %d mis-match expect type %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].field_type, + TAOS_FIELD_TAG); + exit(1); + } if (pFields[i].type != pBind[i].buffer_type) { printf("!!!%s %dth field type %d mis-match expect type %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].type, pBind[i].buffer_type); @@ -1545,17 +1550,17 @@ void bpCheckColTagFields(TAOS_STMT2 *stmt, int32_t fieldNum, TAOS_FIELD_E *pFiel } void bpCheckTagFields(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *pBind) { - int32_t code = 0; - int fieldNum = 0; - TAOS_FIELD_E *pFields = NULL; - code = st_stmt2_get_tag_fields(stmt, &fieldNum, &pFields); + int32_t code = 0; + int fieldNum = 0; + TAOS_FIELD_ALL *pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); if (code != 0) { - printf("!!!st_stmt2_get_tag_fields error:%s\n", st_stmt2_error(stmt)); + printf("!!!taos_stmt2_get_fields error:%s\n", st_stmt2_error(stmt)); exit(1); } bpCheckColTagFields(stmt, fieldNum, pFields, gCurCase->bindTagNum, pBind, BP_BIND_TAG); - taosMemoryFree(pFields); + taos_stmt2_free_fields(stmt, pFields); } void bpCheckColFields(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *pBind) { @@ -1563,10 +1568,10 @@ void bpCheckColFields(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *pBind) { return; } - int32_t code = 0; - int fieldNum = 0; - TAOS_FIELD_E *pFields = NULL; - code = st_stmt2_get_col_fields(stmt, &fieldNum, &pFields); + int32_t code = 0; + int fieldNum = 0; + TAOS_FIELD_ALL *pFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); if (code != 0) { printf("!!!st_stmt2_get_col_fields error:%s\n", st_stmt2_error(stmt)); exit(1); diff --git a/tests/script/api/stmt2.c b/tests/script/api/stmt2.c index 5b9f41baff..0ddff274a6 100644 --- a/tests/script/api/stmt2.c +++ b/tests/script/api/stmt2.c @@ -257,10 +257,10 @@ void veriry_stmt(TAOS* taos) { } printf("param_count: %d\n", param_count); */ - TAOS_FIELD_E* fields = NULL; + TAOS_FIELD_ALL* fields = NULL; int field_count = -1; start = clock(); - code = taos_stmt2_get_fields(stmt, TAOS_FIELD_COL, &field_count, NULL); + code = taos_stmt2_get_fields(stmt, &field_count, NULL); end = clock(); printf("get fields time:%f\n", (double)(end - start) / CLOCKS_PER_SEC); if (code != 0) { From 6a997eec630db5dabe25234aae339eea0b144950 Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Fri, 13 Dec 2024 17:02:02 +0800 Subject: [PATCH 040/108] fix refactor influnce stmt2-text.c --- tests/script/api/stmt2-test.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/tests/script/api/stmt2-test.c b/tests/script/api/stmt2-test.c index d4e617ef66..7edfd40d02 100644 --- a/tests/script/api/stmt2-test.c +++ b/tests/script/api/stmt2-test.c @@ -1328,9 +1328,18 @@ static int st_stmt2_get_param(TAOS_STMT2 *stmt, int idx, int *type, int *bytes) taosMemoryFree(fields); return -1; } + int i = 0; + for (; i < nums; i++) { + if (fields[i].field_type == TAOS_FIELD_COL) { + idx--; + } + if (idx < 0) { + break; + } + } - *type = fields[idx].type; - *bytes = fields[idx].bytes; + *type = fields[i].type; + *bytes = fields[i].bytes; taosMemoryFree(fields); return 0; @@ -1558,7 +1567,12 @@ void bpCheckTagFields(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *pBind) { printf("!!!taos_stmt2_get_fields error:%s\n", st_stmt2_error(stmt)); exit(1); } - + int n = fieldNum; + for (int i = 0; i < n; i++) { + if (pFields[i].field_type != TAOS_FIELD_TAG) { + fieldNum--; + } + } bpCheckColTagFields(stmt, fieldNum, pFields, gCurCase->bindTagNum, pBind, BP_BIND_TAG); taos_stmt2_free_fields(stmt, pFields); } @@ -1571,13 +1585,24 @@ void bpCheckColFields(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *pBind) { int32_t code = 0; int fieldNum = 0; TAOS_FIELD_ALL *pFields = NULL; + TAOS_FIELD_ALL *pColFields = NULL; + code = taos_stmt2_get_fields(stmt, &fieldNum, &pFields); if (code != 0) { printf("!!!st_stmt2_get_col_fields error:%s\n", st_stmt2_error(stmt)); exit(1); } - - bpCheckColTagFields(stmt, fieldNum, pFields, gCurCase->bindColNum, pBind, BP_BIND_COL); + int n = fieldNum; + pColFields = taosMemoryMalloc(fieldNum * sizeof(TAOS_FIELD_ALL)); + int j = 0; + for (int i = 0; i < n; i++) { + if (pFields[i].field_type != TAOS_FIELD_COL) { + fieldNum--; + } else { + pColFields[j++] = pFields[i]; + } + } + bpCheckColTagFields(stmt, fieldNum, pColFields, gCurCase->bindColNum, pBind, BP_BIND_COL); taosMemoryFree(pFields); } From eee8b99819195e6649fb23b8ad8462f29a8483f7 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 15 Dec 2024 23:40:20 +0800 Subject: [PATCH 041/108] fix: set strict --- source/libs/scalar/inc/filterInt.h | 1 + source/libs/scalar/src/filter.c | 114 ++++++++++++++++++++------ tests/system-test/2-query/operator.py | 8 +- 3 files changed, 92 insertions(+), 31 deletions(-) diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index ed07592c73..34f100a2bd 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -267,6 +267,7 @@ struct SFilterInfo { int8_t *blkUnitRes; void *pTable; SArray *blkList; + bool isStrict; SFilterPCtx pctx; }; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index aaf05618ca..8e548c6dcc 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4556,25 +4556,6 @@ typedef struct SRewriteGetInOperContext { bool hasInOper; } SRewriteGetInOperContext; -static EDealRes rewriteInOperForTimerange(SNode **ppNode, void *pContext) { - SRewriteGetInOperContext *pCxt = pContext; - if (nodeType(*ppNode) == QUERY_NODE_OPERATOR && ((SOperatorNode *)(*ppNode))->opType == OP_TYPE_BIT_OR) { - return DEAL_RES_IGNORE_CHILD; - } - if (nodeType(*ppNode) == QUERY_NODE_OPERATOR && ((SOperatorNode *)(*ppNode))->opType == OP_TYPE_IN) { - pCxt->hasInOper = true; - return DEAL_RES_END; - } - return DEAL_RES_CONTINUE; -} - -bool hasAndTypeInOperator(SNode *pNode) { - SRewriteGetInOperContext cxt = {.hasInOper = false}; - nodesRewriteExpr(&pNode, rewriteInOperForTimerange, &cxt); - - return cxt.hasInOper; -} - int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFilterInfo *info = NULL; int32_t code = 0; @@ -4605,11 +4586,7 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { FLT_ERR_JRET(fltSclGetTimeStampDatum(endPt, &end)); win->skey = start.i; win->ekey = end.i; - if (optNode->opType == OP_TYPE_IN || hasAndTypeInOperator(info->sclCtx.node)) { - *isStrict = false; - } else { - *isStrict = true; - } + *isStrict = info->isStrict; goto _return; } else if (taosArrayGetSize(points) == 0) { *win = TSWINDOW_DESC_INITIALIZER; @@ -5108,8 +5085,76 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { return TSDB_CODE_SUCCESS; } +static int32_t fltInOpertoArray(SArray **sclInOper, SFltSclOperator *pInOper) { + if (*sclInOper == NULL) { + *sclInOper = taosArrayInit(4, sizeof(SFltSclOperator *)); + if (NULL == *sclInOper) { + FLT_ERR_RET(terrno); + } + } + if (NULL == taosArrayPush(*sclInOper, &pInOper)) { + FLT_ERR_RET(terrno); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t hasValidInOper(SArray *sclInOper, SArray *colRangeList, bool* has) { + SFltSclOperator **ppSclOper = NULL; + *has = false; + for (int32_t i = 0; i < taosArrayGetSize(sclInOper); ++i) { + ppSclOper = taosArrayGet(sclInOper, i); + if (NULL == ppSclOper) { + return TSDB_CODE_OUT_OF_RANGE; + } + if(*ppSclOper == NULL) { + qError("func: hasValidInOper, invalid in operator"); + return TSDB_CODE_OUT_OF_RANGE; + } + SFltSclColumnRange *colRange = NULL; + SFltSclOperator * pSclOper = *ppSclOper; + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + colRange = taosArrayGet(colRangeList, i); + if (NULL == colRange) { + return TSDB_CODE_OUT_OF_RANGE; + } + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)pSclOper->colNode)) { + SFltSclPoint *startPt = taosArrayGet(colRange->points, 0); + SFltSclPoint *endPt = taosArrayGet(colRange->points, 1); + if (NULL == startPt || NULL == endPt) { + return TSDB_CODE_OUT_OF_RANGE; + } + SNodeListNode *listNode = (SNodeListNode *)pSclOper->valNode; + SListCell *cell = listNode->pNodeList->pHead; + for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + SFltSclDatum valDatum; + FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, valueNode)); + if (valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || + valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) { + if (startPt->val.d != endPt->val.d && (valDatum.d >= startPt->val.d || valDatum.d <= endPt->val.d)) { + *has = true; + return TSDB_CODE_SUCCESS; + } + } else { + if (startPt->val.d != endPt->val.d && (valDatum.i >= startPt->val.i || valDatum.i <= endPt->val.i)) { + *has = true; + return TSDB_CODE_SUCCESS; + } + } + cell = cell->pNext; + } + } + } + } + return TSDB_CODE_SUCCESS; +} + // TODO: process DNF composed of CNF -int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { +static int32_t fltSclProcessCNF(SFilterInfo *pInfo, SArray *sclOpListCNF, SArray *colRangeList) { + int32_t code = TSDB_CODE_SUCCESS; + + pInfo->isStrict = true; + SArray *sclInOper = NULL; size_t sz = taosArrayGetSize(sclOpListCNF); for (int32_t i = 0; i < sz; ++i) { SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); @@ -5133,12 +5178,27 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(points); colRange->points = merged; if(merged->size == 0) { - break; + goto _exit; } } else { taosArrayDestroy(colRange->points); colRange->points = points; } + if (sclOper->type == OP_TYPE_IN) { + fltInOpertoArray(&sclInOper, sclOper); + } + } + bool hasInOper = false; + code = hasValidInOper(sclInOper, colRangeList, &hasInOper); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + if (hasInOper) { + pInfo->isStrict = false; + } +_exit: + if (sclInOper) { + taosArrayDestroy(sclInOper); } return TSDB_CODE_SUCCESS; } @@ -5240,7 +5300,7 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) if (NULL == colRangeList) { FLT_ERR_JRET(terrno); } - FLT_ERR_JRET(fltSclProcessCNF(sclOpList, colRangeList)); + FLT_ERR_JRET(fltSclProcessCNF(pInfo, sclOpList, colRangeList)); pInfo->sclCtx.fltSclRange = colRangeList; for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { diff --git a/tests/system-test/2-query/operator.py b/tests/system-test/2-query/operator.py index 9b2589536a..d8cf2b54cb 100644 --- a/tests/system-test/2-query/operator.py +++ b/tests/system-test/2-query/operator.py @@ -294,10 +294,10 @@ class TDTestCase: tdSql.prepare() tdSql.execute(f"create database if not exists {self.dbname}") - #self.ts5757() - #self.ts5760() - #self.ts5758() - #self.ts5759() + self.ts5757() + self.ts5760() + self.ts5758() + self.ts5759() self.operOnTime() From a65825b88d11b045baff1acff3108c15aee15c65 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 16 Dec 2024 06:20:10 +0800 Subject: [PATCH 042/108] fix: return value --- source/libs/scalar/src/filter.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8e548c6dcc..8230119d8c 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -5159,44 +5159,45 @@ static int32_t fltSclProcessCNF(SFilterInfo *pInfo, SArray *sclOpListCNF, SArray for (int32_t i = 0; i < sz; ++i) { SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); if (NULL == sclOper) { - FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); + FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); } SFltSclColumnRange *colRange = NULL; - FLT_ERR_RET(fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList, &colRange)); + FLT_ERR_JRET(fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList, &colRange)); SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); if (NULL == points) { - FLT_ERR_RET(terrno); + FLT_ERR_JRET(terrno); } - FLT_ERR_RET(fltSclBuildRangePoints(sclOper, points)); + FLT_ERR_JRET(fltSclBuildRangePoints(sclOper, points)); if (taosArrayGetSize(colRange->points) != 0) { SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); if (NULL == merged) { - FLT_ERR_RET(terrno); + FLT_ERR_JRET(terrno); } - FLT_ERR_RET(fltSclIntersect(colRange->points, points, merged)); + FLT_ERR_JRET(fltSclIntersect(colRange->points, points, merged)); taosArrayDestroy(colRange->points); taosArrayDestroy(points); colRange->points = merged; if(merged->size == 0) { - goto _exit; + goto _return; } } else { taosArrayDestroy(colRange->points); colRange->points = points; } if (sclOper->type == OP_TYPE_IN) { - fltInOpertoArray(&sclInOper, sclOper); + code = fltInOpertoArray(&sclInOper, sclOper); + FLT_ERR_JRET(code); } } bool hasInOper = false; code = hasValidInOper(sclInOper, colRangeList, &hasInOper); if (code != TSDB_CODE_SUCCESS) { - goto _exit; + goto _return; } if (hasInOper) { pInfo->isStrict = false; } -_exit: +_return: if (sclInOper) { taosArrayDestroy(sclInOper); } From a18fefc968a8238da2973f2879665ae41ca1cf1d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 16 Dec 2024 08:59:43 +0800 Subject: [PATCH 043/108] fix: time operator test --- tests/system-test/2-query/Now.py | 6 ++++- tests/system-test/2-query/Today.py | 2 +- tests/system-test/2-query/operator.py | 35 +++++++++++++++++++++++++++ tests/system-test/2-query/sum.py | 2 +- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 21ff1f4e06..65734a5566 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -41,8 +41,9 @@ class TDTestCase: ] self.time_unit = ['b','u','a','s','m','h','d','w'] self.symbol = ['+','-','*','/'] - self.error_values = [1.5,'abc','"abc"','!@','today()'] + self.error_values = ['abc','"abc"','!@','today()'] self.db_percision = ['ms','us','ns'] + self.test_values = [1.5, 10, 9223372036854775807, -9223372036854775808] def tbtype_check(self,tb_type): if tb_type == 'normal table' or tb_type == 'child table': tdSql.checkRows(len(self.values_list)) @@ -67,6 +68,9 @@ class TDTestCase: for symbol in self.symbol: for param in self.error_values: tdSql.error(f'select now() {symbol}{param} from {tbname}') + for param in self.test_values: + tdSql.query(f'select now() {symbol}{param} from {tbname}') + tdSql.query(f'select 1 {symbol}{param} from {tbname}') tdSql.query(f'select now(){symbol}null from {tbname}') self.tbtype_check(tb_type) for i in range(len(self.values_list)): diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 77e6bd8cb6..745ed31c2c 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -18,7 +18,7 @@ class TDTestCase: self.today_ts = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").timestamp() self.today_ts_ns = 0 self.time_unit = ['b','u','a','s','m','h','d','w'] - self.error_param = ['1.5','abc','!@#','"abc"','today()'] + self.error_param = ['abc','!@#','"abc"','today()'] self.arithmetic_operators = ['+','-','*','/'] self.relational_operator = ['<','<=','=','>=','>'] # prepare data diff --git a/tests/system-test/2-query/operator.py b/tests/system-test/2-query/operator.py index d8cf2b54cb..3fd5a10ab6 100644 --- a/tests/system-test/2-query/operator.py +++ b/tests/system-test/2-query/operator.py @@ -288,6 +288,41 @@ class TDTestCase: tdSql.checkData(0, 2, 1641024000000) tdSql.checkData(0, 3, 1641024000000) tdSql.checkData(0, 4, 1641024000002) + + tdSql.query(f"SELECT sum(ts + c1), sum(ts+c2) from t2") + tdSql.checkData(0, 0, 1641024000002) + tdSql.checkData(0, 1, 1641024000002) + tdSql.query(f"SELECT sum(ts * c1), sum(ts*c2) from t2") + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000001) + tdSql.query(f"SELECT sum(ts / c1), sum(ts/c2) from t2") + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000001) + tdSql.execute(f"INSERT INTO {self.dbname}.t2(ts, c1, c2) VALUES (1641024000002, 2.0, 2.0);") + tdSql.query(f"SELECT sum(ts + c1), sum(ts+c2) from t2") + tdSql.checkData(0, 0, 3282048000006) + tdSql.checkData(0, 1, 3282048000006) + tdSql.query(f"SELECT sum(ts - c1), sum(ts-c2) from t2") + tdSql.checkData(0, 0, 3282048000000) + tdSql.checkData(0, 1, 3282048000000) + tdSql.query(f"SELECT sum(ts * c1), sum(ts*c2) from t2") + tdSql.checkData(0, 0, 4923072000005) + tdSql.checkData(0, 1, 4923072000005) + tdSql.query(f"SELECT ts / c1, ts/c2 from t2 order by ts") + tdSql.checkData(0, 0, 1641024000001) + tdSql.checkData(0, 1, 1641024000001) + tdSql.checkData(1, 0, 820512000001) + tdSql.checkData(1, 1, 820512000001) + tdSql.query(f"SELECT sum(ts / c1), sum(ts/c2) from t2") + tdSql.checkData(0, 0, 2461536000002) + tdSql.checkData(0, 1, 2461536000002) + + # data overflow + tdSql.query(f"SELECT ts + 9223372036854775807 from t2 order by ts") + tdSql.query(f"SELECT ts - 9223372036854775808 from t2 order by ts") + + tdSql.query(f"SELECT ts + 8223372036854775807 from t2 order by ts") + tdSql.query(f"SELECT ts - 8223372036854775808 from t2 order by ts") def run(self): dbname = "db" diff --git a/tests/system-test/2-query/sum.py b/tests/system-test/2-query/sum.py index 5abd58d3f9..88f2a37aec 100644 --- a/tests/system-test/2-query/sum.py +++ b/tests/system-test/2-query/sum.py @@ -78,7 +78,7 @@ class TDTestCase: ) # sqls.extend( f"select sum( {un_num_col} + {un_num_col_2} ) from {tbanme} " for un_num_col_2 in UN_NUM_COL ) - sqls.extend( f"select sum( {num_col} + {ts_col} ) from {DBNAME}.{tbanme} " for num_col in NUM_COL for ts_col in TS_TYPE_COL) + #sqls.extend( f"select sum( {num_col} + {ts_col} ) from {DBNAME}.{tbanme} " for num_col in NUM_COL for ts_col in TS_TYPE_COL) sqls.extend( ( f"select sum() from {DBNAME}.{tbanme} ", From ff53c4bc5047955a06f2b6b068b19ab3bdaed4c6 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 16 Dec 2024 09:04:50 +0800 Subject: [PATCH 044/108] fix: test case --- tests/system-test/2-query/operator.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/system-test/2-query/operator.py b/tests/system-test/2-query/operator.py index 3fd5a10ab6..ad9387deb5 100644 --- a/tests/system-test/2-query/operator.py +++ b/tests/system-test/2-query/operator.py @@ -289,40 +289,40 @@ class TDTestCase: tdSql.checkData(0, 3, 1641024000000) tdSql.checkData(0, 4, 1641024000002) - tdSql.query(f"SELECT sum(ts + c1), sum(ts+c2) from t2") + tdSql.query(f"SELECT sum(ts + c1), sum(ts+c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 1641024000002) tdSql.checkData(0, 1, 1641024000002) - tdSql.query(f"SELECT sum(ts * c1), sum(ts*c2) from t2") + tdSql.query(f"SELECT sum(ts * c1), sum(ts*c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 1641024000001) tdSql.checkData(0, 1, 1641024000001) - tdSql.query(f"SELECT sum(ts / c1), sum(ts/c2) from t2") + tdSql.query(f"SELECT sum(ts / c1), sum(ts/c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 1641024000001) tdSql.checkData(0, 1, 1641024000001) tdSql.execute(f"INSERT INTO {self.dbname}.t2(ts, c1, c2) VALUES (1641024000002, 2.0, 2.0);") - tdSql.query(f"SELECT sum(ts + c1), sum(ts+c2) from t2") + tdSql.query(f"SELECT sum(ts + c1), sum(ts+c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 3282048000006) tdSql.checkData(0, 1, 3282048000006) - tdSql.query(f"SELECT sum(ts - c1), sum(ts-c2) from t2") + tdSql.query(f"SELECT sum(ts - c1), sum(ts-c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 3282048000000) tdSql.checkData(0, 1, 3282048000000) - tdSql.query(f"SELECT sum(ts * c1), sum(ts*c2) from t2") + tdSql.query(f"SELECT sum(ts * c1), sum(ts*c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 4923072000005) tdSql.checkData(0, 1, 4923072000005) - tdSql.query(f"SELECT ts / c1, ts/c2 from t2 order by ts") + tdSql.query(f"SELECT ts / c1, ts/c2 from t2 order by {self.dbname}.ts") tdSql.checkData(0, 0, 1641024000001) tdSql.checkData(0, 1, 1641024000001) tdSql.checkData(1, 0, 820512000001) tdSql.checkData(1, 1, 820512000001) - tdSql.query(f"SELECT sum(ts / c1), sum(ts/c2) from t2") + tdSql.query(f"SELECT sum(ts / c1), sum(ts/c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 2461536000002) tdSql.checkData(0, 1, 2461536000002) # data overflow - tdSql.query(f"SELECT ts + 9223372036854775807 from t2 order by ts") - tdSql.query(f"SELECT ts - 9223372036854775808 from t2 order by ts") + tdSql.query(f"SELECT ts + 9223372036854775807 from {self.dbname}.t2 order by ts") + tdSql.query(f"SELECT ts - 9223372036854775808 from {self.dbname}.t2 order by ts") - tdSql.query(f"SELECT ts + 8223372036854775807 from t2 order by ts") - tdSql.query(f"SELECT ts - 8223372036854775808 from t2 order by ts") + tdSql.query(f"SELECT ts + 8223372036854775807 from {self.dbname}.t2 order by ts") + tdSql.query(f"SELECT ts - 8223372036854775808 from {self.dbname}.t2 order by ts") def run(self): dbname = "db" From c94a3f85e47227b7b1f744ec9e4e82b270be0193 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 16 Dec 2024 10:48:57 +0800 Subject: [PATCH 045/108] fix: mathOperatorRestype --- source/libs/scalar/inc/sclInt.h | 4 ++++ source/libs/scalar/src/scalar.c | 5 +---- source/libs/scalar/src/sclvector.c | 23 ++++++++++++----------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 34fd5dc8b0..30efb853f0 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -21,6 +21,8 @@ extern "C" { #include "query.h" #include "tcommon.h" #include "thash.h" +#include "querynodes.h" +#include "function.h" typedef struct SOperatorValueType { int32_t opResType; @@ -147,6 +149,8 @@ int32_t vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarPara int32_t _ord, int32_t optr); int32_t vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr); +bool checkOperatorRestypeIsTimestamp(int32_t ldt, int32_t rdt); + #ifdef __cplusplus } #endif diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 370ee4d6e9..8e308e9db7 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1693,10 +1693,7 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { return TSDB_CODE_TSC_INVALID_OPERATION; } - if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) || - (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type)) { + if (checkOperatorRestypeIsTimestamp(ldt.type, rdt.type)) { pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } else { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 0b232de70e..0e8c82d8a7 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1261,12 +1261,7 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p SColumnInfoData *pRightCol = NULL; SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - - if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) || - (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) || - (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BOOL) || - (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && - GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BOOL)) { // timestamp plus duration + if(checkOperatorRestypeIsTimestamp(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp plus duration int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; @@ -1398,11 +1393,7 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) || - (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) || - (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BOOL) || - (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && - GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BOOL)) { // timestamp minus duration + if (checkOperatorRestypeIsTimestamp(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp minus duration int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; @@ -2305,3 +2296,13 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return NULL; } } + +bool checkOperatorRestypeIsTimestamp(int32_t lType, int32_t rType) { + if ((TSDB_DATA_TYPE_TIMESTAMP == lType && IS_INTEGER_TYPE(rType) && rType != TSDB_DATA_TYPE_UBIGINT) || + (TSDB_DATA_TYPE_TIMESTAMP == rType && IS_INTEGER_TYPE(lType) && lType != TSDB_DATA_TYPE_UBIGINT) || + (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_BOOL == rType) || + (TSDB_DATA_TYPE_TIMESTAMP == rType && TSDB_DATA_TYPE_BOOL == lType)) { + return true; + } + return false; +} From c1d51cde99f7687c6c9efe3bdf35a3514505dd32 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Mon, 16 Dec 2024 11:01:28 +0800 Subject: [PATCH 046/108] coverage only branch --- tests/run_local_coverage_only_branch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run_local_coverage_only_branch.sh b/tests/run_local_coverage_only_branch.sh index b889ecf528..9329e448be 100755 --- a/tests/run_local_coverage_only_branch.sh +++ b/tests/run_local_coverage_only_branch.sh @@ -85,7 +85,7 @@ function pullTDengine() { git checkout -- . git checkout $branch git checkout -- . - git clean -dfx + git clean -f git pull # pull tdengine code @@ -96,7 +96,7 @@ function pullTDengine() { git checkout -- . git checkout $branch git checkout -- . - git clean -dfx + git clean -f git pull print_color "$GREEN" "TDengine pull end" From 7f218ab82bc54879186b2d69631508d509da48e0 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 16 Dec 2024 11:01:35 +0800 Subject: [PATCH 047/108] test case --- tests/system-test/2-query/Now.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 65734a5566..273bfaa9f5 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -68,13 +68,13 @@ class TDTestCase: for symbol in self.symbol: for param in self.error_values: tdSql.error(f'select now() {symbol}{param} from {tbname}') - for param in self.test_values: - tdSql.query(f'select now() {symbol}{param} from {tbname}') - tdSql.query(f'select 1 {symbol}{param} from {tbname}') tdSql.query(f'select now(){symbol}null from {tbname}') self.tbtype_check(tb_type) for i in range(len(self.values_list)): tdSql.checkData(i,0,None) + for param in self.test_values: + tdSql.query(f'select now() {symbol}{param} from {tbname}') + tdSql.query(f'select 1 {symbol}{param} from {tbname}') def now_check_ntb(self): for time_unit in self.db_percision: From d0f1be3a4b6f7887111cf9799be22ae114a38e11 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Mon, 16 Dec 2024 11:02:25 +0800 Subject: [PATCH 048/108] test:add local coverage script --- tests/run_local_coverage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh index 30b0df3c1c..b2413c4065 100755 --- a/tests/run_local_coverage.sh +++ b/tests/run_local_coverage.sh @@ -85,7 +85,7 @@ function pullTDengine() { git checkout -- . git checkout $branch git checkout -- . - git clean -dfx + git clean -f git pull # pull tdengine code @@ -96,7 +96,7 @@ function pullTDengine() { git checkout -- . git checkout $branch git checkout -- . - git clean -dfx + git clean -f git pull print_color "$GREEN" "TDengine pull end" From 779c1bd8add7b18a5df9508b1b11d5ba060dc4b9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 16 Dec 2024 13:53:13 +0800 Subject: [PATCH 049/108] refactor(stream): inject the reject dispatch error in stream to test the effect of fix. --- source/dnode/mnode/impl/src/mndStreamUtil.c | 4 ++-- source/libs/stream/src/streamDispatch.c | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index bb666eb6dd..941e7e53a2 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -1377,8 +1377,8 @@ int32_t mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, cons return TSDB_CODE_INVALID_PARA; } - pInfo->pUpdateNodeList = taosArrayInit(4, sizeof(SNodeUpdateInfo)), - pInfo->pDBMap = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + pInfo->pUpdateNodeList = taosArrayInit(4, sizeof(SNodeUpdateInfo)); + pInfo->pDBMap = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); if (pInfo->pUpdateNodeList == NULL || pInfo->pDBMap == NULL) { mndDestroyVgroupChangeInfo(pInfo); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 968744a0c5..0fc007a1fd 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1872,6 +1872,11 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } } +#if 0 + // inject errors, and always refuse the upstream dispatch msg and trigger the task nodeEpset update trans. + status = TASK_INPUT_STATUS__REFUSED; +#endif + { // do send response with the input status int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); From 07aea5e350e7a2ee35f8c5728b1227cf9da4805e Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 16 Dec 2024 14:11:27 +0800 Subject: [PATCH 050/108] fix: test case --- tests/parallel_test/cases.task | 4 +++ tests/system-test/2-query/Now.py | 2 +- tests/system-test/2-query/operator.py | 44 +++++++++++++-------------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 879d93ab3a..f6d2972805 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1071,6 +1071,10 @@ ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/operator.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/operator.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/operator.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/operator.py -Q 4 #tsim test ,,y,script,./test.sh -f tsim/query/timeline.sim diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 273bfaa9f5..9e7d240c0a 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -43,7 +43,7 @@ class TDTestCase: self.symbol = ['+','-','*','/'] self.error_values = ['abc','"abc"','!@','today()'] self.db_percision = ['ms','us','ns'] - self.test_values = [1.5, 10, 9223372036854775807, -9223372036854775808] + self.test_values = [1.5, 10] def tbtype_check(self,tb_type): if tb_type == 'normal table' or tb_type == 'child table': tdSql.checkRows(len(self.values_list)) diff --git a/tests/system-test/2-query/operator.py b/tests/system-test/2-query/operator.py index ad9387deb5..2e2f0af802 100644 --- a/tests/system-test/2-query/operator.py +++ b/tests/system-test/2-query/operator.py @@ -102,35 +102,35 @@ class TDTestCase: tdSql.execute(f"CREATE TABLE {self.dbname}.t1( time TIMESTAMP, c0 INT);") tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c0) VALUES (1641024000000, 1);") - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time - c0) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (time - c0) > 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + c0) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (time + c0) > 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (-(- c0)) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (-(- c0)) > 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE -(- c0) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE -(- c0) > 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE -(- c0) < 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE -(- c0) < 0;") tdSql.checkRows(0) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE -(- c0) = 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE -(- c0) = 0;") tdSql.checkRows(0) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (- c0) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (- c0) > 0;") tdSql.checkRows(0) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (- c0) < 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (- c0) < 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + (- c0)) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (time + (- c0)) > 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + (- c0)) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (time + (- c0)) > 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time - (- (- c0)) ) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (time - (- (- c0)) ) > 0;") tdSql.checkRows(1) - tdSql.query(f"SELECT time, c0 FROM t1 WHERE (time + (-(- c0))) > 0;") + tdSql.query(f"SELECT time, c0 FROM {self.dbname}.t1 WHERE (time + (-(- c0))) > 0;") tdSql.checkRows(1) def ts5758(self): @@ -195,27 +195,27 @@ class TDTestCase: tdSql.execute(f"CREATE TABLE {self.dbname}.t1( time TIMESTAMP, c1 BIGINT);") tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000000, 0);") - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001)") tdSql.checkRows(1) - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 < 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 < 2)") tdSql.checkRows(1) - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (3 < 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (3 < 2)") tdSql.checkRows(1) - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 < 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 < 2)") tdSql.checkRows(1) - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 > 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 > 2)") tdSql.checkRows(1) - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 > 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 > 2)") tdSql.checkRows(0) tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000001, 1);") tdSql.execute(f"INSERT INTO {self.dbname}.t1(time, c1) VALUES (1641024000002, 2);") - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 < 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 < 2)") tdSql.checkRows(3) - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 > 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) OR (1 > 2)") tdSql.checkRows(2) - tdSql.query(f"SELECT c1 FROM t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 < 2)") + tdSql.query(f"SELECT c1 FROM {self.dbname}.t1 WHERE (time BETWEEN 1641024000000 AND 1641024000001) and (1 < 2)") tdSql.checkRows(2) def operOnTime(self): @@ -308,7 +308,7 @@ class TDTestCase: tdSql.query(f"SELECT sum(ts * c1), sum(ts*c2) from {self.dbname}.t2") tdSql.checkData(0, 0, 4923072000005) tdSql.checkData(0, 1, 4923072000005) - tdSql.query(f"SELECT ts / c1, ts/c2 from t2 order by {self.dbname}.ts") + tdSql.query(f"SELECT ts / c1, ts/c2 from {self.dbname}.t2 order by ts") tdSql.checkData(0, 0, 1641024000001) tdSql.checkData(0, 1, 1641024000001) tdSql.checkData(1, 0, 820512000001) From b297aaaba138d71262bbc424e8c78240c20295d9 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 16 Dec 2024 14:31:00 +0800 Subject: [PATCH 051/108] fix null type caused query error and fix show create table crash --- source/common/src/ttypes.c | 4 +-- source/libs/catalog/inc/catalogInt.h | 2 ++ source/libs/catalog/src/ctgAsync.c | 52 ++++++++++++++++----------- source/libs/parser/src/parCalcConst.c | 7 +++- tests/script/tsim/show/basic.sim | 2 ++ tests/system-test/2-query/union.py | 11 ++++++ 6 files changed, 54 insertions(+), 24 deletions(-) diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 766e91f54e..1a0740b2b9 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -18,7 +18,7 @@ #include "tcompression.h" const int32_t TYPE_BYTES[21] = { - -1, // TSDB_DATA_TYPE_NULL + 2, // TSDB_DATA_TYPE_NULL CHAR_BYTES, // TSDB_DATA_TYPE_BOOL CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT SHORT_BYTES, // TSDB_DATA_TYPE_SMALLINT @@ -42,7 +42,7 @@ const int32_t TYPE_BYTES[21] = { }; tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { - {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL}, + {TSDB_DATA_TYPE_NULL, 6, 2, "NOTYPE", 0, 0, NULL, NULL}, {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool}, {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint}, {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index b581e31919..359bdc2b45 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -497,6 +497,8 @@ typedef struct SCtgAsyncFps { ctgDumpTaskResFp dumpResFp; ctgCompTaskFp compFp; ctgCloneTaskResFp cloneFp; + int32_t subTaskFactor; // to indicate how many sub tasks this task will generate by ctgLaunchSubTask + // default to 1, means no sub task, 2 means 1 sub task, 3 means 2 sub tasks... } SCtgAsyncFps; typedef struct SCtgApiStat { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 9bfb4102aa..78f88a5329 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -861,6 +861,16 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum + tbTsmaNum + tbNameNum; + int32_t taskNumWithSubTasks = tbMetaNum * gCtgAsyncFps[CTG_TASK_GET_TB_META].subTaskFactor + dbVgNum * gCtgAsyncFps[CTG_TASK_GET_DB_VGROUP].subTaskFactor + + udfNum * gCtgAsyncFps[CTG_TASK_GET_UDF].subTaskFactor + tbHashNum * gCtgAsyncFps[CTG_TASK_GET_TB_HASH].subTaskFactor + + qnodeNum * gCtgAsyncFps[CTG_TASK_GET_QNODE].subTaskFactor + dnodeNum * gCtgAsyncFps[CTG_TASK_GET_DNODE].subTaskFactor + + svrVerNum * gCtgAsyncFps[CTG_TASK_GET_SVR_VER].subTaskFactor + dbCfgNum * gCtgAsyncFps[CTG_TASK_GET_DB_CFG].subTaskFactor + + indexNum * gCtgAsyncFps[CTG_TASK_GET_INDEX_INFO].subTaskFactor + userNum * gCtgAsyncFps[CTG_TASK_GET_USER].subTaskFactor + + dbInfoNum * gCtgAsyncFps[CTG_TASK_GET_DB_INFO].subTaskFactor + tbIndexNum * gCtgAsyncFps[CTG_TASK_GET_TB_SMA_INDEX].subTaskFactor + + tbCfgNum * gCtgAsyncFps[CTG_TASK_GET_TB_CFG].subTaskFactor + tbTagNum * gCtgAsyncFps[CTG_TASK_GET_TB_TAG].subTaskFactor + + viewNum * gCtgAsyncFps[CTG_TASK_GET_VIEW].subTaskFactor + tbTsmaNum * gCtgAsyncFps[CTG_TASK_GET_TB_TSMA].subTaskFactor + + tsmaNum * gCtgAsyncFps[CTG_TASK_GET_TSMA].subTaskFactor + tbNameNum * gCtgAsyncFps[CTG_TASK_GET_TB_NAME].subTaskFactor; + *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { ctgError("failed to calloc, size:%d,QID:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId); @@ -905,7 +915,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const } #endif - pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask)); + pJob->pTasks = taosArrayInit(taskNumWithSubTasks, sizeof(SCtgTask)); if (NULL == pJob->pTasks) { ctgError("taosArrayInit %d tasks failed", taskNum); CTG_ERR_JRET(terrno); @@ -4166,27 +4176,27 @@ int32_t ctgCloneDbVg(SCtgTask* pTask, void** pRes) { } SCtgAsyncFps gCtgAsyncFps[] = { - {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, - {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, - {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, - {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, - {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, + {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL, 1}, + {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL, 1}, + {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg, 1}, + {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL, 1}, + {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL, 1}, {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, - ctgCloneTbMeta}, - {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, - {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, - {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, - {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, - {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, - {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, - {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, - {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL}, - {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, - {ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL}, - {ctgInitGetViewsTask, ctgLaunchGetViewsTask, ctgHandleGetViewsRsp, ctgDumpViewsRes, NULL, NULL}, - {ctgInitGetTbTSMATask, ctgLaunchGetTbTSMATask, ctgHandleGetTbTSMARsp, ctgDumpTbTSMARes, NULL, NULL}, - {ctgInitGetTSMATask, ctgLaunchGetTSMATask, ctgHandleGetTSMARsp, ctgDumpTSMARes, NULL, NULL}, - {ctgInitGetTbNamesTask, ctgLaunchGetTbNamesTask, ctgHandleGetTbNamesRsp, ctgDumpTbNamesRes, NULL, NULL}, + ctgCloneTbMeta, 1}, + {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL, 1}, + {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL, 1}, + {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL, 2}, + {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL, 1}, + {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL, 1}, + {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL, 2}, + {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL, 1}, + {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL, 1}, + {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL, 1}, + {ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL, 2}, + {ctgInitGetViewsTask, ctgLaunchGetViewsTask, ctgHandleGetViewsRsp, ctgDumpViewsRes, NULL, NULL, 1}, + {ctgInitGetTbTSMATask, ctgLaunchGetTbTSMATask, ctgHandleGetTbTSMARsp, ctgDumpTbTSMARes, NULL, NULL, 1}, + {ctgInitGetTSMATask, ctgLaunchGetTSMATask, ctgHandleGetTSMARsp, ctgDumpTSMARes, NULL, NULL, 1}, + {ctgInitGetTbNamesTask, ctgLaunchGetTbNamesTask, ctgHandleGetTbNamesRsp, ctgDumpTbNamesRes, NULL, NULL, 1}, }; int32_t ctgMakeAsyncRes(SCtgJob* pJob) { diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 79ad7ad6ed..c1c61911f3 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -608,7 +608,12 @@ static void resetProjectNullType(SNode* pStmt) { resetProjectNullTypeImpl(((SSelectStmt*)pStmt)->pProjectionList); break; case QUERY_NODE_SET_OPERATOR: { - resetProjectNullTypeImpl(((SSetOperator*)pStmt)->pProjectionList); + SSetOperator* pSetOp = (SSetOperator*)pStmt; + resetProjectNullTypeImpl(pSetOp->pProjectionList); + if (pSetOp->pLeft) + resetProjectNullType(pSetOp->pLeft); + if (pSetOp->pRight) + resetProjectNullType(pSetOp->pRight); break; } default: diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 4b2e33b45e..0b62debd4d 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -254,5 +254,7 @@ if $rows <= 0 then return -1 endi +system taos -P7100 -d db -s " show create table db.t0" + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index fc6dd4fb32..cd2432f21b 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -436,6 +436,17 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(47) + sql = "select null union select null" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + sql = "select null union all select null" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From 038e8bb844d416e7921130ee37d6d250b7c3b173 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Mon, 16 Dec 2024 14:31:45 +0800 Subject: [PATCH 052/108] test: test with taosadapter --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..524d44da73 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG enh/xftan/TD-33174-3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From b195b7fa124a155cea0949d7f1ef7ffb6fdb1520 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 16 Dec 2024 16:51:45 +0800 Subject: [PATCH 053/108] fix: multi/div with timestamp type --- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/src/scalar.c | 2 +- source/libs/scalar/src/sclvector.c | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 30efb853f0..b04e26ac5d 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -149,7 +149,7 @@ int32_t vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarPara int32_t _ord, int32_t optr); int32_t vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr); -bool checkOperatorRestypeIsTimestamp(int32_t ldt, int32_t rdt); +bool checkOperatorRestypeIsTimestamp(EOperatorType opType, int32_t ldt, int32_t rdt); #ifdef __cplusplus } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 8e308e9db7..5d89df1540 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1693,7 +1693,7 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { return TSDB_CODE_TSC_INVALID_OPERATION; } - if (checkOperatorRestypeIsTimestamp(ldt.type, rdt.type)) { + if (checkOperatorRestypeIsTimestamp(pOp->opType, ldt.type, rdt.type)) { pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } else { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 0e8c82d8a7..06830c780f 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -24,6 +24,7 @@ #include "tcompare.h" #include "tdatablock.h" #include "tdataformat.h" +#include "tdef.h" #include "ttime.h" #include "ttypes.h" #include "geosWrapper.h" @@ -1261,7 +1262,7 @@ int32_t vectorMathAdd(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p SColumnInfoData *pRightCol = NULL; SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - if(checkOperatorRestypeIsTimestamp(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp plus duration + if(checkOperatorRestypeIsTimestamp(OP_TYPE_ADD, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp plus duration int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; @@ -1393,7 +1394,7 @@ int32_t vectorMathSub(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p SCL_ERR_JRET(vectorConvertVarToDouble(pLeft, &leftConvert, &pLeftCol)); SCL_ERR_JRET(vectorConvertVarToDouble(pRight, &rightConvert, &pRightCol)); - if (checkOperatorRestypeIsTimestamp(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp minus duration + if (checkOperatorRestypeIsTimestamp(OP_TYPE_SUB, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight))) { // timestamp minus duration int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft; _getBigintValue_fn_t getVectorBigintValueFnRight; @@ -2297,7 +2298,10 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { } } -bool checkOperatorRestypeIsTimestamp(int32_t lType, int32_t rType) { +bool checkOperatorRestypeIsTimestamp(EOperatorType opType, int32_t lType, int32_t rType) { + if (opType != OP_TYPE_ADD && opType != OP_TYPE_SUB && opType != OP_TYPE_MINUS) { + return false; + } if ((TSDB_DATA_TYPE_TIMESTAMP == lType && IS_INTEGER_TYPE(rType) && rType != TSDB_DATA_TYPE_UBIGINT) || (TSDB_DATA_TYPE_TIMESTAMP == rType && IS_INTEGER_TYPE(lType) && lType != TSDB_DATA_TYPE_UBIGINT) || (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_BOOL == rType) || From 2fcef59962cdc01d5f2464fe0523eee662c5ee33 Mon Sep 17 00:00:00 2001 From: Pengrongkun Date: Mon, 16 Dec 2024 16:55:33 +0800 Subject: [PATCH 054/108] fix ci --- source/client/src/clientStmt2.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 39d222eb48..89a0fd3788 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -39,7 +39,7 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void** } static bool stmtDequeue(STscStmt2* pStmt, SStmtQNode** param) { - while (0 == atomic_load_64(&pStmt->queue.qRemainNum)) { + while (0 == atomic_load_64((int64_t*)&pStmt->queue.qRemainNum)) { taosUsleep(1); return false; } @@ -53,7 +53,7 @@ static bool stmtDequeue(STscStmt2* pStmt, SStmtQNode** param) { *param = node; - (void)atomic_sub_fetch_64(&pStmt->queue.qRemainNum, 1); + (void)atomic_sub_fetch_64((int64_t*)&pStmt->queue.qRemainNum, 1); return true; } @@ -63,7 +63,7 @@ static void stmtEnqueue(STscStmt2* pStmt, SStmtQNode* param) { pStmt->queue.tail = param; pStmt->stat.bindDataNum++; - (void)atomic_add_fetch_64(&pStmt->queue.qRemainNum, 1); + (void)atomic_add_fetch_64((int64_t*)&pStmt->queue.qRemainNum, 1); } static int32_t stmtCreateRequest(STscStmt2* pStmt) { From 7be31eea3062077ae9eddf24cd0d986092a64b20 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 16 Dec 2024 18:40:27 +0800 Subject: [PATCH 055/108] fix: query remain issues --- source/dnode/vnode/src/vnd/vnodeSvr.c | 14 +++++++++++++- source/libs/qworker/inc/qwInt.h | 3 +-- source/libs/qworker/src/qwMem.c | 19 ++++++++++++------- source/libs/qworker/src/qwMsg.c | 6 +++--- source/libs/qworker/src/qworker.c | 13 ++++++++++++- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d9b41869c7..3e3d91cb13 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -830,7 +830,7 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in vnode query queue is processing"); - if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME) && !syncIsReadyForRead(pVnode->sync)) { + if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } @@ -842,9 +842,21 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { SReadHandle handle = {.vnode = pVnode, .pMsgCb = &pVnode->msgCb, .pWorkerCb = pInfo->workerCb}; initStorageAPI(&handle.api); + int32_t code = TSDB_CODE_SUCCESS; + bool redirected = false; switch (pMsg->msgType) { case TDMT_SCH_QUERY: + if (!syncIsReadyForRead(pVnode->sync)) { + pMsg->code = (terrno) ? terrno : TSDB_CODE_SYN_NOT_LEADER; + redirected = true; + } + code = qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg, 0); + if (redirected) { + vnodeRedirectRpcMsg(pVnode, pMsg, pMsg->code); + return 0; + } + return code; case TDMT_SCH_MERGE_QUERY: return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg, 0); case TDMT_SCH_QUERY_CONTINUE: diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 53e1c3d288..65f7cd772f 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -320,9 +320,8 @@ extern SQueryMgmt gQueryMgmt; case QW_PHASE_POST_QUERY: \ case QW_PHASE_PRE_CQUERY: \ case QW_PHASE_POST_CQUERY: \ - atomic_store_8(&(ctx)->phase, _value); \ - break; \ default: \ + atomic_store_8(&(ctx)->phase, _value); \ break; \ } \ } while (0) diff --git a/source/libs/qworker/src/qwMem.c b/source/libs/qworker/src/qwMem.c index 89475bca13..d625bb113a 100644 --- a/source/libs/qworker/src/qwMem.c +++ b/source/libs/qworker/src/qwMem.c @@ -83,13 +83,16 @@ void qwDestroySession(QW_FPARAMS_DEF, SQWJobInfo *pJobInfo, void* session) { taosMemPoolDestroySession(gMemPoolHandle, session); + QW_LOCK(QW_WRITE, &pJobInfo->lock); int32_t remainSessions = atomic_sub_fetch_32(&pJobInfo->memInfo->remainSession, 1); - + if (remainSessions != 0) { + QW_UNLOCK(QW_WRITE, &pJobInfo->lock); + } + QW_TASK_DLOG("task session destoryed, remainSessions:%d", remainSessions); if (0 == remainSessions) { - QW_LOCK(QW_WRITE, &pJobInfo->lock); - if (/*0 == taosHashGetSize(pJobInfo->pSessions) && */0 == atomic_load_32(&pJobInfo->memInfo->remainSession)) { +// if (/*0 == taosHashGetSize(pJobInfo->pSessions) && */0 == atomic_load_32(&pJobInfo->memInfo->remainSession)) { atomic_store_8(&pJobInfo->destroyed, 1); QW_UNLOCK(QW_WRITE, &pJobInfo->lock); @@ -98,10 +101,10 @@ void qwDestroySession(QW_FPARAMS_DEF, SQWJobInfo *pJobInfo, void* session) { TAOS_UNUSED(taosHashRemove(gQueryMgmt.pJobInfo, id2, sizeof(id2))); QW_TASK_DLOG_E("the whole query job removed"); - } else { - QW_TASK_DLOG("job not removed, remainSessions:%d, %d", taosHashGetSize(pJobInfo->pSessions), pJobInfo->memInfo->remainSession); - QW_UNLOCK(QW_WRITE, &pJobInfo->lock); - } +// } else { +// QW_TASK_DLOG("job not removed, remainSessions:%d, %d", taosHashGetSize(pJobInfo->pSessions), pJobInfo->memInfo->remainSession); +// QW_UNLOCK(QW_WRITE, &pJobInfo->lock); +// } } } @@ -147,12 +150,14 @@ int32_t qwRetrieveJobInfo(QW_FPARAMS_DEF, SQWJobInfo** ppJob) { } QW_LOCK(QW_READ, &pJob->lock); + if (atomic_load_8(&pJob->destroyed)) { QW_UNLOCK(QW_READ, &pJob->lock); continue; } (void)atomic_add_fetch_32(&pJob->memInfo->remainSession, 1); + QW_UNLOCK(QW_READ, &pJob->lock); break; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 0452a05fe4..e385bbe5ea 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -498,14 +498,14 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int int64_t rId = msg.refId; int32_t eId = msg.execId; - SQWMsg qwMsg = {.node = node, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType}; + SQWMsg qwMsg = {.node = node, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType, .code = pMsg->code}; qwMsg.msgInfo.explain = msg.explain; qwMsg.msgInfo.taskType = msg.taskType; qwMsg.msgInfo.needFetch = msg.needFetch; qwMsg.msgInfo.compressMsg = msg.compress; - QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, compress:%d, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), - msg.compress, pMsg->info.handle, msg.sql); + QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, compress:%d, handle:%p, SQL:%s, code:0x%x", node, TMSG_INFO(pMsg->msgType), + msg.compress, pMsg->info.handle, msg.sql, qwMsg.code); code = qwProcessQuery(QW_FPARAMS(), &qwMsg, msg.sql); msg.sql = NULL; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 50af01cd6f..d38c8645d7 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -567,6 +567,13 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(ctx->rspCode); } + if (TSDB_CODE_SUCCESS != input->code) { + QW_TASK_ELOG("task already failed at phase %s, code:0x%x", qwPhaseStr(phase), input->code); + (void)qwDropTask(QW_FPARAMS()); + + QW_ERR_JRET(input->code); + } + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC, ctx->dynamicTask)); break; } @@ -631,6 +638,10 @@ _return: if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); + if (QW_PHASE_PRE_CQUERY == phase && code) { + QW_SET_PHASE(ctx, QW_PHASE_POST_CQUERY); + } + QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } @@ -767,7 +778,7 @@ _return: int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { int32_t code = 0; SSubplan *plan = NULL; - SQWPhaseInput input = {0}; + SQWPhaseInput input = {.code = qwMsg->code}; qTaskInfo_t pTaskInfo = NULL; DataSinkHandle sinkHandle = NULL; SQWTaskCtx *ctx = NULL; From 2464f1ba686a5a8096925ed79f6154acb1766ffa Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 16 Dec 2024 19:20:16 +0800 Subject: [PATCH 056/108] fix: void return code --- source/libs/qworker/src/qworker.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index d38c8645d7..fa422b1de4 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -819,10 +819,10 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_APP_ERROR); } - atomic_add_fetch_64(&gQueryMgmt.stat.taskRunNum, 1); + (void)atomic_add_fetch_64(&gQueryMgmt.stat.taskRunNum, 1); uint64_t flags = 0; - dsGetSinkFlags(sinkHandle, &flags); + (void)dsGetSinkFlags(sinkHandle, &flags); ctx->level = plan->level; ctx->dynamicTask = qIsDynamicExecTask(pTaskInfo); @@ -1353,7 +1353,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { ctx.sinkHandle = sinkHandle; uint64_t flags = 0; - dsGetSinkFlags(sinkHandle, &flags); + (void)dsGetSinkFlags(sinkHandle, &flags); ctx.sinkWithMemPool = flags & DS_FLAG_USE_MEMPOOL; From e4447e7bdcc532aac323d5f13a2f86ccdb1da5c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Dec 2024 19:22:25 +0800 Subject: [PATCH 057/108] fix:[TS-5761] error for in operator --- include/libs/function/function.h | 1 + include/libs/scalar/scalar.h | 2 +- source/libs/index/src/indexFilter.c | 2 +- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/src/filter.c | 18 +++-- source/libs/scalar/src/scalar.c | 30 ++++++-- source/libs/scalar/src/sclvector.c | 41 +++++------ tests/parallel_test/cases.task | 2 +- .../system-test/2-query/ts-5761-scalemode.py | 71 +++++++++++++++++-- tests/system-test/2-query/ts-5761.py | 71 +++++++++++++++++-- 10 files changed, 190 insertions(+), 50 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 51d9e752a4..62c837804e 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -288,6 +288,7 @@ struct SScalarParam { bool colAlloced; SColumnInfoData *columnData; SHashObj *pHashFilter; + SHashObj *pHashFilterVar; int32_t hashValueType; void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value int32_t numOfRows; diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 4b89a6a439..1d8c502a75 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -40,7 +40,7 @@ pDst need to freed in caller int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); int32_t scalarGetOperatorParamNum(EOperatorType type); -int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 02e5bd34a6..1d1bc66414 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -326,7 +326,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { indexError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->node.resType.type)); + SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->node.resType.type, 0)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); param->pFilter = NULL; diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 34fd5dc8b0..130dec8f12 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -141,7 +141,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_PRECISON(_c) ((_c)->columnData->info.precision) void sclFreeParam(SScalarParam* param); -int32_t doVectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, +int32_t doVectorCompare(SScalarParam* pLeft, SScalarParam *pLeftVar, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t _ord, int32_t optr); int32_t vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t _ord, int32_t optr); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index fa03f4ccd1..807fb5e237 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2210,7 +2210,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } if (unit->compare.optr == OP_TYPE_IN) { - FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type)); + FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type, 0)); if (fi->data == NULL) { fltError("failed to convert in param"); FLT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -4767,7 +4767,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { return DEAL_RES_CONTINUE; } - if (node->opType == OP_TYPE_NOT_IN || node->opType == OP_TYPE_NOT_LIKE || node->opType > OP_TYPE_IS_NOT_NULL || + if (node->opType == OP_TYPE_NOT_LIKE || node->opType > OP_TYPE_IS_NOT_NULL || node->opType == OP_TYPE_NOT_EQUAL) { stat->scalarMode = true; return DEAL_RES_CONTINUE; @@ -4841,7 +4841,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { } } - if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { + if ((OP_TYPE_IN == node->opType || OP_TYPE_NOT_IN == node->opType) && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight)); stat->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; @@ -4850,7 +4850,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { SColumnNode *refNode = (SColumnNode *)node->pLeft; int32_t type = refNode->node.resType.type; SExprNode *exprNode = NULL; - if (OP_TYPE_IN != node->opType) { + if (OP_TYPE_IN != node->opType && OP_TYPE_NOT_IN != node->opType) { SValueNode *valueNode = (SValueNode *)node->pRight; if (FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP) && TSDB_DATA_TYPE_UBIGINT == valueNode->node.resType.type && valueNode->datum.u <= INT64_MAX) { @@ -4869,16 +4869,14 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; cell = cell->pNext; - if (IS_NUMERIC_TYPE(valueNode->node.resType.type) && IS_NUMERIC_TYPE(type)) { - int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type); - if (tmp != 0){ - type = tmp; - } + int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type); + if (tmp != 0){ + type = tmp; } } exprNode->resType.type = type; } - if (0 != type && type != refNode->node.resType.type) { + if ((0 != type && type != refNode->node.resType.type) || OP_TYPE_NOT_IN == node->opType) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b3610d035f..ed50ccd37b 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -116,7 +116,8 @@ _return: SCL_RET(code); } -int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { +// processType = 0 means all type. 1 means number, 2 means var +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { sclError("taosHashInit failed, size:%d", 256); @@ -138,6 +139,19 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; + if (IS_VAR_DATA_TYPE(valueNode->node.resType.type)){ + if (processType == 1) { + cell = cell->pNext; + continue; + } + } else{ + if (processType == 2) + { + cell = cell->pNext; + continue; + } + } + if (valueNode->node.resType.type != type) { out.columnData->info.type = type; if (IS_VAR_DATA_TYPE(type)) { @@ -187,6 +201,9 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { cell = cell->pNext; } + if (taosHashGetSize(pObj) == 0) { + goto _return; + } *data = pObj; colDataDestroy(out.columnData); @@ -369,12 +386,14 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t type = vectorGetConvertType(ctx->type.selfType, ctx->type.peerType); - if (type == 0) { - type = nodeList->node.resType.type; + int32_t type = ctx->type.peerType; + if (IS_VAR_DATA_TYPE(ctx->type.selfType) && IS_NUMERIC_TYPE(ctx->type.peerType)){ + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 1)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterVar, node, ctx->type.selfType, 2)); + } else { + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 0)); } - SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type)); param->hashValueType = type; param->colAlloced = true; if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { @@ -587,7 +606,6 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); if (paramNum > 1) { - TSWAP(ctx->type.selfType, ctx->type.peerType); SCL_ERR_JRET(sclInitParam(node->pRight, ¶mList[1], ctx, rowNum)); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c6c8333392..b862f42a37 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1069,6 +1069,9 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows) { + if (input->columnData == NULL && input->pHashFilterVar != NULL){ + return TSDB_CODE_SUCCESS; + } output->numOfRows = input->numOfRows; SDataType t = {.type = type}; @@ -1099,36 +1102,18 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara int8_t type = 0; int32_t code = 0; - SScalarParam *param1 = NULL, *paramOut1 = NULL; - SScalarParam *param2 = NULL, *paramOut2 = NULL; + SScalarParam *param1 = pLeft, *paramOut1 = pLeftOut; + SScalarParam *param2 = pRight, *paramOut2 = pRightOut; // always convert least data if (IS_VAR_DATA_TYPE(leftType) && IS_VAR_DATA_TYPE(rightType) && (pLeft->numOfRows != pRight->numOfRows) && leftType != TSDB_DATA_TYPE_JSON && rightType != TSDB_DATA_TYPE_JSON) { - param1 = pLeft; - param2 = pRight; - paramOut1 = pLeftOut; - paramOut2 = pRightOut; - if (pLeft->numOfRows > pRight->numOfRows) { type = leftType; } else { type = rightType; } } else { - // we only define half value in the convert-matrix, so make sure param1 always less equal than param2 - if (leftType < rightType) { - param1 = pLeft; - param2 = pRight; - paramOut1 = pLeftOut; - paramOut2 = pRightOut; - } else { - param1 = pRight; - param2 = pLeft; - paramOut1 = pRightOut; - paramOut2 = pLeftOut; - } - type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2)); if (0 == type) { return TSDB_CODE_SUCCESS; @@ -1991,13 +1976,14 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa return code; } -int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, +int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t _ord, int32_t optr) { int32_t i = 0; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t lType = GET_PARAM_TYPE(pLeft); int32_t rType = GET_PARAM_TYPE(pRight); __compar_fn_t fp = NULL; + __compar_fn_t fpVar = NULL; int32_t compRows = 0; if (lType == rType) { SCL_ERR_RET(filterGetCompFunc(&fp, lType, optr)); @@ -2005,6 +1991,9 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam fp = filterGetCompFuncEx(lType, rType, optr); } + if (pLeftVar != NULL) { + SCL_ERR_RET(filterGetCompFunc(&fpVar, GET_PARAM_TYPE(pLeftVar), optr)); + } if (startIndex < 0) { i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); @@ -2024,6 +2013,10 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam char *pLeftData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter); + if (pLeftVar != NULL && !res){ + pLeftData = colDataGetData(pLeftVar->columnData, i); + res = res || filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterVar); + } colDataSetInt8(pOut->columnData, i, (int8_t *)&res); if (res) { pOut->numOfQualified++; @@ -2041,6 +2034,7 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara SScalarParam pRightOut = {0}; SScalarParam *param1 = NULL; SScalarParam *param2 = NULL; + SScalarParam *param3 = NULL; int32_t code = TSDB_CODE_SUCCESS; if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) { param1 = pLeft; @@ -2049,9 +2043,12 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara SCL_ERR_JRET(vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows)); param1 = (pLeftOut.columnData != NULL) ? &pLeftOut : pLeft; param2 = (pRightOut.columnData != NULL) ? &pRightOut : pRight; + if (pRight->pHashFilterVar != NULL){ + param3 = pLeft; + } } - SCL_ERR_JRET(doVectorCompare(param1, param2, pOut, startIndex, numOfRows, _ord, optr)); + SCL_ERR_JRET(doVectorCompare(param1, param3, param2, pOut, startIndex, numOfRows, _ord, optr)); _return: sclFreeParam(&pLeftOut); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6ceb100169..91254cbe2f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -705,7 +705,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py +,,n,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py diff --git a/tests/system-test/2-query/ts-5761-scalemode.py b/tests/system-test/2-query/ts-5761-scalemode.py index b47a43588e..c6fea20072 100644 --- a/tests/system-test/2-query/ts-5761-scalemode.py +++ b/tests/system-test/2-query/ts-5761-scalemode.py @@ -24,25 +24,88 @@ class TDTestCase: tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32));") # create index for all tags - tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1')") + tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1.7')") + tdSql.execute("INSERT INTO t1 VALUES (1641024000001, 0, 0, 1.7, 0, 0, '0', '')") + tdSql.execute("INSERT INTO t1 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") def check(self): + tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7") + tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)") tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c1 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c2 = 1.7") + tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7, 2)") tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c2 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c3 = 1.7") + tdSql.checkRows(1) tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c3 not in (1.7, 2)") + tdSql.checkRows(2) + + tdSql.query(f"SELECT * FROM t1 WHERE c4 = 1.7") tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7, 2)") - tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c4 not in (1.7, 2)") + tdSql.checkRows(3) + tdSql.query(f"SELECT * FROM t1 WHERE c5 = 1.7") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7, 2)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c5 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c6 = 1.7") + tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c6 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c7 in (1, 2)") + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c6 = 1") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c6 in (1, 2)") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (1, 2)") tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c6 = 0") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c6 in (0, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (0, 2)") + tdSql.checkRows(2) + + tdSql.query(f"SELECT * FROM t1 WHERE c7 = 1.7") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in (1.7, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 not in (1.7, 2)") + tdSql.checkRows(2) + + tdSql.query(f"SELECT * FROM t1 WHERE c7 = 0") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in (0, 2)") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c7 not in (0, 2)") + tdSql.checkRows(1) + + tdSql.query(f"SELECT * FROM t1 WHERE c7 = ''") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in ('', 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 not in ('', 2)") + tdSql.checkRows(2) + def run(self): self.prepareData() self.check() diff --git a/tests/system-test/2-query/ts-5761.py b/tests/system-test/2-query/ts-5761.py index c833661c76..d734a269dd 100644 --- a/tests/system-test/2-query/ts-5761.py +++ b/tests/system-test/2-query/ts-5761.py @@ -23,25 +23,88 @@ class TDTestCase: tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32));") # create index for all tags - tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1')") + tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1.7')") + tdSql.execute("INSERT INTO t1 VALUES (1641024000001, 0, 0, 1.7, 0, 0, '0', '')") + tdSql.execute("INSERT INTO t1 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") def check(self): + tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7") + tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)") tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c1 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c2 = 1.7") + tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c2 in (1.7, 2)") tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c2 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c3 = 1.7") + tdSql.checkRows(1) tdSql.query(f"SELECT * FROM t1 WHERE c3 in (1.7, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c3 not in (1.7, 2)") + tdSql.checkRows(2) + + tdSql.query(f"SELECT * FROM t1 WHERE c4 = 1.7") tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c4 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7, 2)") - tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c4 not in (1.7, 2)") + tdSql.checkRows(3) + tdSql.query(f"SELECT * FROM t1 WHERE c5 = 1.7") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c5 in (1.7, 2)") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c5 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c6 = 1.7") + tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c6 in (1.7, 2)") tdSql.checkRows(0) - tdSql.query(f"SELECT * FROM t1 WHERE c7 in (1, 2)") + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (1.7, 2)") + tdSql.checkRows(3) + + tdSql.query(f"SELECT * FROM t1 WHERE c6 = 1") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c6 in (1, 2)") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (1, 2)") tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c6 = 0") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c6 in (0, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (0, 2)") + tdSql.checkRows(2) + + tdSql.query(f"SELECT * FROM t1 WHERE c7 = 1.7") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in (1.7, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 not in (1.7, 2)") + tdSql.checkRows(2) + + tdSql.query(f"SELECT * FROM t1 WHERE c7 = 0") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in (0, 2)") + tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c7 not in (0, 2)") + tdSql.checkRows(1) + + tdSql.query(f"SELECT * FROM t1 WHERE c7 = ''") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 in ('', 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM t1 WHERE c7 not in ('', 2)") + tdSql.checkRows(2) + def run(self): self.prepareData() self.check() From bd6170c03e961191a1eb2419d5f6ba7fae015141 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 16 Dec 2024 19:47:12 +0800 Subject: [PATCH 058/108] fix:[TS-5761] error for in operator --- source/util/src/thashutil.c | 40 +++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 5499a651c1..b466e1b351 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -143,6 +143,38 @@ uint64_t MurmurHash3_64(const char *key, uint32_t len) { uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; } uint32_t taosIntHash_16(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint16_t *)key; } uint32_t taosIntHash_8(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint8_t *)key; } +uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { + float f = GET_FLOAT_VAL(key); + if (isnan(f)) { + return 0x7fc00000; + } + + if (FLT_EQUAL(f, 0.0)) { + return 0; + } + if (fabs(f) < FLT_MAX / BASE - DLT) { + int32_t t = (int32_t)(round(BASE * (f + DLT))); + return (uint32_t)t; + } else { + return 0x7fc00000; + } +} +uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { + double f = GET_DOUBLE_VAL(key); + if (isnan(f)) { + return 0x7fc00000; + } + + if (FLT_EQUAL(f, 0.0)) { + return 0; + } + if (fabs(f) < DBL_MAX / BASE - DLT) { + int32_t t = (int32_t)(round(BASE * (f + DLT))); + return (uint32_t)t; + } else { + return 0x7fc00000; + } +} uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) { uint64_t val = *(uint64_t *)key; @@ -158,7 +190,6 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_DOUBLE: fn = taosIntHash_64; break; case TSDB_DATA_TYPE_BINARY: @@ -169,7 +200,6 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { break; case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_FLOAT: fn = taosIntHash_32; break; case TSDB_DATA_TYPE_SMALLINT: @@ -181,6 +211,12 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break; + case TSDB_DATA_TYPE_FLOAT: + fn = taosFloatHash; + break; + case TSDB_DATA_TYPE_DOUBLE: + fn = taosDoubleHash; + break; default: fn = taosIntHash_32; break; From ae7f69c7807d91d1c060a5d062b4d0d1630e082e Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 17 Dec 2024 09:02:06 +0800 Subject: [PATCH 059/108] fix: oper param type --- source/libs/scalar/src/filter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 276604f58c..c9e6756d07 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -5100,7 +5100,7 @@ static int32_t hasValidInOper(SArray *sclInOper, SArray *colRangeList, bool* has return TSDB_CODE_SUCCESS; } } else { - if (startPt->val.d != endPt->val.d && (valDatum.i >= startPt->val.i || valDatum.i <= endPt->val.i)) { + if (startPt->val.i != endPt->val.i && (valDatum.i >= startPt->val.i || valDatum.i <= endPt->val.i)) { *has = true; return TSDB_CODE_SUCCESS; } From f603042eef2e0fc9e3fbd151b14a6704a16127ae Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Tue, 17 Dec 2024 09:13:39 +0800 Subject: [PATCH 060/108] chore: revert taosadapter version --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 524d44da73..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG enh/xftan/TD-33174-3.0 + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2013a822f55aeb2780bf5fc8831e9470a1fd3251 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Dec 2024 10:26:19 +0800 Subject: [PATCH 061/108] fix:[TS-5761] error for in operator --- source/libs/scalar/src/filter.c | 4 +++- source/libs/scalar/src/sclvector.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 807fb5e237..e482a1d241 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4874,7 +4874,9 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { type = tmp; } } - exprNode->resType.type = type; + if (IS_NUMERIC_TYPE(type)){ + exprNode->resType.type = type; + } } if ((0 != type && type != refNode->node.resType.type) || OP_TYPE_NOT_IN == node->opType) { stat->scalarMode = true; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index b862f42a37..bc0f12f7b2 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1069,7 +1069,7 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows) { - if (input->columnData == NULL && input->pHashFilterVar != NULL){ + if (input->columnData == NULL && (input->pHashFilter != NULL || input->pHashFilterVar != NULL)){ return TSDB_CODE_SUCCESS; } output->numOfRows = input->numOfRows; From 074c3e7eb283e1453e212901c947642728acbda0 Mon Sep 17 00:00:00 2001 From: Haolin Wang Date: Fri, 13 Dec 2024 17:23:21 +0800 Subject: [PATCH 062/108] fix: enable client dedup for stmt/stmt2 inserts in interlace mode --- include/common/tdataformat.h | 4 +- source/common/src/tdataformat.c | 44 ++++- source/libs/parser/src/parInsertStmt.c | 10 +- tests/script/api/makefile | 4 + tests/script/api/stmt-insert-dupkeys.c | 234 ++++++++++++++++++++++++ tests/script/api/stmt2-insert-dupkeys.c | 212 +++++++++++++++++++++ 6 files changed, 502 insertions(+), 6 deletions(-) create mode 100644 tests/script/api/stmt-insert-dupkeys.c create mode 100644 tests/script/api/stmt2-insert-dupkeys.c diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index cb05f98f45..0cc30edf82 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -378,7 +378,7 @@ typedef struct { TAOS_MULTI_BIND *bind; } SBindInfo; int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray); + SArray *rowArray, bool *orderedDup); // stmt2 binding int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos, @@ -392,7 +392,7 @@ typedef struct { } SBindInfo2; int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray); + SArray *rowArray, bool *orderedDup); #endif diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c1ab7ccff0..0db88f4b2a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -449,9 +449,10 @@ static int32_t tBindInfoCompare(const void *p1, const void *p2, const void *para * `infoSorted` is whether the bind information is sorted by column id * `pTSchema` is the schema of the table * `rowArray` is the array to store the rows + * `orderedDup` is an array to store ordered and duplicateTs */ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray) { + SArray *rowArray, bool *orderedDup) { if (infos == NULL || numOfInfos <= 0 || numOfInfos > pTSchema->numOfCols || pTSchema == NULL || rowArray == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -469,6 +470,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, return terrno; } + SRowKey rowKey, lastRowKey; for (int32_t iRow = 0; iRow < numOfRows; iRow++) { taosArrayClear(colValArray); @@ -507,6 +509,24 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, code = terrno; goto _exit; } + + if (orderedDup) { + tRowGetKey(row, &rowKey); + if (iRow == 0) { + // init to ordered by default + orderedDup[0] = true; + // init to non-duplicate by default + orderedDup[1] = false; + } else { + // no more compare if we already get disordered or duplicate rows + if (orderedDup[0] && !orderedDup[1]) { + int32_t code = tRowKeyCompare(&rowKey, &lastRowKey); + orderedDup[0] = (code >= 0); + orderedDup[1] = (code == 0); + } + } + lastRowKey = rowKey; + } } _exit: @@ -3235,9 +3255,10 @@ _exit: * `infoSorted` is whether the bind information is sorted by column id * `pTSchema` is the schema of the table * `rowArray` is the array to store the rows + * `orderedDup` is an array to store ordered and duplicateTs */ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray) { + SArray *rowArray, bool *orderedDup) { if (infos == NULL || numOfInfos <= 0 || numOfInfos > pTSchema->numOfCols || pTSchema == NULL || rowArray == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -3266,6 +3287,7 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte } } + SRowKey rowKey, lastRowKey; for (int32_t iRow = 0; iRow < numOfRows; iRow++) { taosArrayClear(colValArray); @@ -3317,6 +3339,24 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte code = terrno; goto _exit; } + + if (orderedDup) { + tRowGetKey(row, &rowKey); + if (iRow == 0) { + // init to ordered by default + orderedDup[0] = true; + // init to non-duplicate by default + orderedDup[1] = false; + } else { + // no more compare if we already get disordered or duplicate rows + if (orderedDup[0] && !orderedDup[1]) { + int32_t code = tRowKeyCompare(&rowKey, &lastRowKey); + orderedDup[0] = (code >= 0); + orderedDup[1] = (code == 0); + } + } + lastRowKey = rowKey; + } } _exit: diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 4ecc18d189..cdc9416d36 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -323,6 +323,7 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind int32_t code = 0; int16_t lastColId = -1; bool colInOrder = true; + bool orderedDup[2]; if (NULL == *pTSchema) { *pTSchema = tBuildTSchema(pSchema, pDataBlock->pMeta->tableInfo.numOfColumns, pDataBlock->pMeta->sversion); @@ -368,7 +369,9 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind // } } - code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols); + code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols, orderedDup); + pDataBlock->ordered = orderedDup[0]; + pDataBlock->duplicateTs = orderedDup[1]; qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); @@ -689,6 +692,7 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin int16_t lastColId = -1; bool colInOrder = true; int ncharColNums = 0; + bool orderedDup[2]; if (NULL == *pTSchema) { *pTSchema = tBuildTSchema(pSchema, pDataBlock->pMeta->tableInfo.numOfColumns, pDataBlock->pMeta->sversion); @@ -745,7 +749,9 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin pBindInfos[c].bytes = pColSchema->bytes; } - code = tRowBuildFromBind2(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols); + code = tRowBuildFromBind2(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols, orderedDup); + pDataBlock->ordered = orderedDup[0]; + pDataBlock->duplicateTs = orderedDup[1]; qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); diff --git a/tests/script/api/makefile b/tests/script/api/makefile index b871c5f3ff..a270a6c0ed 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -29,6 +29,8 @@ exe: # gcc $(CFLAGS) ./stmt2-get-fields.c -o $(ROOT)stmt2-get-fields $(LFLAGS) # gcc $(CFLAGS) ./stmt2-nohole.c -o $(ROOT)stmt2-nohole $(LFLAGS) gcc $(CFLAGS) ./stmt-crash.c -o $(ROOT)stmt-crash $(LFLAGS) + gcc $(CFLAGS) ./stmt-insert-dupkeys.c -o $(ROOT)stmt-insert-dupkeys $(LFLAGS) + gcc $(CFLAGS) ./stmt2-insert-dupkeys.c -o $(ROOT)stmt2-insert-dupkeys $(LFLAGS) clean: rm $(ROOT)batchprepare @@ -47,3 +49,5 @@ clean: rm $(ROOT)stmt2-get-fields rm $(ROOT)stmt2-nohole rm $(ROOT)stmt-crash + rm $(ROOT)stmt-insert-dupkeys + rm $(ROOT)stmt2-insert-dupkeys diff --git a/tests/script/api/stmt-insert-dupkeys.c b/tests/script/api/stmt-insert-dupkeys.c new file mode 100644 index 0000000000..b564fbb21d --- /dev/null +++ b/tests/script/api/stmt-insert-dupkeys.c @@ -0,0 +1,234 @@ +// compile with +// gcc -o stmt-insert-dupkeys stmt-insert-dupkeys.c -ltaos +#include +#include +#include +#include "taos.h" + +#define NUMROWS 3 + +/** + * @brief execute sql only and ignore result set + * + * @param taos + * @param sql + */ +void executeSQL(TAOS *taos, const char *sql) { + TAOS_RES *res = taos_query(taos, sql); + int code = taos_errno(res); + if (code != 0) { + printf("%s\n", taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(EXIT_FAILURE); + } + taos_free_result(res); +} + +/** + * @brief exit program when error occur. + * + * @param stmt + * @param code + * @param msg + */ +void checkErrorCode(TAOS_STMT *stmt, int code, const char *msg) { + if (code != 0) { + printf("%s. error: %s\n", msg, taos_stmt_errstr(stmt)); + exit(EXIT_FAILURE); + } +} + +void prepareBindTags(TAOS_MULTI_BIND *tags) { + // bind table name and tags + char *location = "California.SanFrancisco"; + int groupId = 2; + tags[0].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[0].buffer_length = strlen(location); + tags[0].length = (int32_t *)&tags[0].buffer_length; + tags[0].buffer = location; + tags[0].is_null = NULL; + + tags[1].buffer_type = TSDB_DATA_TYPE_INT; + tags[1].buffer_length = sizeof(int); + tags[1].length = (int32_t *)&tags[1].buffer_length; + tags[1].buffer = &groupId; + tags[1].is_null = NULL; +} + +void prepareBindParams(TAOS_MULTI_BIND *params, int64_t *ts, float *current, int *voltage, float *phase) { + // is_null array + char is_null[NUMROWS] = {0}; + // length array + int32_t int64Len[NUMROWS] = {sizeof(int64_t)}; + int32_t floatLen[NUMROWS] = {sizeof(float)}; + int32_t intLen[NUMROWS] = {sizeof(int)}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(int64_t); + params[0].buffer = ts; + params[0].length = int64Len; + params[0].is_null = is_null; + params[0].num = NUMROWS; + + params[1].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[1].buffer_length = sizeof(float); + params[1].buffer = current; + params[1].length = floatLen; + params[1].is_null = is_null; + params[1].num = NUMROWS; + + params[2].buffer_type = TSDB_DATA_TYPE_INT; + params[2].buffer_length = sizeof(int); + params[2].buffer = voltage; + params[2].length = intLen; + params[2].is_null = is_null; + params[2].num = NUMROWS; + + params[3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[3].buffer_length = sizeof(float); + params[3].buffer = phase; + params[3].length = floatLen; + params[3].is_null = is_null; + params[3].num = NUMROWS; +} + +/** + * @brief insert data using stmt API + * + * @param taos + */ +void insertData(TAOS *taos, int64_t *ts, float *current, int *voltage, float *phase) { + // init + TAOS_STMT *stmt = taos_stmt_init(taos); + + // prepare + const char *sql = "INSERT INTO ? USING meters TAGS(?, ?) values(?, ?, ?, ?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + checkErrorCode(stmt, code, "failed to execute taos_stmt_prepare"); + + // bind table name and tags + TAOS_MULTI_BIND tags[2]; + prepareBindTags(tags); + code = taos_stmt_set_tbname_tags(stmt, "d1001", tags); + checkErrorCode(stmt, code, "failed to execute taos_stmt_set_tbname_tags"); + + TAOS_MULTI_BIND params[4]; + prepareBindParams(params, ts, current, voltage, phase); + + code = taos_stmt_bind_param_batch(stmt, params); // bind batch + checkErrorCode(stmt, code, "failed to execute taos_stmt_bind_param_batch"); + + code = taos_stmt_add_batch(stmt); // add batch + checkErrorCode(stmt, code, "failed to execute taos_stmt_add_batch"); + + // execute + code = taos_stmt_execute(stmt); + checkErrorCode(stmt, code, "failed to execute taos_stmt_execute"); + + int affectedRows = taos_stmt_affected_rows(stmt); + printf("successfully inserted %d rows\n", affectedRows); + + // close + (void)taos_stmt_close(stmt); +} + +void insertDataInterlace(TAOS *taos, int64_t *ts, float *current, int *voltage, float *phase) { + // init with interlace mode + TAOS_STMT_OPTIONS op; + op.reqId = 0; + op.singleStbInsert = true; + op.singleTableBindOnce = true; + TAOS_STMT *stmt = taos_stmt_init_with_options(taos, &op); + + // prepare + const char *sql = "INSERT INTO ? values(?, ?, ?, ?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + checkErrorCode(stmt, code, "failed to execute taos_stmt_prepare"); + + // bind table name and tags + TAOS_MULTI_BIND tags[2]; + prepareBindTags(tags); + code = taos_stmt_set_tbname_tags(stmt, "d1001", tags); + checkErrorCode(stmt, code, "failed to execute taos_stmt_set_tbname_tags"); + + TAOS_MULTI_BIND params[4]; + prepareBindParams(params, ts, current, voltage, phase); + + code = taos_stmt_bind_param_batch(stmt, params); // bind batch + checkErrorCode(stmt, code, "failed to execute taos_stmt_bind_param_batch"); + + code = taos_stmt_add_batch(stmt); // add batch + checkErrorCode(stmt, code, "failed to execute taos_stmt_add_batch"); + + // execute + code = taos_stmt_execute(stmt); + checkErrorCode(stmt, code, "failed to execute taos_stmt_execute"); + + int affectedRows = taos_stmt_affected_rows(stmt); + printf("successfully inserted %d rows\n", affectedRows); + + // close + (void)taos_stmt_close(stmt); +} + +int main() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 6030); + if (taos == NULL) { + printf("failed to connect to server\n"); + exit(EXIT_FAILURE); + } + executeSQL(taos, "DROP DATABASE IF EXISTS power"); + executeSQL(taos, "CREATE DATABASE power"); + executeSQL(taos, "USE power"); + executeSQL(taos, + "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), " + "groupId INT)"); + + // initial insert, expect insert 3 rows + int64_t ts0[] = {1648432611234, 1648432611345, 1648432611456}; + float current0[] = {10.1f, 10.2f, 10.3f}; + int voltage0[] = {216, 217, 218}; + float phase0[] = {0.31f, 0.32f, 0.33f}; + insertData(taos, ts0, current0, voltage0, phase0); + + // insert with interlace mode, send non-duplicate ts, expect insert 3 overlapped rows + int64_t ts1[] = {1648432611234, 1648432611345, 1648432611456}; + int voltage1[] = {219, 220, 221}; + insertDataInterlace(taos, ts1, current0, voltage1, phase0); + + // insert with interlace mode, send duplicate ts, expect insert 2 rows with dups merged + int64_t ts2[] = {1648432611678, 1648432611678, 1648432611789}; + int voltage2[] = {222, 223, 224}; + insertDataInterlace(taos, ts2, current0, voltage2, phase0); + + // insert with interlace mode, send disordered rows, expect insert 3 sorted rows + int64_t ts3[] = {1648432611900, 1648432611890, 1648432611910}; + int voltage3[] = {225, 226, 227}; + insertDataInterlace(taos, ts3, current0, voltage3, phase0); + + // insert with interlace mode, send disordered and duplicate rows, expect insert 2 sorted and dup-merged rows + int64_t ts4[] = {1648432611930, 1648432611920, 1648432611930}; + int voltage4[] = {228, 229, 230}; + insertDataInterlace(taos, ts4, current0, voltage4, phase0); + + taos_close(taos); + taos_cleanup(); + + // final results + // taos> select * from d1001; + // ts | current | voltage | phase | + // ====================================================================================== + // 2022-03-28 09:56:51.234 | 10.1000004 | 219 | 0.3100000 | + // 2022-03-28 09:56:51.345 | 10.1999998 | 220 | 0.3200000 | + // 2022-03-28 09:56:51.456 | 10.3000002 | 221 | 0.3300000 | + // 2022-03-28 09:56:51.678 | 10.1999998 | 223 | 0.3200000 | + // 2022-03-28 09:56:51.789 | 10.3000002 | 224 | 0.3300000 | + // 2022-03-28 09:56:51.890 | 10.1999998 | 226 | 0.3200000 | + // 2022-03-28 09:56:51.900 | 10.1000004 | 225 | 0.3100000 | + // 2022-03-28 09:56:51.910 | 10.3000002 | 227 | 0.3300000 | + // 2022-03-28 09:56:51.920 | 10.1999998 | 229 | 0.3200000 | + // 2022-03-28 09:56:51.930 | 10.3000002 | 230 | 0.3300000 | + // Query OK, 10 row(s) in set (0.005083s) +} + diff --git a/tests/script/api/stmt2-insert-dupkeys.c b/tests/script/api/stmt2-insert-dupkeys.c new file mode 100644 index 0000000000..c056e1bcb0 --- /dev/null +++ b/tests/script/api/stmt2-insert-dupkeys.c @@ -0,0 +1,212 @@ +#include +#include +#include +#include +#include +#include "taos.h" + +int CTB_NUMS = 3; +int ROW_NUMS = 3; + +void do_query(TAOS* taos, const char* sql) { + TAOS_RES* result = taos_query(taos, sql); + int code = taos_errno(result); + if (code) { + printf("failed to query: %s, reason:%s\n", sql, taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); +} + +void createdb(TAOS* taos) { + do_query(taos, "drop database if exists db"); + do_query(taos, "create database db"); + do_query(taos, "create stable db.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); + do_query(taos, "use db"); +} + +#define INIT(tbs, ts, ts_len, b, b_len, tags, paramv) \ +do { \ + /* tbname */ \ + tbs = (char**)malloc(CTB_NUMS * sizeof(char*)); \ + for (int i = 0; i < CTB_NUMS; i++) { \ + tbs[i] = (char*)malloc(sizeof(char) * 20); \ + sprintf(tbs[i], "ctb_%d", i); \ + } \ + /* col params */ \ + ts = (int64_t**)malloc(CTB_NUMS * sizeof(int64_t*)); \ + b = (char**)malloc(CTB_NUMS * sizeof(char*)); \ + ts_len = (int*)malloc(ROW_NUMS * sizeof(int)); \ + b_len = (int*)malloc(ROW_NUMS * sizeof(int)); \ + for (int i = 0; i < ROW_NUMS; i++) { \ + ts_len[i] = sizeof(int64_t); \ + b_len[i] = 1; \ + } \ + for (int i = 0; i < CTB_NUMS; i++) { \ + ts[i] = (int64_t*)malloc(ROW_NUMS * sizeof(int64_t)); \ + b[i] = (char*)malloc(ROW_NUMS * sizeof(char)); \ + for (int j = 0; j < ROW_NUMS; j++) { \ + ts[i][j] = 1591060628000 + j; \ + b[i][j] = (char)('a' + j); \ + } \ + } \ + /*tag params */ \ + int t1 = 0; \ + int t1len = sizeof(int); \ + int t2len = 3; \ + /* bind params */ \ + paramv = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); \ + tags = (TAOS_STMT2_BIND**)malloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); \ + for (int i = 0; i < CTB_NUMS; i++) { \ + /* create tags */ \ + tags[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); \ + tags[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_INT, &t1, &t1len, NULL, 0}; \ + tags[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, "after", &t2len, NULL, 0}; \ + /* create col params */ \ + paramv[i] = (TAOS_STMT2_BIND*)malloc(2 * sizeof(TAOS_STMT2_BIND)); \ + paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; \ + paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; \ + } \ +} while (0) + +#define UINIT(tbs, ts, ts_len, b, b_len, tags, paramv) \ +do { \ + for (int i = 0; i < CTB_NUMS; i++) { \ + free(tbs[i]); \ + } \ + free(tbs); \ + for (int i = 0; i < CTB_NUMS; i++) { \ + free(ts[i]); \ + free(b[i]); \ + } \ + free(ts); \ + free(b); \ + free(ts_len); \ + free(b_len); \ + for (int i = 0; i < CTB_NUMS; i++) { \ + free(tags[i]); \ + free(paramv[i]); \ + } \ + free(tags); \ + free(paramv); \ +} while (0) + +void insert(TAOS* taos, char **tbs, TAOS_STMT2_BIND **tags, TAOS_STMT2_BIND **paramv, const char* sql) +{ + clock_t start, end; + double cpu_time_used; + + TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; + TAOS_STMT2 *stmt = taos_stmt2_init(taos, &option); + int code = taos_stmt2_prepare(stmt, sql, 0); + if (code != 0) { + printf("failed to execute taos_stmt2_prepare. error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + exit(EXIT_FAILURE); + } + + // bind + start = clock(); + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, tags, paramv}; + if (taos_stmt2_bind_param(stmt, &bindv, -1)) { + printf("failed to execute taos_stmt2_bind_param statement.error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + exit(EXIT_FAILURE); + } + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("stmt2-bind [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + start = clock(); + + // exec + if (taos_stmt2_exec(stmt, NULL)) { + printf("failed to execute taos_stmt2_exec statement.error:%s\n", taos_stmt2_error(stmt)); + taos_stmt2_close(stmt); + exit(EXIT_FAILURE); + } + end = clock(); + cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; + printf("stmt2-exec [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + + taos_stmt2_close(stmt); +} + +void insert_dist(TAOS* taos, const char *sql) { + char **tbs, **b; + int64_t **ts; + int *ts_len, *b_len; + TAOS_STMT2_BIND **paramv, **tags; + + INIT(tbs, ts, ts_len, b, b_len, tags, paramv); + + insert(taos, tbs, tags, paramv, sql); + + UINIT(tbs, ts, ts_len, b, b_len, tags, paramv); +} + +void insert_dup(TAOS* taos, const char *sql) { + char **tbs, **b; + int64_t **ts; + int *ts_len, *b_len; + TAOS_STMT2_BIND **paramv, **tags; + + INIT(tbs, ts, ts_len, b, b_len, tags, paramv); + + // insert duplicate rows + for (int i = 0; i < CTB_NUMS; i++) { + for (int j = 0; j < ROW_NUMS; j++) { + ts[i][j] = 1591060628000; + b[i][j] = (char)('x' + j); + } + } + for (int i = 0; i < CTB_NUMS; i++) { + paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + } + insert(taos, tbs, tags, paramv, sql); + + UINIT(tbs, ts, ts_len, b, b_len, tags, paramv); +} + +int main() { + TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); + if (!taos) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(EXIT_FAILURE); + } + + createdb(taos); + // insert distinct rows + insert_dist(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); + // insert duplicate rows + insert_dup(taos, "insert into db.? values(?,?)"); + + taos_close(taos); + taos_cleanup(); +} + +// final results +// taos> select * from ctb_0; +// ts | b | +// ========================================= +// 2020-06-02 09:17:08.000 | z | +// 2020-06-02 09:17:08.001 | b | +// 2020-06-02 09:17:08.002 | c | +// Query OK, 3 row(s) in set (0.003975s) +// +// taos> select * from ctb_1; +// ts | b | +// ========================================= +// 2020-06-02 09:17:08.000 | z | +// 2020-06-02 09:17:08.001 | b | +// 2020-06-02 09:17:08.002 | c | +// Query OK, 3 row(s) in set (0.007241s) + +// taos> select * from ctb_2; +// ts | b | +// ========================================= +// 2020-06-02 09:17:08.000 | z | +// 2020-06-02 09:17:08.001 | b | +// 2020-06-02 09:17:08.002 | c | +// Query OK, 3 row(s) in set (0.005443s) From 25c345a11b8f7fb9e14f4f4be82e83e73b841cbe Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 17 Dec 2024 11:35:18 +0800 Subject: [PATCH 063/108] add more tests --- source/libs/parser/src/parTranslater.c | 1 + tests/system-test/2-query/union.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index be3acbcff9..20e986c825 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2243,6 +2243,7 @@ static bool dataTypeEqual(const SDataType* l, const SDataType* r) { // 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { + if (l->type == TSDB_DATA_TYPE_NULL) return -1; if (l->type != r->type) { return 1; } diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index cd2432f21b..3a6247e62a 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -447,6 +447,18 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, None) + sql = "select null union select 1" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 1) + + sql = "select null union select 'asd'" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(2) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 'asd') + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From 85ec00768729bb3ce151aa6ca42aa42aef1216b1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 17 Dec 2024 11:39:49 +0800 Subject: [PATCH 064/108] fix: task release prior to response issue --- source/libs/qworker/src/qworker.c | 1 + source/libs/scheduler/src/schRemote.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index fa422b1de4..641fa03f7a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -569,6 +569,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu if (TSDB_CODE_SUCCESS != input->code) { QW_TASK_ELOG("task already failed at phase %s, code:0x%x", qwPhaseStr(phase), input->code); + ctx->ctrlConnInfo.handle = NULL; (void)qwDropTask(QW_FPARAMS()); QW_ERR_JRET(input->code); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index fd255f53cf..d15ac7a791 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1361,14 +1361,14 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, taosMemoryFree(msg); SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); } else { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) { + SCH_ERR_JRET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId)); + } + SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; SCH_ERR_JRET(code); - - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY) { - SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId)); - } } return TSDB_CODE_SUCCESS; From 2f6923ea3efbe40d29718fe1df469154c33bcbd4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 Dec 2024 11:44:54 +0800 Subject: [PATCH 065/108] merge: set git tags --- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taostools_CMakeLists.txt.in | 2 +- cmake/taosws_CMakeLists.txt.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index b013d45911..17446d184d 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 5972054c2de166aaf3cb03c3efcd1ab38483acd7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 Dec 2024 14:33:44 +0800 Subject: [PATCH 066/108] fix:[TD-33225]remove the limition when creating index or modifing column compression if there is a topic/stream/tsma --- source/dnode/mnode/impl/src/mndIndex.c | 11 ------- source/dnode/mnode/impl/src/mndStb.c | 5 --- tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/td-33225.py | 44 ++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 tests/system-test/7-tmq/td-33225.py diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 718c34e85a..f5dac9df65 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -673,8 +673,6 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb code = TSDB_CODE_MND_TAG_NOT_EXIST; TAOS_RETURN(code); } - col_id_t colId = pOld->pTags[tag].colId; - TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); SSchema *pTag = pNew->pTags + tag; @@ -806,16 +804,7 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re TAOS_RETURN(code); } - col_id_t colId = pStb->pTags[tag].colId; - TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId)); - - // SSchema *pTag = pStb->pTags + tag; - // if (IS_IDX_ON(pTag)) { - // terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; - // return -1; - // } code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj); - TAOS_RETURN(code); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 2db76f6312..d46968a22d 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1805,7 +1805,6 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj * } SSchema *pTarget = &pOld->pColumns[idx]; col_id_t colId = pTarget->colId; - TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId)); TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew)); code = validColCmprByType(pTarget->type, p->bytes); @@ -3702,10 +3701,6 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *ta terrno = TSDB_CODE_MND_TAG_NOT_EXIST; return -1; } - col_id_t colId = pOld->pTags[tag].colId; - if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { - return -1; - } if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4aedc0991e..6d091e1b4c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -315,6 +315,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts4563.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td32526.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py diff --git a/tests/system-test/7-tmq/td-33225.py b/tests/system-test/7-tmq/td-33225.py new file mode 100644 index 0000000000..f39e402b55 --- /dev/null +++ b/tests/system-test/7-tmq/td-33225.py @@ -0,0 +1,44 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.execute(f'create database if not exists db_33225') + tdSql.execute(f'use db_33225') + tdSql.execute(f'create stable if not exists s33225 (ts timestamp, c1 int, c2 int) tags (t binary(32), t2 int)') + tdSql.execute(f'insert into t1 using s33225 tags("__devicid__", 1) values(1669092069068, 0, 1)') + + tdSql.execute("create topic db_33225_topic as select ts,c1,t2 from s33225") + tdSql.execute(f'create stream s1 into st1 as select _wstart, count(*), avg(c2),t2 from s33225 PARTITION BY tbname INTERVAL(1m)') + + tdSql.execute(f'alter table s33225 modify column c2 COMPRESS "zlib"') + tdSql.execute(f'create index dex1 on s33225(t2)') + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From bab22150948cb97b45ebdcfc138344a9464f445b Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 17 Dec 2024 14:47:09 +0800 Subject: [PATCH 067/108] enh: time range --- source/libs/scalar/src/filter.c | 101 ++++---------------------------- 1 file changed, 13 insertions(+), 88 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c9e6756d07..af7e5fb771 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3859,6 +3859,11 @@ int32_t fltInitFromNode(SNode *tree, SFilterInfo *info, uint32_t options) { if (tctx.ignore) { FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); } + if (FILTER_EMPTY_RES(info)) { + info->func = filterExecuteImplEmpty; + taosArrayDestroyEx(group, filterFreeGroup); + return TSDB_CODE_SUCCESS; + } code = filterConvertGroupFromArray(info, group); if (TSDB_CODE_SUCCESS != code) { taosArrayDestroyEx(group, filterFreeGroup); @@ -5049,122 +5054,42 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { return TSDB_CODE_SUCCESS; } -static int32_t fltInOpertoArray(SArray **sclInOper, SFltSclOperator *pInOper) { - if (*sclInOper == NULL) { - *sclInOper = taosArrayInit(4, sizeof(SFltSclOperator *)); - if (NULL == *sclInOper) { - FLT_ERR_RET(terrno); - } - } - if (NULL == taosArrayPush(*sclInOper, &pInOper)) { - FLT_ERR_RET(terrno); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t hasValidInOper(SArray *sclInOper, SArray *colRangeList, bool* has) { - SFltSclOperator **ppSclOper = NULL; - *has = false; - for (int32_t i = 0; i < taosArrayGetSize(sclInOper); ++i) { - ppSclOper = taosArrayGet(sclInOper, i); - if (NULL == ppSclOper) { - return TSDB_CODE_OUT_OF_RANGE; - } - if(*ppSclOper == NULL) { - qError("func: hasValidInOper, invalid in operator"); - return TSDB_CODE_OUT_OF_RANGE; - } - SFltSclColumnRange *colRange = NULL; - SFltSclOperator * pSclOper = *ppSclOper; - for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { - colRange = taosArrayGet(colRangeList, i); - if (NULL == colRange) { - return TSDB_CODE_OUT_OF_RANGE; - } - if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)pSclOper->colNode)) { - SFltSclPoint *startPt = taosArrayGet(colRange->points, 0); - SFltSclPoint *endPt = taosArrayGet(colRange->points, 1); - if (NULL == startPt || NULL == endPt) { - return TSDB_CODE_OUT_OF_RANGE; - } - SNodeListNode *listNode = (SNodeListNode *)pSclOper->valNode; - SListCell *cell = listNode->pNodeList->pHead; - for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; - SFltSclDatum valDatum; - FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, valueNode)); - if (valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || - valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) { - if (startPt->val.d != endPt->val.d && (valDatum.d >= startPt->val.d || valDatum.d <= endPt->val.d)) { - *has = true; - return TSDB_CODE_SUCCESS; - } - } else { - if (startPt->val.i != endPt->val.i && (valDatum.i >= startPt->val.i || valDatum.i <= endPt->val.i)) { - *has = true; - return TSDB_CODE_SUCCESS; - } - } - cell = cell->pNext; - } - } - } - } - return TSDB_CODE_SUCCESS; -} - // TODO: process DNF composed of CNF static int32_t fltSclProcessCNF(SFilterInfo *pInfo, SArray *sclOpListCNF, SArray *colRangeList) { - int32_t code = TSDB_CODE_SUCCESS; - pInfo->isStrict = true; - SArray *sclInOper = NULL; size_t sz = taosArrayGetSize(sclOpListCNF); for (int32_t i = 0; i < sz; ++i) { SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); if (NULL == sclOper) { - FLT_ERR_JRET(TSDB_CODE_OUT_OF_RANGE); + FLT_ERR_RET(TSDB_CODE_OUT_OF_RANGE); } SFltSclColumnRange *colRange = NULL; - FLT_ERR_JRET(fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList, &colRange)); + FLT_ERR_RET(fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList, &colRange)); SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); if (NULL == points) { - FLT_ERR_JRET(terrno); + FLT_ERR_RET(terrno); } - FLT_ERR_JRET(fltSclBuildRangePoints(sclOper, points)); + FLT_ERR_RET(fltSclBuildRangePoints(sclOper, points)); if (taosArrayGetSize(colRange->points) != 0) { SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); if (NULL == merged) { - FLT_ERR_JRET(terrno); + FLT_ERR_RET(terrno); } - FLT_ERR_JRET(fltSclIntersect(colRange->points, points, merged)); + FLT_ERR_RET(fltSclIntersect(colRange->points, points, merged)); taosArrayDestroy(colRange->points); taosArrayDestroy(points); colRange->points = merged; if(merged->size == 0) { - goto _return; + return TSDB_CODE_SUCCESS; } } else { taosArrayDestroy(colRange->points); colRange->points = points; } if (sclOper->type == OP_TYPE_IN) { - code = fltInOpertoArray(&sclInOper, sclOper); - FLT_ERR_JRET(code); + pInfo->isStrict = false; } } - bool hasInOper = false; - code = hasValidInOper(sclInOper, colRangeList, &hasInOper); - if (code != TSDB_CODE_SUCCESS) { - goto _return; - } - if (hasInOper) { - pInfo->isStrict = false; - } -_return: - if (sclInOper) { - taosArrayDestroy(sclInOper); - } return TSDB_CODE_SUCCESS; } From 944470cf99356395d465764979e20a00bea501d8 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 17 Dec 2024 14:48:45 +0800 Subject: [PATCH 068/108] unused code --- source/libs/scalar/src/filter.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index af7e5fb771..a149384163 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4557,10 +4557,6 @@ static int32_t fltSclGetTimeStampDatum(SFltSclPoint *point, SFltSclDatum *d) { return TSDB_CODE_SUCCESS; } -typedef struct SRewriteGetInOperContext { - bool hasInOper; -} SRewriteGetInOperContext; - int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFilterInfo *info = NULL; int32_t code = 0; From 9fd2d86ae3cfbfb34a6bfd8b9a7c9da7ee976d3e Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Dec 2024 07:11:16 +0000 Subject: [PATCH 069/108] enh: memory safe function --- include/libs/crypt/crypt.h | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libs/crypt/crypt.h b/include/libs/crypt/crypt.h index 5f981b7ac8..78753874bd 100644 --- a/include/libs/crypt/crypt.h +++ b/include/libs/crypt/crypt.h @@ -26,7 +26,7 @@ typedef struct SCryptOpts { char* source; char* result; int32_t unitLen; - char key[17]; + char key[ENCRYPT_KEY_LEN + 1]; } SCryptOpts; int32_t CBC_Decrypt(SCryptOpts* opts); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 83043b4393..0f0dbfb3f1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -211,7 +211,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { #if defined(TD_ENTERPRISE) pCfg->tdbEncryptAlgorithm = pCreate->encryptAlgorithm; if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { - tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } #else pCfg->tdbEncryptAlgorithm = 0; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 5d4ffc604a..f4a678e629 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -327,7 +327,7 @@ struct SVnodeCfg { int16_t hashSuffix; int32_t tsdbPageSize; int32_t tdbEncryptAlgorithm; - char tdbEncryptKey[ENCRYPT_KEY_LEN]; + char tdbEncryptKey[ENCRYPT_KEY_LEN + 1]; int32_t s3ChunkSize; int32_t s3KeepLocal; int8_t s3Compact; diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 9c153bc8a1..653f525e09 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -303,7 +303,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tsEncryptKey[0] == 0) { return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; } else { - tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); + tstrncpy(pCfg->tdbEncryptKey, tsEncryptKey, ENCRYPT_KEY_LEN + 1); } } #endif From 28136db3f184fd3cdc4dbf92c8d85be1c2b54e5f Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Tue, 17 Dec 2024 14:12:50 +0800 Subject: [PATCH 070/108] fix enterprise comipile --- source/client/src/clientStmt2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index dba6aa3beb..8909eb6ccb 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -77,7 +77,7 @@ static int32_t stmtCreateRequest(STscStmt2* pStmt) { } if (pStmt->db != NULL) { taosMemoryFreeClear(pStmt->exec.pRequest->pDb); - pStmt->exec.pRequest->pDb = strdup(pStmt->db); + pStmt->exec.pRequest->pDb = taosStrdup(pStmt->db); } if (TSDB_CODE_SUCCESS == code) { pStmt->exec.pRequest->syncQuery = true; From 95ef1ca6567682e30e803ebb629edb86f417503e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 Dec 2024 15:38:36 +0800 Subject: [PATCH 071/108] test: for unittest --- source/os/src/osThread.c | 62 +++++------ source/os/test/osThreadTests.cpp | 186 +++++++++++++++++++++++++++++-- 2 files changed, 203 insertions(+), 45 deletions(-) diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 2810c5e196..abb88d2c6b 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -28,9 +28,9 @@ int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start) return code; } -int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { +int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { OS_PARAM_CHECK(attr); - int32_t code = pthread_attr_destroy(attr); + int32_t code = pthread_attr_destroy(attr); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } @@ -99,7 +99,7 @@ int32_t taosThreadAttrGetStackSize(const TdThreadAttr *attr, size_t *stacksize) int32_t taosThreadAttrInit(TdThreadAttr *attr) { OS_PARAM_CHECK(attr); - int32_t code = pthread_attr_init(attr); + int32_t code = pthread_attr_init(attr); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } @@ -160,8 +160,8 @@ int32_t taosThreadAttrSetStackSize(TdThreadAttr *attr, size_t stacksize) { return code; } -int32_t taosThreadCancel(TdThread thread) { - int32_t code = pthread_cancel(thread); +int32_t taosThreadCancel(TdThread thread) { + int32_t code = pthread_cancel(thread); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } @@ -253,7 +253,7 @@ int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const return TAOS_SYSTEM_WINAPI_ERROR(error); #else int32_t code = pthread_cond_timedwait(cond, mutex, abstime); - if(code == ETIMEDOUT) { + if (code == ETIMEDOUT) { return TSDB_CODE_TIMEOUT_ERROR; } else if (code) { return TAOS_SYSTEM_ERROR(code); @@ -333,20 +333,18 @@ int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) { #endif } -int32_t taosThreadDetach(TdThread thread) { - int32_t code = pthread_detach(thread); +int32_t taosThreadDetach(TdThread thread) { + int32_t code = pthread_detach(thread); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadEqual(TdThread t1, TdThread t2) { - return pthread_equal(t1, t2); -} +int32_t taosThreadEqual(TdThread t1, TdThread t2) { return pthread_equal(t1, t2); } -void taosThreadExit(void *valuePtr) { - if(valuePtr) return pthread_exit(valuePtr); +void taosThreadExit(void *valuePtr) { + if (valuePtr) return pthread_exit(valuePtr); } int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_param *param) { @@ -359,12 +357,10 @@ int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_p return code; } -void *taosThreadGetSpecific(TdThreadKey key) { - return pthread_getspecific(key); -} +void *taosThreadGetSpecific(TdThreadKey key) { return pthread_getspecific(key); } -int32_t taosThreadJoin(TdThread thread, void **valuePtr) { - int32_t code = pthread_join(thread, valuePtr); +int32_t taosThreadJoin(TdThread thread, void **valuePtr) { + int32_t code = pthread_join(thread, valuePtr); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } @@ -380,16 +376,16 @@ int32_t taosThreadKeyCreate(TdThreadKey *key, void (*destructor)(void *)) { return code; } -int32_t taosThreadKeyDelete(TdThreadKey key) { - int32_t code = pthread_key_delete(key); +int32_t taosThreadKeyDelete(TdThreadKey key) { + int32_t code = pthread_key_delete(key); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadKill(TdThread thread, int32_t sig) { - int32_t code = pthread_kill(thread, sig); +int32_t taosThreadKill(TdThread thread, int32_t sig) { + int32_t code = pthread_kill(thread, sig); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } @@ -584,7 +580,7 @@ int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { */ return 0; #else - OS_PARAM_CHECK(rwlock); + OS_PARAM_CHECK(rwlock); int32_t code = pthread_rwlock_destroy(rwlock); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); @@ -747,16 +743,16 @@ int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr *attr, int32_t pshared TdThread taosThreadSelf(void) { return pthread_self(); } -int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate) { - int32_t code = pthread_setcancelstate(state, oldstate); +int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate) { + int32_t code = pthread_setcancelstate(state, oldstate); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype) { - int32_t code = pthread_setcanceltype(type, oldtype); +int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype) { + int32_t code = pthread_setcanceltype(type, oldtype); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } @@ -772,9 +768,9 @@ int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sc return code; } -int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { +int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { OS_PARAM_CHECK(value); - int32_t code = pthread_setspecific(key, value); + int32_t code = pthread_setspecific(key, value); if (code) { return (terrno = TAOS_SYSTEM_ERROR(code)); } @@ -847,13 +843,11 @@ int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { #endif } -void taosThreadTestCancel(void) { - return pthread_testcancel(); -} +void taosThreadTestCancel(void) { return pthread_testcancel(); } -void taosThreadClear(TdThread *thread) { +void taosThreadClear(TdThread *thread) { if (!thread) return; - (void)memset(thread, 0, sizeof(TdThread)); + (void)memset(thread, 0, sizeof(TdThread)); } #ifdef WINDOWS diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index 5c0e40c3ca..5650f694fd 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -31,13 +31,40 @@ static int32_t globalVar = 0; -static void *funcPtr100(void *param) { - taosMsleep(100); +static void funcPtrKey(void *param) { taosMsleep(100); } + +static void *funcPtr200(void *param) { + taosMsleep(200); + TdThread thread = taosThreadSelf(); + + TdThreadKey key = {0}; + taosThreadKeyCreate(&key, funcPtrKey); + void *oldVal = taosThreadGetSpecific(key); + taosThreadSetSpecific(key, oldVal); + taosThreadKeyDelete(key); + + int32_t oldType = 0; + taosThreadSetCancelType(-1, &oldType); + taosThreadSetCancelType(0, &oldType); + + int32_t oldState = 0; + taosThreadSetCancelState(-1, &oldState); + taosThreadSetCancelState(0, &oldState); + + int32_t policy; + struct sched_param para; + taosThreadGetSchedParam(thread, &policy, ¶); + taosThreadGetSchedParam(thread, NULL, ¶); + taosThreadGetSchedParam(thread, &policy, NULL); + taosThreadSetSchedParam(NULL, 0, ¶); + taosThreadSetSchedParam(thread, 0, ¶); + return NULL; } static void *funcPtr500(void *param) { taosMsleep(500); + TdThread thread = taosThreadSelf(); return NULL; } @@ -51,18 +78,16 @@ static void *funcPtrExit2(void *param) { return NULL; } -static void funcPtrKey(void *param) { taosMsleep(100); } - TEST(osThreadTests, thread) { TdThread tid1 = {0}; TdThread tid2 = {0}; int32_t reti = 0; - reti = taosThreadCreate(NULL, NULL, funcPtr100, NULL); + reti = taosThreadCreate(NULL, NULL, funcPtr200, NULL); EXPECT_NE(reti, 0); reti = taosThreadCreate(&tid1, NULL, NULL, NULL); EXPECT_NE(reti, 0); - reti = taosThreadCreate(&tid1, NULL, funcPtr100, NULL); + reti = taosThreadCreate(&tid1, NULL, funcPtr200, NULL); EXPECT_EQ(reti, 0); reti = taosThreadCancel(tid1); @@ -91,14 +116,17 @@ TEST(osThreadTests, thread) { taosThreadKill(tid2, SIGINT); int32_t policy; - struct sched_param param; - taosThreadGetSchedParam(tid2, &policy, ¶m); - taosThreadGetSchedParam(tid2, NULL, ¶m); + struct sched_param para; + taosThreadGetSchedParam(tid2, &policy, ¶); + taosThreadGetSchedParam(tid2, NULL, ¶); taosThreadGetSchedParam(tid2, &policy, NULL); + taosThreadSetSchedParam(NULL, 0, ¶); + taosThreadSetSchedParam(tid2, 0, ¶); - TdThreadKey key; + TdThreadKey key = {0}; taosThreadKeyCreate(&key, funcPtrKey); - taosThreadGetSpecific(key); + void *oldVal = taosThreadGetSpecific(key); + taosThreadSetSpecific(key, oldVal); taosThreadKeyDelete(key); } @@ -247,6 +275,142 @@ TEST(osThreadTests, cond) { EXPECT_EQ(reti, 0); } +TEST(osThreadTests, mutex) { + int32_t reti = 0; + TdThreadMutex mutex; + reti = taosThreadMutexInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadMutexInit(&mutex, 0); + EXPECT_EQ(reti, 0); + + reti = taosThreadMutexTryLock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexTryLock(&mutex); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexTryLock(&mutex); + EXPECT_NE(reti, 0); + + reti = taosThreadMutexUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexUnlock(&mutex); + EXPECT_EQ(reti, 0); + + reti = taosThreadMutexLock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexLock(&mutex); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexUnlock(&mutex); + EXPECT_EQ(reti, 0); + + reti = taosThreadMutexDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexDestroy(&mutex); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, mutexAttr) { + int32_t reti = 0; + TdThreadMutexAttr mutexAttr; + reti = taosThreadMutexAttrInit(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrInit(&mutexAttr); + EXPECT_EQ(reti, 0); + + int32_t pshared; + reti = taosThreadMutexAttrGetPshared(&mutexAttr, &pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetPshared(&mutexAttr, pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetPshared(&mutexAttr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrSetPshared(NULL, pshared); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetPshared(NULL, &pshared); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetPshared(&mutexAttr, NULL); + EXPECT_NE(reti, 0); + + int32_t kind; + reti = taosThreadMutexAttrGetType(&mutexAttr, &kind); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetType(&mutexAttr, kind); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetType(&mutexAttr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrSetType(NULL, kind); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetType(NULL, &kind); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetType(&mutexAttr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadMutexAttrDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrDestroy(&mutexAttr); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, rwlock) { + int32_t reti = 0; + TdThreadRwlock rwlock; + reti = taosThreadRwlockInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockInit(&rwlock, 0); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockTryRdlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockTryRdlock(&rwlock); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockTryRdlock(&rwlock); + EXPECT_NE(reti, 0); + + reti = taosThreadRwlockUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockRdlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockRdlock(&rwlock); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockDestroy(&rwlock); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, rdlockAttr) { + int32_t reti = 0; + TdThreadRwlockAttr rdlockAttr; + reti = taosThreadRwlockAttrInit(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrInit(&rdlockAttr); + EXPECT_EQ(reti, 0); + + int32_t pshared; + reti = taosThreadRwlockAttrGetPshared(&rdlockAttr, &pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockAttrSetPshared(&rdlockAttr, pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockAttrSetPshared(&rdlockAttr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrSetPshared(NULL, pshared); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrGetPshared(NULL, &pshared); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrGetPshared(&rdlockAttr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadRwlockAttrDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrDestroy(&rdlockAttr); + EXPECT_EQ(reti, 0); +} + TEST(osThreadTests, spinlock) { int32_t reti = 0; From 2e5fa13787c2bce4612948d6d8fe372f15299804 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 Dec 2024 17:38:51 +0800 Subject: [PATCH 072/108] ut: minor changes --- source/os/src/osAtomic.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 5da2307cb3..d82c56c99a 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -355,7 +355,7 @@ void atomic_store_double(double volatile* ptr, double val) { double_number ret_num = {0}; ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return; + if (ret_num.i == old_num.i) break; } } @@ -414,6 +414,8 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { } double atomic_exchange_double(double volatile* ptr, double val) { + double ret = 0; + for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -425,9 +427,11 @@ double atomic_exchange_double(double volatile* ptr, double val) { ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) { - return ret_num.d; + ret = ret_num.d; + break; } } + return ret; } void* atomic_exchange_ptr(void* ptr, void* val) { @@ -589,6 +593,8 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { } double atomic_fetch_add_double(double volatile* ptr, double val) { + double ret = 0; + for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -599,8 +605,13 @@ double atomic_fetch_add_double(double volatile* ptr, double val) { double_number ret_num = {0}; ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return ret_num.d; + if (ret_num.i == old_num.i) { + ret = ret_num.d; + break; + } } + + return ret; } void* atomic_fetch_add_ptr(void* ptr, int64_t val) { @@ -710,6 +721,8 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { } double atomic_fetch_sub_double(double volatile* ptr, double val) { + double ret = 0; + for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -720,8 +733,13 @@ double atomic_fetch_sub_double(double volatile* ptr, double val) { double_number ret_num = {0}; ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return ret_num.d; + if (ret_num.i == old_num.i) { + ret = ret_num.d; + break; + } } + + return ret; } void* atomic_fetch_sub_ptr(void* ptr, int64_t val) { From 53645e86e1e60065d917e4b380a5bac6423779f9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Dec 2024 09:47:08 +0000 Subject: [PATCH 073/108] enh: compile problem in release mode --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/dnode/vnode/src/vnd/vnodeAsync.c | 2 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index bf79b2482d..3d50c81a5d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -104,7 +104,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF } pFD->path = (char *)&pFD[1]; - tstrncpy(pFD->path, path, strlen(path) + 1); + memcpy(pFD->path, path, strlen(path) + 1); pFD->szPage = szPage; pFD->flag = flag; pFD->szPage = szPage; diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 424ed0f325..3f40ace6c4 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -330,7 +330,7 @@ static int32_t vnodeAsyncInit(SVAsync **async, const char *label) { return terrno; } - tstrncpy((char *)((*async) + 1), label, strlen(label) + 1); + memcpy((char *)((*async) + 1), label, strlen(label) + 1); (*async)->label = (const char *)((*async) + 1); (void)taosThreadMutexInit(&(*async)->mutex, NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index b9e686932e..945b4cbeae 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -417,7 +417,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC } pVnode->path = (char *)&pVnode[1]; - tstrncpy(pVnode->path, path, strlen(path) + 1); + memcpy(pVnode->path, path, strlen(path) + 1); pVnode->config = info.config; pVnode->state.committed = info.state.committed; pVnode->state.commitTerm = info.state.commitTerm; From 63a6428e341ffd0a3d012332daea125d16976b16 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 17 Dec 2024 17:59:53 +0800 Subject: [PATCH 074/108] fix(stream):adjust status of ins_streams --- source/dnode/mnode/impl/src/mndStream.c | 11 -- source/dnode/mnode/impl/src/mndStreamUtil.c | 48 +++++- tests/script/tsim/stream/pauseAndResume.sim | 162 ++++++++++++++++++++ 3 files changed, 207 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0a107518df..32446a6fd3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1680,11 +1680,6 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { mInfo("stream:%s,%" PRId64 " start to pause stream", pauseReq.name, pStream->uid); - if (pStream->status == STREAM_STATUS__PAUSE) { - sdbRelease(pMnode->pSdb, pStream); - return 0; - } - if ((code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb)) != 0) { sdbRelease(pMnode->pSdb, pStream); return code; @@ -1773,7 +1768,6 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { // pause stream taosWLockLatch(&pStream->lock); - pStream->status = STREAM_STATUS__PAUSE; code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); if (code) { taosWUnLockLatch(&pStream->lock); @@ -1824,11 +1818,6 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { } } - if (pStream->status != STREAM_STATUS__PAUSE) { - sdbRelease(pMnode->pSdb, pStream); - return 0; - } - mInfo("stream:%s,%" PRId64 " start to resume stream from pause", resumeReq.name, pStream->uid); if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { sdbRelease(pMnode->pSdb, pStream); diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index bb666eb6dd..ef0b3fa9cc 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -854,8 +854,7 @@ int32_t mndResetChkptReportInfo(SHashObj* pHash, int64_t streamId) { return TSDB_CODE_MND_STREAM_NOT_EXIST; } -static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { - int8_t status = atomic_load_8(&pStream->status); +static void mndShowStreamStatus(char *dst, int8_t status) { if (status == STREAM_STATUS__NORMAL) { strcpy(dst, "ready"); } else if (status == STREAM_STATUS__STOP) { @@ -891,6 +890,41 @@ static void int64ToHexStr(int64_t id, char *pBuf, int32_t bufLen) { varDataSetLen(pBuf, len + 2); } +static int32_t isAllTaskPaused(SStreamObj *pStream, bool *pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStreamTaskIter *pIter = NULL; + bool isPaused = true; + + taosRLockLatch(&pStream->lock); + code = createStreamTaskIter(pStream, &pIter); + TSDB_CHECK_CODE(code, lino, _end); + + while (streamTaskIterNextTask(pIter)) { + SStreamTask *pTask = NULL; + code = streamTaskIterGetCurrent(pIter, &pTask); + TSDB_CHECK_CODE(code, lino, _end); + + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); + if (pe == NULL) { + continue; + } + if (pe->status != TASK_STATUS__PAUSE) { + isPaused = false; + } + } + (*pRes) = isPaused; + +_end: + destroyStreamTaskIter(pIter); + taosRUnLockLatch(&pStream->lock); + if (code != TSDB_CODE_SUCCESS) { + mError("error happens when get stream status, lino:%d, code:%s", lino, tstrerror(code)); + } + return code; +} + int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_t numOfRows) { int32_t code = 0; int32_t cols = 0; @@ -939,7 +973,15 @@ int32_t setStreamAttrInResBlock(SStreamObj *pStream, SSDataBlock *pBlock, int32_ char status[20 + VARSTR_HEADER_SIZE] = {0}; char status2[20] = {0}; - mndShowStreamStatus(status2, pStream); + bool isPaused = false; + code = isAllTaskPaused(pStream, &isPaused); + TSDB_CHECK_CODE(code, lino, _end); + + int8_t streamStatus = atomic_load_8(&pStream->status); + if (isPaused) { + streamStatus = STREAM_STATUS__PAUSE; + } + mndShowStreamStatus(status2, streamStatus); STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); TSDB_CHECK_NULL(pColInfo, code, lino, _end, terrno); diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim index 1f4caf5c03..66d26dad49 100644 --- a/tests/script/tsim/stream/pauseAndResume.sim +++ b/tests/script/tsim/stream/pauseAndResume.sim @@ -398,4 +398,166 @@ endi print ===== step5 over +print ===== step6 +sql drop database if exists test6; +sql create database test7 vgroups 1; +sql use test7; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); + +sql create stream streams8 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt8 as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams9 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt9 as select _wstart, count(*) c1 from st partition by tbname interval(10s); + +run tsim/stream/checkTaskStatus.sim + +$loop_count = 0 +loop7: + +$loop_count = $loop_count + 1 +if $loop_count == 40 then + return -1 +endi + +sleep 500 + +sql select status, * from information_schema.ins_streams where status != "ready"; + +if $rows != 0 then + print $data00 $data01 $data02 $data03 $data04 + print $data10 $data11 $data12 $data13 $data14 + print $data20 $data21 $data22 $data23 $data24 + print $data30 $data31 $data32 $data33 $data34 + print $data40 $data41 $data42 $data43 $data44 + print $data50 $data51 $data52 $data53 $data54 + goto loop7 +endi + +sql pause stream streams8; + +sql pause stream streams9; + +sql pause stream streams8; + +sql pause stream streams9; + +sleep 1000 + +sql pause stream streams8; + +sql pause stream streams9; + +sleep 1000 + +$loop_count = 0 +loop80: + +$loop_count = $loop_count + 1 +if $loop_count == 40 then + print pause stream failed + goto end_step_6 +endi + +sleep 1000 + +sql select status, * from information_schema.ins_stream_tasks where status != "paused"; + +if $rows != 2 then + print $data00 $data01 $data02 $data03 $data04 + print $data10 $data11 $data12 $data13 $data14 + print $data20 $data21 $data22 $data23 $data24 + print $data30 $data31 $data32 $data33 $data34 + print $data40 $data41 $data42 $data43 $data44 + print $data50 $data51 $data52 $data53 $data54 + goto loop80 +endi + +$loop_count = 0 +loop8: + +$loop_count = $loop_count + 1 +if $loop_count == 40 then + return -1 +endi + +sleep 1000 + +sql select status, * from information_schema.ins_streams where status == "paused"; + +if $rows != 2 then + print $data00 $data01 $data02 $data03 $data04 + print $data10 $data11 $data12 $data13 $data14 + print $data20 $data21 $data22 $data23 $data24 + print $data30 $data31 $data32 $data33 $data34 + print $data40 $data41 $data42 $data43 $data44 + print $data50 $data51 $data52 $data53 $data54 + goto loop8 +endi + + +sql resume stream streams8; + +sql resume stream streams9; + +sql resume stream streams8; + +sql resume stream streams9; + +sleep 1000 + +sql resume stream streams8; + +sql resume stream streams9; + +sleep 1000 + + +$loop_count = 0 +loop90: + +$loop_count = $loop_count + 1 +if $loop_count == 40 then + print pause stream failed + goto end_step_6 +endi + +sleep 1000 + +sql select status, * from information_schema.ins_stream_tasks where status == "paused"; + +if $rows != 0 then + print $data00 $data01 $data02 $data03 $data04 + print $data10 $data11 $data12 $data13 $data14 + print $data20 $data21 $data22 $data23 $data24 + print $data30 $data31 $data32 $data33 $data34 + print $data40 $data41 $data42 $data43 $data44 + print $data50 $data51 $data52 $data53 $data54 + goto loop90 +endi + +$loop_count = 0 +loop9: + +$loop_count = $loop_count + 1 +if $loop_count == 40 then + return -1 +endi + +sleep 1000 + +sql select status, * from information_schema.ins_streams where status != "paused"; + +if $rows != 2 then + print $data00 $data01 $data02 $data03 $data04 + print $data10 $data11 $data12 $data13 $data14 + print $data20 $data21 $data22 $data23 $data24 + print $data30 $data31 $data32 $data33 $data34 + print $data40 $data41 $data42 $data43 $data44 + print $data50 $data51 $data52 $data53 $data54 + goto loop9 +endi + +end_step_6: + +print ===== step6 over + system sh/stop_dnodes.sh From 8fab3e00dd381ddc751ed8905ddcd6d5cf839e63 Mon Sep 17 00:00:00 2001 From: Jing Sima Date: Tue, 17 Dec 2024 18:08:29 +0800 Subject: [PATCH 075/108] docs:[TD-33244] Add version support description to function documentation. --- .../14-reference/03-taos-sql/10-function.md | 55 ++++++++++++++++++- .../14-reference/03-taos-sql/10-function.md | 55 ++++++++++++++++++- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index 4eb7b083fb..e6cfa20bd4 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -190,6 +190,7 @@ ROUND(expr[, digits]) - `digits` less than zero means discarding the decimal places and rounding the number to the left of the decimal point by `digits` places. If the number of places to the left of the decimal point is less than `digits`, returns 0. - Since the DECIMAL type is not yet supported, this function will use DOUBLE and FLOAT to represent results containing decimals, but DOUBLE and FLOAT have precision limits, and using this function may be meaningless when there are too many digits. - Can only be used with regular columns, selection (Selection), projection (Projection) functions, and cannot be used with aggregation (Aggregation) functions. +- `digits` is supported from version 3.3.3.0. **Example**: @@ -249,6 +250,8 @@ TAN(expr) **Function Description**: Obtains the tangent result of the specified field. +**Version**: ver-3.3.3.0 + **Return Result Type**: DOUBLE. **Applicable Data Types**: Numeric types. @@ -297,6 +300,8 @@ TRUNCATE(expr, digits) **Function Description**: Gets the truncated value of the specified field to the specified number of digits. +**Version**: ver-3.3.3.0 + **Return Type**: Consistent with the original data type of the `expr` field. **Applicable Data Types**: @@ -340,6 +345,8 @@ EXP(expr) **Function Description**: Returns the value of e (the base of natural logarithms) raised to the specified power. +**Version**: ver-3.3.3.0 + **Return Type**: DOUBLE. **Applicable Data Types**: Numeric type. @@ -370,6 +377,8 @@ LN(expr) **Function Description**: Returns the natural logarithm of the specified parameter. +**Version**: ver-3.3.3.0 + **Return Type**: DOUBLE. **Applicable Data Types**: Numeric type. @@ -401,6 +410,8 @@ MOD(expr1, expr2) **Function Description**: Calculates the result of expr1 % expr2. +**Version**: ver-3.3.3.0 + **Return Type**: DOUBLE. **Applicable Data Types**: Numeric type. @@ -437,6 +448,8 @@ RAND([seed]) **Function Description**: Returns a uniformly distributed random number from 0 to 1. +**Version**: ver-3.3.3.0 + **Return Result Type**: DOUBLE. **Applicable Data Types**: @@ -484,6 +497,8 @@ SIGN(expr) **Function Description**: Returns the sign of the specified parameter. +**Version**: ver-3.3.3.0 + **Return Result Type**: Consistent with the original data type of the specified field. **Applicable Data Types**: Numeric types. @@ -527,6 +542,8 @@ DEGREES(expr) **Function Description**: Calculates the value of the specified parameter converted from radians to degrees. +**Version**: ver-3.3.3.0 + **Return Result Type**: DOUBLE. **Applicable Data Types**: Numeric types. @@ -558,6 +575,8 @@ RADIANS(expr) **Function Description**: Calculates the value of the specified parameter converted from degrees to radians. +**Version**: ver-3.3.3.0 + **Return Type**: DOUBLE. **Applicable Data Types**: Numeric types. @@ -729,6 +748,8 @@ TRIM([remstr FROM] expr) **Function Description**: Returns the string expr with all prefixes or suffixes of remstr removed. +**Version**: ver-3.3.3.0 + **Return Result Type**: Same as the original type of the input field expr. **Applicable Data Types**: @@ -807,6 +828,8 @@ SUBSTRING/SUBSTR(expr FROM pos [FOR len]) - If `len` is less than 1, returns an empty string. - `pos` is 1-based; if `pos` is 0, returns an empty string. - If `pos` + `len` exceeds `len(expr)`, returns the substring from `pos` to the end of the string, equivalent to executing `substring(expr, pos)`. +- Function `SUBSTRING` is equal to `SUBSTR`, supported from ver-3.3.3.0. +- Syntax `SUBSTRING/SUBSTR(expr FROM pos [FOR len])` is supported from ver-3.3.3.0. **Examples**: @@ -845,6 +868,8 @@ SUBSTRING_INDEX(expr, delim, count) **Function Description**: Returns a substring of `expr` cut at the position where the delimiter appears the specified number of times. +**Version**: ver-3.3.3.0 + **Return Result Type**: Same as the original type of the input field `expr`. **Applicable Data Types**: @@ -902,6 +927,8 @@ CHAR(expr1 [, expr2] [, expr3] ...) **Function Description**: Treats the input parameters as integers and returns the characters corresponding to these integers in ASCII encoding. +**Version**: ver-3.3.3.0 + **Return Result Type**: VARCHAR. **Applicable Data Types**: Integer types, VARCHAR, NCHAR. @@ -949,6 +976,8 @@ ASCII(expr) **Function Description**: Returns the ASCII code of the first character of the string. +**Version**: ver-3.3.3.0 + **Return Result Data Type**: BIGINT. **Applicable Data Types**: VARCHAR, NCHAR. @@ -979,6 +1008,8 @@ POSITION(expr1 IN expr2) **Function Description**: Calculates the position of string `expr1` in string `expr2`. +**Version**: ver-3.3.3.0 + **Return Result Type**: BIGINT. **Applicable Data Types**: @@ -1026,6 +1057,8 @@ REPLACE(expr, from_str, to_str) **Function Description**: Replaces all occurrences of `from_str` in the string with `to_str`. +**Version**: ver-3.3.3.0 + **Return Type**: Same as the original type of the input field `expr`. **Applicable Data Types**: @@ -1061,6 +1094,8 @@ REPEAT(expr, count) **Function Description**: Returns a string that repeats the string `expr` a specified number of times. +**Version**: ver-3.3.3.0 + **Return Type**: Same as the original type of the input field `expr`. **Applicable Data Types**: @@ -1319,6 +1354,7 @@ TIMEDIFF(expr1, expr2 [, time_unit]) - `expr1`: BIGINT, TIMESTAMP types representing timestamps, or VARCHAR, NCHAR types in ISO8601/RFC3339 standard date-time format. - `expr2`: BIGINT, TIMESTAMP types representing timestamps, or VARCHAR, NCHAR types in ISO8601/RFC3339 standard date-time format. - `time_unit`: See usage instructions. +- `timediff` return the absolute value of the difference between timestamp `expr1` and `expr2` before ver-3.3.3.0. **Nested Subquery Support**: Applicable to both inner and outer queries. @@ -1423,6 +1459,8 @@ WEEK(expr [, mode]) **Function Description**: Returns the week number of the input date. +**Version**: ver-3.3.3.0 + **Return Result Type**: BIGINT. **Applicable Data Types**: @@ -1490,6 +1528,8 @@ WEEKOFYEAR(expr) **Function Description**: Returns the week number of the input date. +**Version**: ver-3.3.3.0 + **Return Type**: BIGINT. **Applicable Data Types**: BIGINT, TIMESTAMP types representing timestamps, or VARCHAR, NCHAR types in ISO8601/RFC3339 date-time format. @@ -1521,6 +1561,8 @@ WEEKDAY(expr) **Function Description**: Returns the weekday of the input date. +**Version**: ver-3.3.3.0 + **Return Type**: BIGINT. **Applicable Data Types**: BIGINT, TIMESTAMP types representing timestamps, or VARCHAR, NCHAR types in ISO8601/RFC3339 date-time format. @@ -1552,6 +1594,8 @@ DAYOFWEEK(expr) **Function Description**: Returns the weekday of the input date. +**Version**: ver-3.3.3.0 + **Return Type**: BIGINT. **Applicable Data Types**: BIGINT, TIMESTAMP types representing timestamps, or VARCHAR, NCHAR types in ISO8601/RFC3339 date-time format. @@ -1707,6 +1751,9 @@ STDDEV/STDDEV_POP(expr) **Applicable to**: Tables and supertables. +**Description**: +- Function `STDDEV_POP` equals `STDDEV` and is supported from ver-3.3.3.0. + **Example**: ```sql @@ -1733,6 +1780,8 @@ VAR_POP(expr) **Function Description**: Calculates the population variance of a column in a table. +**Version**: ver-3.3.3.0 + **Return Data Type**: DOUBLE. **Applicable Data Types**: Numeric types. @@ -1975,7 +2024,8 @@ MAX(expr) **Applicable to**: Tables and supertables. -**Usage Instructions**: The max function can accept strings as input parameters, and when the input parameter is a string type, it returns the largest string value. +**Usage Instructions**: +- The max function can accept strings as input parameters, and when the input parameter is a string type, it returns the largest string value(supported from ver-3.3.3.0, function `max` only accept numeric parameter before ver-3.3.3.0). ### MIN @@ -1991,7 +2041,8 @@ MIN(expr) **Applicable to**: Tables and supertables. -**Usage Instructions**: The min function can accept strings as input parameters, and when the input parameter is a string type, it returns the largest string value. +**Usage Instructions**: +- The min function can accept strings as input parameters, and when the input parameter is a string type, it returns the largest string value(supported from ver-3.3.3.0, function `min` only accept numeric parameter before ver-3.3.3.0). ### MODE diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 7799e6f50e..c075545ff3 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -193,6 +193,7 @@ ROUND(expr[, digits]) - `digits` 小于零表示丢掉小数位，并将数字四舍五入到小数点左侧 `digits` 位。若小数点左侧的位数小于 `digits`位，返回 0。 - 由于暂未支持 DECIMAL 类型，所以该函数会用 DOUBLE 和 FLOAT 来表示包含小数的结果，但是 DOUBLE 和 FLOAT 是有精度上限的，当位数太多时使用该函数可能没有意义。 - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +- `digits` 从 ver-3.3.3.0 开始支持。 **举例**： ```sql @@ -268,6 +269,8 @@ PI() **功能说明**：返回圆周率 π 的值。 +**版本**: ver-3.3.3.0 + **返回结果类型**：DOUBLE。 **适用数据类型**：无。 @@ -295,6 +298,8 @@ TRUNCATE(expr, digits) **功能说明**：获得指定字段按照指定位数截断的值。 +**版本**: ver-3.3.3.0 + **返回结果类型**：与 `expr` 字段的原始数据类型一致。 **适用数据类型**： @@ -333,6 +338,8 @@ EXP(expr) ``` **功能说明**：返回 e（自然对数的底）的指定乘方后的值。 +**版本**: ver-3.3.3.0 + **返回结果类型**：DOUBLE。 **适用数据类型**：数值类型。 @@ -360,6 +367,8 @@ LN(expr) **功能说明**：返回指定参数的自然对数。 +**版本**: ver-3.3.3.0 + **返回结果类型**：DOUBLE。 **适用数据类型**：数值类型。 @@ -388,6 +397,8 @@ MOD(expr1, expr2) **功能说明**：计算 expr1 % expr2 的结果。 +**版本**: ver-3.3.3.0 + **返回结果类型**：DOUBLE。 **适用数据类型**：数值类型。 @@ -421,6 +432,8 @@ RAND([seed]) **功能说明**：返回一个从0到1均匀分布的随机数。 +**版本**: ver-3.3.3.0 + **返回结果类型**：DOUBLE。 **适用数据类型**： @@ -464,6 +477,8 @@ SIGN(expr) **功能说明**：返回指定参数的符号。 +**版本**: ver-3.3.3.0 + **返回结果类型**：与指定字段的原始数据类型一致。 **适用数据类型**：数值类型。 @@ -504,6 +519,8 @@ DEGREES(expr) **功能说明**：计算指定参数由弧度值转为角度后的值。 +**版本**: ver-3.3.3.0 + **返回结果类型**：DOUBLE。 **适用数据类型**：数值类型。 @@ -532,6 +549,8 @@ RADIANS(expr) **功能说明**：计算指定参数由角度值转为弧度后的值。 +**版本**: ver-3.3.3.0 + **返回结果类型**：DOUBLE。 **适用数据类型**：数值类型。 @@ -702,6 +721,8 @@ TRIM([remstr FROM] expr) **功能说明**：返回去掉了所有 remstr 前缀或后缀的字符串 epxr 。 +**版本**: ver-3.3.3.0 + **返回结果类型**：与输入字段 epxr 的原始类型相同。 **适用数据类型**： @@ -773,6 +794,8 @@ SUBSTRING/SUBSTR(expr FROM pos [FOR len]) - 若 `len` 小于 1，返回空串。 - `pos` 是 1-base 的，若 `pos` 为 0，返回空串。 - 若 `pos` + `len` 大于 `len(expr)`，返回从 `pos` 开始到字符串结尾的子串，等同于执行 `substring(expr, pos)`。 +- `SUBSTRING` 函数等价于 `SUBSTR`, 从 ver-3.3.3.0 开始支持。 +- `SUBSTRING/SUBSTR(expr FROM pos [FOR len])` 语法从 ver-3.3.3.0 开始支持。 **举例**： ```sql @@ -809,6 +832,8 @@ SUBSTRING_INDEX(expr, delim, count) **功能说明**：返回字符串 `expr` 在出现指定次数分隔符的位置截取的子串。 +**版本**: ver-3.3.3.0 + **返回结果类型**：与输入字段 `expr` 的原始类型相同。 **适用数据类型**： @@ -862,6 +887,8 @@ CHAR(expr1 [, expr2] [, epxr3] ...) **功能说明**：将输入参数当作整数，并返回这些整数在 ASCII 编码中对应的字符。 +**版本**: ver-3.3.3.0 + **返回结果类型**：VARCHAR。 **适用数据类型**：整数类型,VARCHAR,NCHAR。 @@ -906,6 +933,8 @@ ASCII(expr) **功能说明**：返回字符串第一个字符的 ASCII 码。 +**版本**: ver-3.3.3.0 + **返回结果数据类型**：BIGINT。 **适用数据类型**：VARCHAR, NCHAR。 @@ -933,6 +962,8 @@ POSITION(expr1 IN expr2) **功能说明**：计算字符串 `expr1` 在字符串 `expr2` 中的位置。 +**版本**: ver-3.3.3.0 + **返回结果类型**：BIGINT。 **适用数据类型**： @@ -975,6 +1006,8 @@ REPLACE(expr, from_str, to_str) ``` **功能说明**：将字符串中的 `from_str` 全部替换为 `to_str`。 +**版本**: ver-3.3.3.0 + **返回结果类型**：与输入字段 `expr` 的原始类型相同。 **适用数据类型**： @@ -1005,6 +1038,8 @@ REPEAT(expr, count) ``` **功能说明**：返回将字符串重复指定次数得到的字符串。 +**版本**: ver-3.3.3.0 + **返回结果类型**：与输入字段 `expr` 的原始类型相同。 **适用数据类型**： @@ -1260,6 +1295,7 @@ TIMEDIFF(expr1, expr2 [, time_unit]) - `expr1`：表示时间戳的 BIGINT, TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR, NCHAR 类型。 - `expr2`：表示时间戳的 BIGINT, TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR, NCHAR 类型。 - `time_unit`：见使用说明。 +- ver-3.3.3.0 之前该函数结果为时间戳 `expr1` 和 `expr2` 的差值的绝对值，结果为正数。 **嵌套子查询支持**：适用于内层查询和外层查询。 @@ -1361,6 +1397,8 @@ WEEK(expr [, mode]) ``` **功能说明**：返回输入日期的周数。 +**版本**: ver-3.3.3.0 + **返回结果类型**：BIGINT。 **适用数据类型**： @@ -1422,6 +1460,8 @@ WEEKOFYEAR(expr) ``` **功能说明**：返回输入日期的周数。 +**版本**: ver-3.3.3.0 + **返回结果类型**：BIGINT。 **适用数据类型**：表示时间戳的 BIGINT, TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR, NCHAR 类型。 @@ -1449,6 +1489,8 @@ WEEKDAY(expr) ``` **功能说明**：返回输入日期是周几。 +**版本**: ver-3.3.3.0 + **返回结果类型**：BIGINT。 **适用数据类型**：表示 表示时间戳的 BIGINT, TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR, NCHAR 类型。 @@ -1476,6 +1518,8 @@ DAYOFWEEK(expr) ``` **功能说明**：返回输入日期是周几。 +**版本**: ver-3.3.3.0 + **返回结果类型**：BIGINT。 **适用数据类型**：表示时间戳的 BIGINT, TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR, NCHAR 类型。 @@ -1633,6 +1677,9 @@ STDDEV/STDDEV_POP(expr) **适用于**：表和超级表。 +**说明**： +- `STDDEV_POP` 函数等价于 `STDDEV` 函数，从 ver-3.3.3.0 开始支持。 + **举例**： ```sql taos> select id from test_stddev; @@ -1656,6 +1703,8 @@ VAR_POP(expr) **功能说明**：统计表中某列的总体方差。 +**版本**: ver-3.3.3.0 + **返回数据类型**：DOUBLE。 **适用数据类型**：数值类型。 @@ -1898,7 +1947,8 @@ MAX(expr) **适用于**：表和超级表。 -**使用说明**：max 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值。 +**使用说明**： +- max 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值，从 ver-3.3.3.0 开始支持，之前的版本不支持字符串参数。 ### MIN @@ -1914,7 +1964,8 @@ MIN(expr) **适用于**：表和超级表。 -**使用说明**：min 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值。 +**使用说明**： +- min 函数可以接受字符串作为输入参数，当输入参数为字符串类型时，返回最大的字符串值，从 ver-3.3.3.0 开始支持，之前的版本不支持字符串参数。 ### MODE From 7488acc3b1dd8d93c7c562d5510c17f0e6b33c47 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 Dec 2024 18:41:30 +0800 Subject: [PATCH 076/108] ut: minor changes --- source/os/src/osThread.c | 1 + source/os/test/osThreadTests.cpp | 57 +++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index abb88d2c6b..603f5da3f7 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -702,6 +702,7 @@ int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr *attr) { } int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr *attr, int32_t *pshared) { + OS_PARAM_CHECK(attr); OS_PARAM_CHECK(pshared); #ifdef __USE_WIN_THREAD if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE; diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index 5650f694fd..4badfcdb4d 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -56,13 +56,31 @@ static void *funcPtr200(void *param) { taosThreadGetSchedParam(thread, &policy, ¶); taosThreadGetSchedParam(thread, NULL, ¶); taosThreadGetSchedParam(thread, &policy, NULL); - taosThreadSetSchedParam(NULL, 0, ¶); + // taosThreadSetSchedParam(NULL, 0, ¶); taosThreadSetSchedParam(thread, 0, ¶); return NULL; } -static void *funcPtr500(void *param) { +static void *funcPtr501(void *param) { + taosMsleep(500); + TdThread thread = taosThreadSelf(); + return NULL; +} + +static void *funcPtr502(void *param) { + taosMsleep(500); + TdThread thread = taosThreadSelf(); + return NULL; +} + +static void *funcPtr503(void *param) { + taosMsleep(500); + TdThread thread = taosThreadSelf(); + return NULL; +} + +static void *funcPtr504(void *param) { taosMsleep(500); TdThread thread = taosThreadSelf(); return NULL; @@ -92,14 +110,14 @@ TEST(osThreadTests, thread) { reti = taosThreadCancel(tid1); EXPECT_EQ(reti, 0); - reti = taosThreadCreate(&tid2, NULL, funcPtr500, NULL); + reti = taosThreadCreate(&tid2, NULL, funcPtr501, NULL); EXPECT_EQ(reti, 0); taosMsleep(1000); reti = taosThreadCancel(tid2); EXPECT_EQ(reti, 0); taosThreadDetach(tid1); - reti = taosThreadCreate(&tid2, NULL, funcPtr500, NULL); + reti = taosThreadCreate(&tid2, NULL, funcPtr502, NULL); EXPECT_EQ(reti, 0); reti = taosThreadDetach(tid2); @@ -111,16 +129,18 @@ TEST(osThreadTests, thread) { reti = taosThreadCreate(&tid2, NULL, funcPtrExit2, NULL); EXPECT_EQ(reti, 0); - reti = taosThreadCreate(&tid2, NULL, funcPtr500, NULL); - EXPECT_EQ(reti, 0); - taosThreadKill(tid2, SIGINT); + taosMsleep(1000); + + // reti = taosThreadCreate(&tid2, NULL, funcPtr503, NULL); + // EXPECT_EQ(reti, 0); + // taosThreadKill(tid2, SIGINT); int32_t policy; struct sched_param para; taosThreadGetSchedParam(tid2, &policy, ¶); taosThreadGetSchedParam(tid2, NULL, ¶); taosThreadGetSchedParam(tid2, &policy, NULL); - taosThreadSetSchedParam(NULL, 0, ¶); + // taosThreadSetSchedParam(NULL, 0, ¶); taosThreadSetSchedParam(tid2, 0, ¶); TdThreadKey key = {0}; @@ -249,7 +269,7 @@ TEST(osThreadTests, cond) { reti = taosThreadCondTimedWait(NULL, &mutex, &abstime); EXPECT_NE(reti, 0); reti = taosThreadCondTimedWait(&cond, &mutex, NULL); - EXPECT_NE(reti, 0); + EXPECT_EQ(reti, 0); TdThreadCondAttr condattr = {0}; (void)taosThreadCondAttrInit(&condattr); @@ -362,8 +382,6 @@ TEST(osThreadTests, rwlock) { EXPECT_NE(reti, 0); reti = taosThreadRwlockTryRdlock(&rwlock); EXPECT_EQ(reti, 0); - reti = taosThreadRwlockTryRdlock(&rwlock); - EXPECT_NE(reti, 0); reti = taosThreadRwlockUnlock(NULL); EXPECT_NE(reti, 0); @@ -416,15 +434,24 @@ TEST(osThreadTests, spinlock) { TdThreadSpinlock lock = {0}; reti = taosThreadSpinInit(&lock, -1); - EXPECT_NE(reti, 0); + EXPECT_EQ(reti, 0); reti = taosThreadSpinLock(&lock); - EXPECT_NE(reti, 0); + EXPECT_EQ(reti, 0); reti = taosThreadSpinTrylock(&lock); EXPECT_NE(reti, 0); reti = taosThreadSpinUnlock(&lock); - EXPECT_NE(reti, 0); + EXPECT_EQ(reti, 0); reti = taosThreadSpinDestroy(&lock); - EXPECT_NE(reti, 0); + EXPECT_EQ(reti, 0); + + reti = taosThreadSpinInit(&lock, -1); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinTrylock(&lock); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinUnlock(&lock); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinDestroy(&lock); + EXPECT_EQ(reti, 0); reti = taosThreadSpinInit(NULL, 0); EXPECT_NE(reti, 0); From 5ea16d3ada367a956e6e60a7ede32191922c82da Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 Dec 2024 19:27:22 +0800 Subject: [PATCH 077/108] ut: minor changes --- source/os/test/osThreadTests.cpp | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index 4badfcdb4d..221c53cbc3 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -34,7 +34,6 @@ static int32_t globalVar = 0; static void funcPtrKey(void *param) { taosMsleep(100); } static void *funcPtr200(void *param) { - taosMsleep(200); TdThread thread = taosThreadSelf(); TdThreadKey key = {0}; @@ -58,6 +57,7 @@ static void *funcPtr200(void *param) { taosThreadGetSchedParam(thread, &policy, NULL); // taosThreadSetSchedParam(NULL, 0, ¶); taosThreadSetSchedParam(thread, 0, ¶); + taosMsleep(200); return NULL; } @@ -107,6 +107,8 @@ TEST(osThreadTests, thread) { EXPECT_NE(reti, 0); reti = taosThreadCreate(&tid1, NULL, funcPtr200, NULL); EXPECT_EQ(reti, 0); + taosMsleep(300); + reti = taosThreadCancel(tid1); EXPECT_EQ(reti, 0); @@ -289,6 +291,9 @@ TEST(osThreadTests, cond) { reti = taosThreadCondAttrGetPshared(&condattr, NULL); EXPECT_NE(reti, 0); + reti = taosThreadCondAttrSetclock(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrDestroy(NULL); EXPECT_NE(reti, 0); reti = taosThreadCondAttrDestroy(&condattr); @@ -399,6 +404,33 @@ TEST(osThreadTests, rwlock) { EXPECT_NE(reti, 0); reti = taosThreadRwlockDestroy(&rwlock); EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockInit(&rwlock, 0); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockTryWrlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockTryWrlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockWrlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockWrlock(&rwlock); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockDestroy(&rwlock); + EXPECT_EQ(reti, 0); } TEST(osThreadTests, rdlockAttr) { From 8b6b653c7865e5de2a44f9ddf741b244fc2c3cee Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 17 Dec 2024 11:38:53 +0000 Subject: [PATCH 078/108] chore: update Jenkinsfile2 --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 249a8d1c9d..71f9fd00f0 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -505,7 +505,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_59 || slave1_63"} options { skipDefaultCheckout() } when { changeRequest() From 02bd68be22a9a59e98142bef26a5433410cd9b53 Mon Sep 17 00:00:00 2001 From: Haolin Wang Date: Mon, 16 Dec 2024 15:50:54 +0800 Subject: [PATCH 079/108] morefix: not allow duplicate table names in stmt2 bind with interlace mode --- include/common/tdataformat.h | 4 +-- source/client/src/clientMain.c | 40 +++++++++++++++++++------ source/common/src/tdataformat.c | 38 +++++++++++------------ source/libs/parser/src/parInsertStmt.c | 10 ++----- tests/script/api/stmt2-insert-dupkeys.c | 27 +++++++++++++++-- 5 files changed, 78 insertions(+), 41 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 0cc30edf82..0b34e882c8 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -378,7 +378,7 @@ typedef struct { TAOS_MULTI_BIND *bind; } SBindInfo; int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray, bool *orderedDup); + SArray *rowArray, bool *pOrdered, bool *pDupTs); // stmt2 binding int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos, @@ -392,7 +392,7 @@ typedef struct { } SBindInfo2; int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray, bool *orderedDup); + SArray *rowArray, bool *pOrdered, bool *pDupTs); #endif diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6eb2f7e9e9..ef2a18d95e 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2166,19 +2166,38 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col pStmt->semWaited = true; } - int32_t code = 0; + SSHashObj *hashTbnames = tSimpleHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR)); + if (NULL == hashTbnames) { + tscError("stmt2 bind failed: %s", tstrerror(terrno)); + return terrno; + } + + int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < bindv->count; ++i) { if (bindv->tbnames && bindv->tbnames[i]) { + if (pStmt->sql.stbInterlaceMode) { + if (tSimpleHashGet(hashTbnames, bindv->tbnames[i], strlen(bindv->tbnames[i])) != NULL) { + code = terrno = TSDB_CODE_PAR_TBNAME_DUPLICATED; + tscError("stmt2 bind failed: %s %s", tstrerror(terrno), bindv->tbnames[i]); + goto out; + } + + code = tSimpleHashPut(hashTbnames, bindv->tbnames[i], strlen(bindv->tbnames[i]), NULL, 0); + if (code) { + goto out; + } + } + code = stmtSetTbName2(stmt, bindv->tbnames[i]); if (code) { - return code; + goto out; } } if (bindv->tags && bindv->tags[i]) { code = stmtSetTbTags2(stmt, bindv->tags[i]); if (code) { - return code; + goto out; } } @@ -2187,26 +2206,29 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col if (bind->num <= 0 || bind->num > INT16_MAX) { tscError("invalid bind num %d", bind->num); - terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; - return terrno; + code = terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; + goto out; } int32_t insert = 0; (void)stmtIsInsert2(stmt, &insert); if (0 == insert && bind->num > 1) { tscError("only one row data allowed for query"); - terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; - return terrno; + code = terrno = TSDB_CODE_TSC_STMT_BIND_NUMBER_ERROR; + goto out; } code = stmtBindBatch2(stmt, bind, col_idx); if (TSDB_CODE_SUCCESS != code) { - return code; + goto out; } } } - return TSDB_CODE_SUCCESS; +out: + tSimpleHashCleanup(hashTbnames); + + return code; } int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 0db88f4b2a..f1aacfed15 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -449,10 +449,11 @@ static int32_t tBindInfoCompare(const void *p1, const void *p2, const void *para * `infoSorted` is whether the bind information is sorted by column id * `pTSchema` is the schema of the table * `rowArray` is the array to store the rows - * `orderedDup` is an array to store ordered and duplicateTs + * `pOrdered` is the pointer to store ordered + * `pDupTs` is the pointer to store duplicateTs */ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray, bool *orderedDup) { + SArray *rowArray, bool *pOrdered, bool *pDupTs) { if (infos == NULL || numOfInfos <= 0 || numOfInfos > pTSchema->numOfCols || pTSchema == NULL || rowArray == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -510,19 +511,17 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, goto _exit; } - if (orderedDup) { + if (pOrdered && pDupTs) { tRowGetKey(row, &rowKey); if (iRow == 0) { - // init to ordered by default - orderedDup[0] = true; - // init to non-duplicate by default - orderedDup[1] = false; + *pOrdered = true; + *pDupTs = false; } else { // no more compare if we already get disordered or duplicate rows - if (orderedDup[0] && !orderedDup[1]) { + if (*pOrdered && !*pDupTs) { int32_t code = tRowKeyCompare(&rowKey, &lastRowKey); - orderedDup[0] = (code >= 0); - orderedDup[1] = (code == 0); + *pOrdered = (code >= 0); + *pDupTs = (code == 0); } } lastRowKey = rowKey; @@ -3255,10 +3254,11 @@ _exit: * `infoSorted` is whether the bind information is sorted by column id * `pTSchema` is the schema of the table * `rowArray` is the array to store the rows - * `orderedDup` is an array to store ordered and duplicateTs + * `pOrdered` is the pointer to store ordered + * `pDupTs` is the pointer to store duplicateTs */ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, - SArray *rowArray, bool *orderedDup) { + SArray *rowArray, bool *pOrdered, bool *pDupTs) { if (infos == NULL || numOfInfos <= 0 || numOfInfos > pTSchema->numOfCols || pTSchema == NULL || rowArray == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -3340,19 +3340,17 @@ int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, bool infoSorte goto _exit; } - if (orderedDup) { + if (pOrdered && pDupTs) { tRowGetKey(row, &rowKey); if (iRow == 0) { - // init to ordered by default - orderedDup[0] = true; - // init to non-duplicate by default - orderedDup[1] = false; + *pOrdered = true; + *pDupTs = false; } else { // no more compare if we already get disordered or duplicate rows - if (orderedDup[0] && !orderedDup[1]) { + if (*pOrdered && !*pDupTs) { int32_t code = tRowKeyCompare(&rowKey, &lastRowKey); - orderedDup[0] = (code >= 0); - orderedDup[1] = (code == 0); + *pOrdered = (code >= 0); + *pDupTs = (code == 0); } } lastRowKey = rowKey; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index cdc9416d36..37d7b2f431 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -323,7 +323,6 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind int32_t code = 0; int16_t lastColId = -1; bool colInOrder = true; - bool orderedDup[2]; if (NULL == *pTSchema) { *pTSchema = tBuildTSchema(pSchema, pDataBlock->pMeta->tableInfo.numOfColumns, pDataBlock->pMeta->sversion); @@ -369,9 +368,7 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind // } } - code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols, orderedDup); - pDataBlock->ordered = orderedDup[0]; - pDataBlock->duplicateTs = orderedDup[1]; + code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols, &pDataBlock->ordered, &pDataBlock->duplicateTs); qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); @@ -692,7 +689,6 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin int16_t lastColId = -1; bool colInOrder = true; int ncharColNums = 0; - bool orderedDup[2]; if (NULL == *pTSchema) { *pTSchema = tBuildTSchema(pSchema, pDataBlock->pMeta->tableInfo.numOfColumns, pDataBlock->pMeta->sversion); @@ -749,9 +745,7 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin pBindInfos[c].bytes = pColSchema->bytes; } - code = tRowBuildFromBind2(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols, orderedDup); - pDataBlock->ordered = orderedDup[0]; - pDataBlock->duplicateTs = orderedDup[1]; + code = tRowBuildFromBind2(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols, &pDataBlock->ordered, &pDataBlock->duplicateTs); qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); diff --git a/tests/script/api/stmt2-insert-dupkeys.c b/tests/script/api/stmt2-insert-dupkeys.c index c056e1bcb0..adab3ddf39 100644 --- a/tests/script/api/stmt2-insert-dupkeys.c +++ b/tests/script/api/stmt2-insert-dupkeys.c @@ -145,7 +145,7 @@ void insert_dist(TAOS* taos, const char *sql) { UINIT(tbs, ts, ts_len, b, b_len, tags, paramv); } -void insert_dup(TAOS* taos, const char *sql) { +void insert_dup_rows(TAOS* taos, const char *sql) { char **tbs, **b; int64_t **ts; int *ts_len, *b_len; @@ -169,6 +169,27 @@ void insert_dup(TAOS* taos, const char *sql) { UINIT(tbs, ts, ts_len, b, b_len, tags, paramv); } +void insert_dup_tables(TAOS* taos, const char *sql) { + char **tbs, **b; + int64_t **ts; + int *ts_len, *b_len; + TAOS_STMT2_BIND **paramv, **tags; + + INIT(tbs, ts, ts_len, b, b_len, tags, paramv); + + for (int i = 0; i < CTB_NUMS; i++) { + sprintf(tbs[i], "ctb_%d", i % 2); + } + + for (int i = 0; i < CTB_NUMS; i++) { + paramv[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + } + insert(taos, tbs, tags, paramv, sql); + + UINIT(tbs, ts, ts_len, b, b_len, tags, paramv); +} + int main() { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); if (!taos) { @@ -180,7 +201,9 @@ int main() { // insert distinct rows insert_dist(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); // insert duplicate rows - insert_dup(taos, "insert into db.? values(?,?)"); + insert_dup_rows(taos, "insert into db.? values(?,?)"); + // insert duplicate tables + insert_dup_tables(taos, "insert into db.? values(?,?)"); taos_close(taos); taos_cleanup(); From 972538cdebf32b3112d1b663fcfc4ef0d18eebea Mon Sep 17 00:00:00 2001 From: Kaili Xu Date: Tue, 17 Dec 2024 21:45:35 +0800 Subject: [PATCH 080/108] chore: restore Jenkinsfile2 --- Jenkinsfile2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 71f9fd00f0..0816bc6310 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -505,8 +505,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_59 || slave1_63"} - options { skipDefaultCheckout() } + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} when { changeRequest() } From 973a2ac93822330877a43fa4bf29de9e55739575 Mon Sep 17 00:00:00 2001 From: Kaili Xu Date: Tue, 17 Dec 2024 21:47:06 +0800 Subject: [PATCH 081/108] chore: restore Jenkinsfile2 --- Jenkinsfile2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 0816bc6310..249a8d1c9d 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -505,7 +505,8 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} + options { skipDefaultCheckout() } when { changeRequest() } From 9564f3ac39b5c2d699754d65c47f1fb5c2ae1a95 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 Dec 2024 23:45:42 +0800 Subject: [PATCH 082/108] ut: fix ci errors --- source/os/test/CMakeLists.txt | 2 ++ tests/unit-test/test.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index c8aa86c852..4022b73819 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -88,12 +88,14 @@ add_test( COMMAND osSystemTests ) +if(TD_LINUX) add_executable(osThreadTests "osThreadTests.cpp") target_link_libraries(osThreadTests os util gtest_main) add_test( NAME osThreadTests COMMAND osThreadTests ) +endif() add_executable(osTimeTests "osTimeTests.cpp") target_link_libraries(osTimeTests os util gtest_main) diff --git a/tests/unit-test/test.sh b/tests/unit-test/test.sh index 21461bc6a5..1789d4a2bd 100755 --- a/tests/unit-test/test.sh +++ b/tests/unit-test/test.sh @@ -40,7 +40,7 @@ pgrep taosd || taosd >> /dev/null 2>&1 & sleep 10 -ctest -E "cunit_test" -j8 +ctest -E "cunit_test" -j8 --verbose ret=$? exit $ret From d56d67f59c00416800c14b21ecac4592f4d17faf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Dec 2024 00:00:31 +0800 Subject: [PATCH 083/108] merge: set git tags --- cmake/taosadapter_CMakeLists.txt.in | 2 +- cmake/taostools_CMakeLists.txt.in | 2 +- cmake/taosws_CMakeLists.txt.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ef6ed4af1d..13826a1a74 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9a6a5329ae..9bbda8309f 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 17446d184d..b013d45911 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 3.0 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From d1ed7c08847a43fc186c141e3166ed274ba4511c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 Dec 2024 09:14:19 +0800 Subject: [PATCH 084/108] ut: fix ci errors --- source/os/test/CMakeLists.txt | 2 ++ source/os/test/osThreadTests.cpp | 8 +++----- tests/unit-test/test.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index 4022b73819..d592168166 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -14,12 +14,14 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) +if(TD_LINUX) add_executable(osAtomicTests "osAtomicTests.cpp") target_link_libraries(osAtomicTests os util gtest_main) add_test( NAME osAtomicTests COMMAND osAtomicTests ) +endif() if(TD_LINUX) add_executable(osDirTests "osDirTests.cpp") diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index 221c53cbc3..20964c86bc 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -109,14 +109,12 @@ TEST(osThreadTests, thread) { EXPECT_EQ(reti, 0); taosMsleep(300); - reti = taosThreadCancel(tid1); - - EXPECT_EQ(reti, 0); + (void)taosThreadCancel(tid1); + reti = taosThreadCreate(&tid2, NULL, funcPtr501, NULL); EXPECT_EQ(reti, 0); taosMsleep(1000); - reti = taosThreadCancel(tid2); - EXPECT_EQ(reti, 0); + (void)taosThreadCancel(tid2); taosThreadDetach(tid1); reti = taosThreadCreate(&tid2, NULL, funcPtr502, NULL); diff --git a/tests/unit-test/test.sh b/tests/unit-test/test.sh index 1789d4a2bd..21461bc6a5 100755 --- a/tests/unit-test/test.sh +++ b/tests/unit-test/test.sh @@ -40,7 +40,7 @@ pgrep taosd || taosd >> /dev/null 2>&1 & sleep 10 -ctest -E "cunit_test" -j8 --verbose +ctest -E "cunit_test" -j8 ret=$? exit $ret From 116c703b1ca1a41db7d676500b56394ea678a9a3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 18 Dec 2024 09:47:36 +0800 Subject: [PATCH 085/108] add test --- tests/script/tsim/stream/pauseAndResume.sim | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim index 66d26dad49..4cc193dd79 100644 --- a/tests/script/tsim/stream/pauseAndResume.sim +++ b/tests/script/tsim/stream/pauseAndResume.sim @@ -556,6 +556,44 @@ if $rows != 2 then goto loop9 endi +run tsim/stream/checkTaskStatus.sim + +sql insert into ts1 values(1648791213001,1,12,3,1.0); + +$loop_count = 0 +loop11: + +$loop_count = $loop_count + 1 +if $loop_count == 40 then + return -1 +endi + +sleep 1000 + +sql select * from streamt8; + +if $rows != 1 then + print $data00 $data01 $data02 $data03 $data04 + print $data10 $data11 $data12 $data13 $data14 + print $data20 $data21 $data22 $data23 $data24 + print $data30 $data31 $data32 $data33 $data34 + print $data40 $data41 $data42 $data43 $data44 + print $data50 $data51 $data52 $data53 $data54 + goto loop11 +endi + +sql select * from streamt9; + +if $rows != 1 then + print $data00 $data01 $data02 $data03 $data04 + print $data10 $data11 $data12 $data13 $data14 + print $data20 $data21 $data22 $data23 $data24 + print $data30 $data31 $data32 $data33 $data34 + print $data40 $data41 $data42 $data43 $data44 + print $data50 $data51 $data52 $data53 $data54 + goto loop11 +endi + end_step_6: print ===== step6 over From 2950fdc210c00fc0356f5d2fcea742714fd16b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=8A=A0=E5=88=A9?= Date: Wed, 18 Dec 2024 10:42:47 +0800 Subject: [PATCH 086/108] =?UTF-8?q?fix:=20=E5=88=A0=E9=99=A4=E5=A4=9A?= =?UTF-8?q?=E4=BD=99=E7=9A=84=E7=BB=93=E5=B0=BE=E5=8D=8A=E6=8B=AC=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/zh/06-advanced/03-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index f5202cddad..333dabd189 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -175,7 +175,7 @@ TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项 用户可以为每个 partition 对应的子表生成自定义的 TAG 值，如下创建流的语句， ```sql -CREATE STREAM output_tag trigger at_once INTO output_tag_s TAGS(alias_tag varchar(100)) as select _wstart, count(*) from power.meters partition by concat("tag-", tbname) as alias_tag interval(10s)); +CREATE STREAM output_tag trigger at_once INTO output_tag_s TAGS(alias_tag varchar(100)) as select _wstart, count(*) from power.meters partition by concat("tag-", tbname) as alias_tag interval(10s); ``` 在 PARTITION 子句中，为 concat（"tag-"， tbname）定义了一个别名 alias_tag， 对应超级表 output_tag_s 的自定义 TAG 的名字。在上述示例中，流新创建的子表的 TAG 将以前缀 'tag-' 连接原表名作为 TAG 的值。会对 TAG 信息进行如下检查。 From f71dfa40e581ca9f5e13c1f3ccd027473a62bd06 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 18 Dec 2024 11:01:23 +0800 Subject: [PATCH 087/108] fix:[TS-5761] error for in operator --- source/libs/scalar/src/filter.c | 22 +++++++++++-------- source/libs/scalar/src/scalar.c | 20 ++++++++++++----- source/libs/scalar/src/sclvector.c | 14 +++++++++--- source/util/src/thashutil.c | 8 +++---- tests/script/tsim/scalar/in.sim | 4 ++-- .../system-test/2-query/ts-5761-scalemode.py | 21 +++++++++++++++++- tests/system-test/2-query/ts-5761.py | 21 +++++++++++++++++- 7 files changed, 85 insertions(+), 25 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e482a1d241..2f3ca504e4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4848,7 +4848,6 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { } SColumnNode *refNode = (SColumnNode *)node->pLeft; - int32_t type = refNode->node.resType.type; SExprNode *exprNode = NULL; if (OP_TYPE_IN != node->opType && OP_TYPE_NOT_IN != node->opType) { SValueNode *valueNode = (SValueNode *)node->pRight; @@ -4857,29 +4856,34 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { valueNode->node.resType.type = TSDB_DATA_TYPE_BIGINT; } exprNode = &valueNode->node; - type = vectorGetConvertType(refNode->node.resType.type, exprNode->resType.type); + int32_t type = vectorGetConvertType(refNode->node.resType.type, exprNode->resType.type); + if (0 != type && type != refNode->node.resType.type) { + stat->scalarMode = true; + } + return DEAL_RES_CONTINUE; } else { SNodeListNode *listNode = (SNodeListNode *)node->pRight; - if (LIST_LENGTH(listNode->pNodeList) > 10) { + if (LIST_LENGTH(listNode->pNodeList) > 10 || OP_TYPE_NOT_IN == node->opType) { stat->scalarMode = true; - return DEAL_RES_CONTINUE; } + int32_t type = -1; exprNode = &listNode->node; SListCell *cell = listNode->pNodeList->pHead; for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; cell = cell->pNext; - int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type); + int32_t tmp = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); if (tmp != 0){ - type = tmp; + stat->scalarMode = true; + if (IS_NUMERIC_TYPE(tmp) && tmp > type){ + type = tmp; + } } + } if (IS_NUMERIC_TYPE(type)){ exprNode->resType.type = type; } - } - if ((0 != type && type != refNode->node.resType.type) || OP_TYPE_NOT_IN == node->opType) { - stat->scalarMode = true; return DEAL_RES_CONTINUE; } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index ed50ccd37b..56f0d5a60f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -201,9 +201,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_ cell = cell->pNext; } - if (taosHashGetSize(pObj) == 0) { - goto _return; - } *data = pObj; colDataDestroy(out.columnData); @@ -386,8 +383,21 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t type = ctx->type.peerType; - if (IS_VAR_DATA_TYPE(ctx->type.selfType) && IS_NUMERIC_TYPE(ctx->type.peerType)){ + int32_t type = -1; + SListCell *cell = nodeList->pNodeList->pHead; + for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + cell = cell->pNext; + int32_t tmp = vectorGetConvertType(ctx->type.selfType, valueNode->node.resType.type); + if (tmp != 0 && IS_NUMERIC_TYPE(tmp) && tmp > type){ + type = tmp; + } + } + if (IS_NUMERIC_TYPE(type)){ + ctx->type.peerType = type; + } + type = ctx->type.peerType; + if (IS_VAR_DATA_TYPE(ctx->type.selfType) && IS_NUMERIC_TYPE(type)){ SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 1)); SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterVar, node, ctx->type.selfType, 2)); } else { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index bc0f12f7b2..4c53658c32 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -2013,9 +2013,17 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara char *pLeftData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter); - if (pLeftVar != NULL && !res){ - pLeftData = colDataGetData(pLeftVar->columnData, i); - res = res || filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterVar); + if (pLeftVar != NULL && taosHashGetSize(pRight->pHashFilterVar) > 0){ + do{ + if (optr == OP_TYPE_IN && res){ + break; + } + if (optr == OP_TYPE_NOT_IN && !res){ + break; + } + pLeftData = colDataGetData(pLeftVar->columnData, i); + res = filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterVar); + }while(0); } colDataSetInt8(pOut->columnData, i, (int8_t *)&res); if (res) { diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index b466e1b351..67441f8bb0 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -152,8 +152,8 @@ uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { if (FLT_EQUAL(f, 0.0)) { return 0; } - if (fabs(f) < FLT_MAX / BASE - DLT) { - int32_t t = (int32_t)(round(BASE * (f + DLT))); + if (fabs(f) < INT32_MAX) { + int32_t t = (int32_t)(floor(f)); return (uint32_t)t; } else { return 0x7fc00000; @@ -168,8 +168,8 @@ uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { if (FLT_EQUAL(f, 0.0)) { return 0; } - if (fabs(f) < DBL_MAX / BASE - DLT) { - int32_t t = (int32_t)(round(BASE * (f + DLT))); + if (fabs(f) < INT32_MAX) { + int32_t t = (int32_t)(floor(f)); return (uint32_t)t; } else { return 0x7fc00000; diff --git a/tests/script/tsim/scalar/in.sim b/tests/script/tsim/scalar/in.sim index a2164675f0..0ffe6f5100 100644 --- a/tests/script/tsim/scalar/in.sim +++ b/tests/script/tsim/scalar/in.sim @@ -44,7 +44,7 @@ if $data20 != @ Time Range: [-9223372036854775808, 9223372036854775807]@ th endi sql select * from tb1 where fbool in (0, 3); -if $rows != 5 then +if $rows != 3 then return -1 endi @@ -69,7 +69,7 @@ if $rows != 10 then endi sql select * from st1 where tbool in (0, 3); -if $rows != 15 then +if $rows != 5 then return -1 endi diff --git a/tests/system-test/2-query/ts-5761-scalemode.py b/tests/system-test/2-query/ts-5761-scalemode.py index c6fea20072..3fbc023eef 100644 --- a/tests/system-test/2-query/ts-5761-scalemode.py +++ b/tests/system-test/2-query/ts-5761-scalemode.py @@ -21,12 +21,17 @@ class TDTestCase: tdSql.execute(f"use db") # super tableUNSIGNED - tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32));") + tdSql.execute("CREATE TABLE st( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32)) tags(t1 binary(32), t2 nchar(32))") + tdSql.execute("create table t1 using st tags('1', '1.7')") + tdSql.execute("create table t2 using st tags('0', '')") + tdSql.execute("create table t3 using st tags('1', 'er')") # create index for all tags tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1.7')") tdSql.execute("INSERT INTO t1 VALUES (1641024000001, 0, 0, 1.7, 0, 0, '0', '')") tdSql.execute("INSERT INTO t1 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") + tdSql.execute("INSERT INTO t2 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") + tdSql.execute("INSERT INTO t3 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") def check(self): tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7") @@ -84,6 +89,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (0, 2)") tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (0, 2, 'sef')") + tdSql.checkRows(2) tdSql.query(f"SELECT * FROM t1 WHERE c7 = 1.7") tdSql.checkRows(1) @@ -106,6 +113,18 @@ class TDTestCase: tdSql.query(f"SELECT * FROM t1 WHERE c7 not in ('', 2)") tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM st WHERE t2 in ('', 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM st WHERE t2 not in ('', 2)") + tdSql.checkRows(4) + + tdSql.query(f"SELECT * FROM st WHERE t1 in ('d343', 0, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM st WHERE t1 in (0, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM st WHERE t1 not in (0, 2)") + tdSql.checkRows(4) + def run(self): self.prepareData() self.check() diff --git a/tests/system-test/2-query/ts-5761.py b/tests/system-test/2-query/ts-5761.py index d734a269dd..de80835a00 100644 --- a/tests/system-test/2-query/ts-5761.py +++ b/tests/system-test/2-query/ts-5761.py @@ -20,12 +20,17 @@ class TDTestCase: tdSql.execute(f"use db") # super tableUNSIGNED - tdSql.execute("CREATE TABLE t1( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32));") + tdSql.execute("CREATE TABLE st( time TIMESTAMP, c1 BIGINT, c2 smallint, c3 double, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32)) tags(t1 binary(32), t2 nchar(32))") + tdSql.execute("create table t1 using st tags('1', '1.7')") + tdSql.execute("create table t2 using st tags('0', '')") + tdSql.execute("create table t3 using st tags('1', 'er')") # create index for all tags tdSql.execute("INSERT INTO t1 VALUES (1641024000000, 1, 1, 1, 1, 1, '1', '1.7')") tdSql.execute("INSERT INTO t1 VALUES (1641024000001, 0, 0, 1.7, 0, 0, '0', '')") tdSql.execute("INSERT INTO t1 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") + tdSql.execute("INSERT INTO t2 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") + tdSql.execute("INSERT INTO t3 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") def check(self): tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7") @@ -83,6 +88,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (0, 2)") tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM t1 WHERE c6 not in (0, 2, 'sef')") + tdSql.checkRows(2) tdSql.query(f"SELECT * FROM t1 WHERE c7 = 1.7") tdSql.checkRows(1) @@ -105,6 +112,18 @@ class TDTestCase: tdSql.query(f"SELECT * FROM t1 WHERE c7 not in ('', 2)") tdSql.checkRows(2) + tdSql.query(f"SELECT * FROM st WHERE t2 in ('', 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM st WHERE t2 not in ('', 2)") + tdSql.checkRows(4) + + tdSql.query(f"SELECT * FROM st WHERE t1 in ('d343', 0, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM st WHERE t1 in (0, 2)") + tdSql.checkRows(1) + tdSql.query(f"SELECT * FROM st WHERE t1 not in (0, 2)") + tdSql.checkRows(4) + def run(self): self.prepareData() self.check() From 8a62d318701ea5bc3aa5ef2c1bae91f4784e7114 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 18 Dec 2024 11:06:33 +0800 Subject: [PATCH 088/108] fix:[TS-5761] error for in operator --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91254cbe2f..6ceb100169 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -705,7 +705,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py -,,n,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py From 6d1648012398b88699d1289c7d4d438701aaf1bf Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:26:45 +0800 Subject: [PATCH 089/108] docs/optimize the description of "SHOW QUERIES".md --- docs/zh/14-reference/03-taos-sql/24-show.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/24-show.md b/docs/zh/14-reference/03-taos-sql/24-show.md index 110c9cee6e..4596a03281 100644 --- a/docs/zh/14-reference/03-taos-sql/24-show.md +++ b/docs/zh/14-reference/03-taos-sql/24-show.md @@ -155,7 +155,7 @@ SHOW QNODES; SHOW QUERIES; ``` -显示当前系统中正在进行的查询。 +显示当前系统中正在进行的写入（更新）/查询/删除。（由于内部 API 命名原因，所以统称 QUERIES） ## SHOW SCORES From 4063de39c578ab0f87f61eb3a9b48fe5f352ab2d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 18 Dec 2024 11:49:32 +0800 Subject: [PATCH 090/108] fix:[TS-5761] error for in operator --- source/libs/scalar/src/filter.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 2f3ca504e4..9eaa7613da 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4860,7 +4860,6 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; } - return DEAL_RES_CONTINUE; } else { SNodeListNode *listNode = (SNodeListNode *)node->pRight; if (LIST_LENGTH(listNode->pNodeList) > 10 || OP_TYPE_NOT_IN == node->opType) { @@ -4873,7 +4872,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { SValueNode *valueNode = (SValueNode *)cell->pNode; cell = cell->pNext; int32_t tmp = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); - if (tmp != 0){ + if (tmp != 0 && tmp != refNode->node.resType.type){ stat->scalarMode = true; if (IS_NUMERIC_TYPE(tmp) && tmp > type){ type = tmp; @@ -4884,7 +4883,6 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (IS_NUMERIC_TYPE(type)){ exprNode->resType.type = type; } - return DEAL_RES_CONTINUE; } } From 8fb4e390b22f56458728a2d130c3a0d857548672 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 Dec 2024 13:48:34 +0800 Subject: [PATCH 091/108] fix: quick response invalid write and job remain issues --- source/libs/qworker/src/qwMem.c | 1 + source/libs/qworker/src/qworker.c | 102 +++++++++++++++++------------- 2 files changed, 60 insertions(+), 43 deletions(-) diff --git a/source/libs/qworker/src/qwMem.c b/source/libs/qworker/src/qwMem.c index d625bb113a..69d4093221 100644 --- a/source/libs/qworker/src/qwMem.c +++ b/source/libs/qworker/src/qwMem.c @@ -153,6 +153,7 @@ int32_t qwRetrieveJobInfo(QW_FPARAMS_DEF, SQWJobInfo** ppJob) { if (atomic_load_8(&pJob->destroyed)) { QW_UNLOCK(QW_READ, &pJob->lock); + taosHashRelease(gQueryMgmt.pJobInfo, pJob); continue; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 641fa03f7a..fdbe93965a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -449,51 +449,64 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes return TSDB_CODE_SUCCESS; } -int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32_t code) { - if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) { - if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - void *rsp = NULL; - int32_t dataLen = 0; - int32_t rawLen = 0; - SOutputData sOutput = {0}; - if (TSDB_CODE_SUCCESS == code) { - code = qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput); +int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWMsg *qwMsg, int32_t code) { + if (QUERY_RSP_POLICY_QUICK != tsQueryRspPolicy) { + return TSDB_CODE_SUCCESS; + } + + SQWTaskCtx *ctx = NULL; + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + if (!QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + goto _return; + } + + void *rsp = NULL; + int32_t dataLen = 0; + int32_t rawLen = 0; + SOutputData sOutput = {0}; + if (TSDB_CODE_SUCCESS == code) { + code = qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput); + } + + if (code) { + qwFreeFetchRsp(rsp); + rsp = NULL; + dataLen = 0; + } + + if (NULL == rsp && TSDB_CODE_SUCCESS == code) { + goto _return; + } + + if (NULL != rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + if (!ctx->dynamicTask) { + qwFreeSinkHandle(ctx); } - - if (code) { - qwFreeFetchRsp(rsp); - rsp = NULL; - dataLen = 0; - } - - if (NULL == rsp && TSDB_CODE_SUCCESS == code) { - return TSDB_CODE_SUCCESS; - } - - if (NULL != rsp) { - bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - - qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); - if (qComplete) { - atomic_store_8((int8_t *)&ctx->queryEnd, true); - if (!ctx->dynamicTask) { - qwFreeSinkHandle(ctx); - } - } - } - - qwMsg->connInfo = ctx->dataConnInfo; - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - - QW_ERR_RET(qwBuildAndSendFetchRsp(ctx, ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); - rsp = NULL; - - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), - dataLen); } } - return TSDB_CODE_SUCCESS; + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + QW_ERR_JRET(qwBuildAndSendFetchRsp(ctx, ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); + +_return: + + if (ctx) { + qwReleaseTaskCtx(mgmt, ctx); + } + + return code; } int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg) { @@ -748,6 +761,8 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + QW_LOCK(QW_WRITE, &ctx->lock); + ctx->ctrlConnInfo = qwMsg->connInfo; ctx->sId = sId; ctx->phase = -1; @@ -767,7 +782,8 @@ _return: if (code) { (void)qwDropTask(QW_FPARAMS()); } - + + QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } @@ -848,7 +864,7 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); - QW_ERR_RET(qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code)); + QW_ERR_RET(qwQuickRspFetchReq(QW_FPARAMS(), qwMsg, code)); QW_RET(TSDB_CODE_SUCCESS); } From 1513f87dfe8a87c81b4f97f78b4534b359eb09ea Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Wed, 18 Dec 2024 13:29:50 +0800 Subject: [PATCH 092/108] docs(taosx): add health state in taosx --- .../06-advanced/05-data-in/health-options.png | Bin 0 -> 19699 bytes docs/zh/06-advanced/05-data-in/index.md | 25 +++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 docs/zh/06-advanced/05-data-in/health-options.png diff --git a/docs/zh/06-advanced/05-data-in/health-options.png b/docs/zh/06-advanced/05-data-in/health-options.png new file mode 100644 index 0000000000000000000000000000000000000000..d20a520a95ac07c47657905a276008ac2eacabfd GIT binary patch literal 19699 zcmc$`WmH_B9snSIgVNnF;Go}9ruiA5KM;08;)*aZ zFiRV<>i|Fmh<_DSbWS;5ann#VeC0Stw)4lvOZe5`&B8cLH1`RESb>W6GyJ8pXxwRI z#S1gt;*X)~<;q%3&7m?2^EeC1#^7DOKts9TG4+(xf4KZK0+vGB5f~vLScVga64wYd zgg$_gh+SNHUO@Bg@$9*evb4MJa*kc$vFA;}!VvqR2*ETU^~`_~y{9sl@N={Od9lBT zkxb;D6#Z{1Dxvq3Et8b^$M;l3u&zW9=xJg<4Fm=G_sm@|}74I>sQlrIG5&QL? z70tqlV-%^_qC`_B3ldFDVyU0&a*_5I^xBNxj1Bbvc-q=%=ct&MmJ z@~qg-&aOXT!O!MMKyR^g;)Kqcu^}sXZV3z4`PpGe7m{nX8u(c6okN~yb`ymYhOZiHD5F*P zv&Q|KsY6HMU2N3OShF4Sm8*=2qNQs|dqnwjSI+Z>6Xl5JZ2|dfgYo3&K3k4}#%hkE zdWnKlW|VfFiXOorEakD7n-Ir1n{ne82P<-3%P|_30JRDoxvPS`*B`|Ms^-BP1Cbww z(6GA6p|n(JLdAAe+}5}{oS%F8;_hXK!LUlC)3FK4Pj>ov60TdL?&rs~I%{vDmqpB` z+Ex?UnDPYQ2N`Gm68FwR?d*6A-t9Pj=<8OQ8T~?X%Z*0IS~OM@W)=J*nV#IIAsies|ywVI*_|hiHFx5XZn4F00#A4)O(<@+bvD!%I>3&4})0-kf zIG7G4f9yqSVdaRyKCyQXd~l$&I_BqQ_wB;RfaAJ0tBG6S*;;1;8Av0xGS@_b0DxIG zrC9<)L6P7_==!Mf>1n@vnzME^ieMZmse>xy@PzOmfwewmL3R>f9PlBnMZ~EF3ck|< zfVGKaFmr2>JX^A~eH^X)#qs4lqbvZJ**Wa0~ZM6MOd%nDR4q)`V>#Unr23lT23}#cm#R3xD-Vk3j#1JDki7SwqOtMzCa6 zC(HJQED*Uj@2lOX7h_t#?gu8`Lj4&o>41kzHZZ{C=K)Oo1Ji!|4Qt!uwXpBnl{!sVjRMT`X;f_=30__UId z%QH*+rjHQQEDMc|{QQ$*Ts{8)Nl+_;VYh#2NE+;jAt7Pw{e|v9isFOenTvt?3K{gd z2=mxS;kh(=`UL#QdJZ4ST7p4pz8EDQc4od1@Z2l0Ib)M;ai_Kib^5N1hqFC@tCxTl zJ88(etIgTFt5||bvheQTE(#XBMzv3d(5xwAYt>?3a^2AZpdJ+r>&MIp`Je5O+On+e zb68i6?m>(Rb1-B)(Q{M^hA>pu$@b#0r$(YJe!QMsjtgtLTx{2bR>^Y&wFbSk5YrM` z>cK&d41bDQk-NRk*qp+0{|a2Js|LrLQv3kurEFP?XiL|9!-m~fE15d%#<%n2Cum8b zoWJl&hZ791ECO(=^0W!r>P}a}oam%BJk%kSioU>1Pu6h+{^Di{KRLXJj}))jkv>^pT(mw<{Wpl}=zC z2@tqNa8h-1Qjm^roD!N`c6pr~FEGaeyrtLVw6ngbr6|f0w#jj>P0c$u3Rw}aKyCzC z6Ugw3^zVd_xvefmK3Le4Ol*<`COxe$-k1(dYJ0@N0O*#DZoW@z@a!UM-gcqMXzMt# zr>(cMk)cUZNp_P?yBbQ1>9vZKb~r6@%~27bU;rXD9FNNVZ{j(ghD5TTzOWeT?3EK8 z4d{!pmWX-}HF9CE@?b}I0Xo*sy1#_oND?n-2#(TM(GRq2DmNlJ48ZG~aewa<1MVgy z#@R#-6bGTN2h#GCD%6tFP|!e6LcEGakd$gXdM6aXDJxz(O3zkN@vMxdoF2@6+L<&!+Ss~#86v&83NZUEu=@n-ML(_TSHn+Fir3T zSolLXTUNTR5tvwa+4I3eAr)OWCCv#5Z;(bC8=nRe05(FJlzXr1#N8QL%CgI(MMDZ3 zGjBHBj6)9etaugI)~y@vBEZ=nz24t&SHP~2Uitw(>?aj|13dyeDnZ%gDbz7)u5 zoj8)B_7FaaCZ~TR7MR&0R=z03)4D&He;6t>qbxJMPWVWRlQ~h+`1{f*k5q%$Xkkg| z3&xipR{rTt4LQxL!-<$oEAWd)Kg|l&d9t2WM=fU<101U7X0*ZE4l4LgjVxSZ3wueJ zQ^aa&XI1RS^Qdy4Nr<|0xBULTW}s#l$Dm4$ zU8$~rpsrTfYE?600VVIX-z5uWtK1X{VEaP>;|mcE<@Fh}H+U{Ku}58tTW}jBWf~OIyqgN$3DIA_w2g2_~p0^TO3^N5|+HafiEI+2|*isg6Yr zwPNGTW6cv~*@tVLw~o1G@%=+wT<&4aCNz`N)~I&5kHLDDeCn3F)&ryTMJr&Zt()Hx z@5VLyElmo3UTr!zDU>4h_Gx@yFscxqS@==(c|P5;f$I|>W~|^LZC2MKzIQM43yDSL zBf&b-ZE0NLBb!H=uh1O|TM8w>Ipe?k1rYr-M3?LFe0}fIA*Mxyp9y1JNT5y*LQe}yV<3iK1qXF z_6H+U6W83%yT2>guuLt>3M?$yIVn%^z?LTA6!!d_75a5?4LC$SxDPO>(>gN%Vba?v zt!c;kuby3}7E`97r>`INWT}A}c1q{Qg;1#RsoXlLk1~5qQ~-E^oLjB*(h-&c{%N_7 zWyvqX6LBC{j(*f|+N76*W7e#?dD)mF3`NN$c( zr?Rte$wwc0D|x*Znva$?*-%9Js&{3}@_$_{~D-HEvntH|`YS8{5 zh8Xs5R#mIA;g;B7DFvZFQ+77%s-lhRcmz1^Q>7;)9RE&(NTBc}>u+dyf%F<0_9YfQik?OIcH{m>gk%<8=r3d<(h@5y)zBAY=)T5ZTVxDx%~BxNXC)D z#~+3PJgdgV#!Re-2q|MlUdC8%%4-W_U`h_O`+SSwM-sxlE zc{CP|(5a#yygk-ZS`|JjDwhPp9azy6jeP$;$Lu6}ev`hXtaElsOwwuOrclDAWFs6L zZWZB`P+jy@7`{m#C*ZiasDxPtHl5&DvWj(XHg0ZQpTr%k-I7&7s8esjvYPmfuDB#r z3CigH8buq*)UtqkZhmq!9v=9n?u1NYuuID}@|V|1Yij?9*TSy@Uj-VCz=%z&dcgR@ zgS;q=sET%}e+Ggg2g2>BR+Qh;k7<8duDafH~MFdNQ(+6-=(+Nj0U}(mD=o1!qBKFPw;7Et-+8tT%ZRf>ZlW^Uo!> z8%UC>!=zKjI$t4zdx*H{FrzFh>l>;b$G+({u^g`$;f5p#llWQI=Vy~mDL?NFgw}+w zva>rXZ?mp0WAP`z0~t?;IcsldT95Vj!8)7vk4bxJE1T(6HjR&2{IA6P&g!EJ(Eb*0 z9v2t<=14Tl<_}ZR+=20qn4X;YgRd=oH;Ff^Yo5aT+LmKs8L%N8b6$bz>PR=56 zNRSM|2JFLgIST)2za)$1SuxD)GB=s|Ut$_DmR5uK&gkIw>0=LY;VG4v)}cL0o-~f3>7%FO-Ywc=-I_0D(-VZ%rf2xrK*6L&2V-2xErs z@x(JYq^uWk@B8U`W1g-u@@gs_GTLXAGabGEFyJmFFNatUC=#=8>3F?eye_jKaFsCd zo?NLpZWRA%A7HK<$Si6;Z~JTKtBv*J5n5V-r(U5YA80?nHQadl$gJoc%|)Dj(Pj6cF`>frI!K7ZAn zuP^mr^wUOLM}qJN07~W4x=7ymvIw6!oSahpMr0_J)HmG$ESxz2UA29f?peS zC@#!ur`I;D@fBGsY-Eh%L#aZ#)-S&9`*WTSrJ`JT0y&5pEdHCK#NxJoW_I%@Y?_kn zZa>GbVVKq?w6qrV9J$0%DPuakS}a+EqV2=1RiVWj8xT(=ZL$FI#Z;t8@rMVhdUUuC zmMz~=V5Ds$#|U|z>%?3Rvj|IN(@Orh?k~^<%s_h6pN)SX*uqs2ah*u;aWOSRb>Iw; z%-m?Q8Z^pjz0?746DIrWPFGqSF(bY9#Wn`x6^w1Xdjlmrc$I7~hX_LfU|21jKHLD@ z+d#Sa>)bSg-0KMzo!wk%Dow^O9j!%4L-`k_4j6mIw|*VI}6x*weDXf8^maJ7ZLHHVt+-a zkjD4Wpo{niye1Ag6DjUW*Iutf`)~zwjxBAwe=J;`Tt!P$ZFVY;9 z2>d{Z{}GN)LwC4N#*a2v2adiJ&;xZfkktbhg`8SKMlnP)uwj`C0eH)`>qgMI^x={c zvpP7q+}iNA3-nDZ(XmzkJx)tx5_wR=7W;(!dyuC%mLkZOTlFHCM2u@+|Aw~Q+QJ5u#r8D1v(1A+;73#_j&f+|L46)rMl(7#RLC_oH6GS*Cz*WUC zQ;p(Ssp588?WiLQl!ii51O9C8cl#Mk0dSW5b(y#T0LvB}TJ1QSWXKurq!k7;r9hPL$VPRvfDP_OIXYa=0p4=FeUw0^+;ToTJCS&0o0&K10Pz5=(0;BC zMZ48$blI@SB?j=fo@N2FE; z-y#^8r*GB@q!6+Fp*;Y!U*I%vP2asq&*g5x_`SU0CklCip#VhU9(QNm&~K{`@{c%K zL`DNw>g@ak4$;N#;)Wmm#PP9?h8hnrJ6U)-$FAb(5Z{K#&Bq^T(zUk}HDltPSj+l4 z&UFHFCUdI5_BUhO0%uNDEZ4sGV2JOovaqzaWK@;~1+Y?useD(xul?XyoJ^G|sel=f z-$uRm`khlHTa{fOkBSPQcs+>iMC)0gQ2MY|fWW3xHQy=lkRR-AtB=_XA(HN)opxbb zF(7F8bdF)S#|U`y+ik&54iFxVoJM$@3@eM66*L8oGCUY~#T+_**7=?aErQdu!o zO6RD2g%o(3N=84D*Wc=auxcT;OG_<#tKXlFrB_Rre=;Ju?|{(77AYvsyHaqBziB!gl{M?ju0!l0B71Rim(Zf5bIVML|R z^BQ}nI|7(?hKgBW>vYKeB_6}dO?l<-)Y>rPDpB3#-^OPma z4EwQkoj6jz+Ra?Rm$Y~^eQdmhxN>MUB7ljAr&!ZV9hTZehjL zp*5GQgF~LotKyu;0!?O%s%vfs>*=oT^>iAqiu{~72j17coJj>Testb4i#s}f~OAbQAK@gr)&+~@@e2C539+Wx>k!ysA&{y zZ@E%O(}lvPVKqel8fFy zZxrCmIIGFwR5-hD&6S#6(Rkn4gA{;9%sSC~C$nL%{B{&*8Y#(s{3@t6HLZH3uo_Ub z=TCleKX72a__lrRXN){;znBu2Bs?}6=*nC*a8SI^Kuf+cLWn~rBl(hPd=}<*@kDQN z3-7et@Tk4llu}E-b2`H`rE+HB$YsVS4(G9ftG*`w(~$@7#=LaOv}obNT52+x--K>- zbm}1d>+j*GE493nqr-rE!gaxj3=yLVOace>{-hH~PZDqhsXk zU6ggN#JOnZ^^@?2z)e`Q0^c(&3A-#IL5qaaccbtGs_22Eq@SbO@~k}U`qQ20c!ng27NuFwgH?f3{O^2h|%%hu>3)!`?LZOJW76k5$bqxd#$0u8s) zr9zL4XR}j#!HVUNkIq$7SbK483O(YxHIl2oybMq_gJ7^~43%J}r&8hL6b`h12j&Op z2Np2zV^fR#N|_y(i-1=<1L@cRZX6|ksv{;zdGN#SW88e4|TWv+ja zswH6AMG!{~1IIkWrb#&)f$n7WmSNhe2v}o%_)lR?qg3h9SO%7d=vdr}avB>vxH4kL5HN1&8HMAGh*1KxYJciHrT2 z%;k{~M$7tg%4;p9|H6mgD;KnTk9Awj3b%&@%swC8P`m&?1Gus2?t4;~T-jgAEFTG* zI|(FOW&eHveq2kQG|!GR`dMP|@9wQ7N30)G zTEfkyS;D7|9H_4iF1st6rs1H`CK(X&2O_S)zw`fzPKkH%@#O0&ZU&p4V|{60&+8`va1e=1JZ2g!b9` zl5kKAObaP_jiK4^s9HCsM4cHA=t2+a;_no{D5lAAQXx2{snQb;344M8?UV8k{4G|~ z#R6v=m}B~|J*9Z%dM31m=5cU%T{*eL2mKM1T2N}nnx?(_*oCbYeZlOfpl&aA=PP z((U3|aX`8q3PROAGoKu~G=a3VG~LJud9-Og#`wD;v#kf7Iqr3+NFVQZL;jyK<3_04 zd*T16i=Y2VVSx&GZvjlhr~l_d<^TJvSPSOO*@AmE4=LZPXHV9&A``X!P2H588QsrQ z>0gZ-7@3KDmsR(F{+}8Ll3iW4AcRahl~j~#Yigw75pQm8a^%XN2#Nj1V?Bq*+SrLZ zX1*`C9y>PG)+m)I5AE2t3Rw3jACrkh`*r9fVB%2MAFANJ_X0ZvPAHoaEfkO{U4R1Zio?KJ8C8pjj|39VVJ}lQhy5 zvvw~xF1@KC5?v^+Qf3P6O;l0FBf!YQ(>^2kUdX6w*en{sQfQ*4#Ke>Bmhi5gdN#6G0r!Jj8@zl3&zNH35_)v>$upOe5675A8tv}sL` znIqbF&bNs(si;`G654jDp0JTynkce7>*@Xx)YcPk%5MC99G0+pWt|!S(XkL)il?*kTj>7OyC;ocb*H(^V(I(;o9g!q`ZdcCFkpNd7A^JXk{d|>j}`W~?S zm2D>d=^oxtJZqxY0QMz^iUzflp1}DOUSDtVOgTW|>2MCDvt4{LyHS5v5IH~iLuHGY z5w*WvhN=G5om;yMcS!i68h* z{3pFZ$;n6`5jW*OVjQW$n)c>%PKAR=;Hr3Pe>NAU6PW>^9M$53HpX7e&qFSl^`I#E zD1|39y?|SHo8CxU!>Wa^y5a^Ml5xFcK>@)jteE)p{j~CCu(^zKN&8lnS~Zdil4AA4 z;eu_&k43WltoOXBqN(HbopFFSJFu%OZp^2kNXdRoS_lf6lvFYsnBDvyjmEI)Qspoagc#)@lu(-Y!Rsxh0ZK-D`&obfVUXy9@*FkhS! zFUne2oa`KC@h$qqA%T=GXX~RMI#=59Qg+T@biu99W&*kYg1)!aGYZXG?U=i(6C!CFwi zv|nDhV&{oISEL20i}kK~i{zU5*OeUzAYZ`C-5VbM4^(Q58McGo1=hO4QZ%myuMPge zV+j6a36@!MG0;w#_|l)my93ci)Z2+K+cN@#Sif~D#_p-avZ{>>T1<81o1Od$XlHG2 znPSS#SNd;He-E~M$;x*3HF4vMG@lyM=bjYDCtSq`YKQBlSfD*OMSJr6!r3CGjb`tB z^1fw91kVFRXEd=CVpTyB3~48^qZfdglFP4OUz{3JJVZ3p|FCOqZEs`lVWux?*#~Wq zNVr?S4!XFrG5Y`d97uuKuK5{b;IS_%2$hgvxb#28y+n4|-fc$U6P}i;&Kr_L-&m}i z(3;eY*{q$8*W=wRfpNQn%ugOnQ_0Vic!?J}# z1RM64(O@RJ9r7Y5@IdEpDW0h1=xb3g2dWsF*c@_S2yZ=}TaTK!BImt>CUkoMrSIEu zVV#WO%Y2e^595NCl4E2fFNx{aWUF#)uk(H|p9S_!QX%dS!d9nV=O4P0L_d*U3`cUk z;vZUgRBi3A2>E>Wkkhn3=f9juj`1{c?(fIBB{VY3mJP(3^AmBx$ut9TY$E0A#1TL=qV-dKu4Y4Hf0Imz9Q;7U=2P z*t6k11SfK_mB7wnA1I}~)Qt`!VJr>~h%LBMf&Ao~@u`{@hZzB-CfL&9fW80;Lm&Zf zi$`;RIHv!%CU(=C_Aj2-(*ADlp3aRQQU6j4kCJ|UJjPUc#=UUt~k^u|}cm4NcI z=QDG7mEPIb)I2cd^)y-e#+T?}d&TR7$7|1QVCY%t2g~Rry(sL)j!*4E#S9gHEdrA% z9yVamhz1E3M(?BJIu1;~t=KTTcJ0+YwX?|v+W(9_6oN2pv>%LM7Zx-2C>iVxkI|In zgVRr1CEb*KeBA1Ot|k~Md>KqxoG+Tg!1+N&dr-k!f%t)<2rekBbc{yE({e)a$JBx`J^I9)TG`jID2 zVRceS`Gnrxf9!cXU!)C_u)@2OI+|?k$CM&T|5Tv6`=XNhCQQo3}!h0bummE$8~N zFl7r%(mbkrLBAgw9{>>Ip2kxwti1~wf9L~k+t54Ad}64SNN5_j+_n$iD!Cf{zmPOj zLwj}*Jej;|J}8%Ya8aueL{{n@1mK=VVJmWCl{T;f{uEfhlvd3K)IlXzQ89 z_O8h6<>qRVO0gIIB{t};g+JMiG45BlURRG9FZfq|2Db`mkd^{_8)TyViY2IT3nh^# zMXuV|c#PTEZGhyttK_27*0%mpDl7L9N!U(36(7Lk<+{l{ zV_ir4NQeLsUF-Um5I+_Dbmf@IFr}n4FTcefnYYd#?~gtA!%M2EkuZg{M9*Wu)fc1M zwX3W`f?(&O@~jV0RF7#ZO2Q=Huw6JEFH*sz4u+ArTzN5*v1v&VI1GYQ67ni{4cq}o z@--jeUL}IfEGQbP3O@paLXyz0P}9jTrKYSahfU)7K^vT2KuO*IOgYy`^m#bm=!AK8 z=A-Ta3q}@$jVrGxm!@hJb5SJu-4={m2HjLXXwQ$~K1u2BjyRT$JxFI(=?!eCt@Swz zI!f4_x%dnKT(L-w4i>^W?F=CdvO!eToG)#3+Vk;WE6pvE^!sp%8Hd}~;uWxHebbYTz`|m!)qp?u2f0F0s?tz z$qYsWQ)X^>2(nu9$9@j*(BzxrJ%LQkDI~Y|&IjeZF=?Oq>bCV(opu%ppc&Po1Y8d7 z?BWm*I59?pB@6obua!!p1`=zWO^2KPW`ezZQEpUr zmiVdng3ynQ{xrDv^kNZR9aW~h1hBxq+i*6%w{2RtN6LsnoHW?E1l&O3#*XZaX~w1C zi1?e`Mpqgyx6#F>UT8k_H3doA=YGXZs@3zzl=nV57LWc(Cnk_ z=dv?5`=~$}Nj`!+WAFvRZMe)S_hgK>>a_)eMTo597dVh_F`Zf_KNbYz4!V<8(pQj0mwj)9WbB74X{$A!^f9|bT2ER&ibO4^Pkh0An=eL{uj*q zVqbCqcVz7BsjUwJ=e33WCS6P2)^WLS;^nTuPcy2-O-t4&zOBvi25ZV`$10@P^L-`n zXzX!+_}=(%A&Z{w{`?1kV_#_zc03=#I2=K4p%|X86Xm)PTt)irL2NnwNwZTmkLeIM zHc{fwLn!#wL^8aETN{eoeA`Zl#@#`ba$iEU+H;Tt{o>T%_P2cYmYT8OPYj=*D> z(fZ{3GkD#K4U2sQQ5^DDWp{%F&Tds8JY#ut#Od|C@&cUzHpJFK>R_W2Jsf&Sb|Po_ zs4kcjiowR9yTVL$Cz^t~qE&iT#qjPxhH6zJ!Jj`-7PTqt{h5rZdugH6L7QQBi!=T@$&60a{2L=J&0D~l7mxVn0JDDV%U zr(2wzrBcY%&V z3X*nJ0|vHs4y5m(voT44@;aR>b_joW;xZxXUz`K(-TRyWJp=PkL;ojx^}llX|Aqh& zwQ~Sqq!Yrz!q`e`OG-i|a!?^R4y`TWrO2l?tj{0ZTjln*wzdwfK^0voTn5qPMfw6J z;P&?R8qWuL~jYx_iIOSvyKhZRzAqC`UN;Yp1-!Dxk<6JTie;OIN}si(~BGYmE*TJl{L5Ji}$oz|(GgG9>ZWaL0hDD5hbiK{%nFSFrV}$hk_)d7rBMs7CBct^K-;DZ7Usrsp z{Uddeq-QCf=Dc#4Lbco~8L8QM-v(|gZRVE#Awo7mnwjszKIkHsPFZrJGr1llOow@k@tC7$6P)Uo*9Y`uyAQB41>wl2m5>cQTS^9%3%_<( zuocpnzvL;*9(GVLin40lWXwBhxzT@sLcqr48d#iSN&evI8Y!W@$dfelRe zQdw-Im>n^i-Pti#<`(5|-_~Ko>D4}P_kTR@>CKDlMWj=#J1nVv)%7n3@7A2e0g)#Z zpTC0Y1tTq!C2jTB8jWy!+v9tZQtr3lzu?{l#gs*9sr6E&&zl-=eA#?wS7@9wTqP6c zmEZkQO>HGQqlm~&g67)X22rZcU)H`Y0KflsncW@km1?E2MF@fe&XB#33c)xt7`1;s zaS8bTFx(=eIJe`oDw!4v`II4+H4Qk0DT;!Dy0JyMUp0=ksNz1Nas;QvMAm+$@Vc-( zPsJ)k@$6CkwE~GHqTECC(b2oS75Am3gj4(ZOinF%7Qvjb?E!u(J*eSO%l)N?=gT+L z&8P=qNi{o6F&YW@a11scjL*gqCD>L3{TO7&F7$sHu*#O-{zbmAfPJgFY|>8@N#cS4 zOXKNDaWRF@(SS>kG2p*wuo-AnDi8$5vAli}>plcXJ^+ll^QF3ehp|s@HLV)WT#+TT zmc>(!rm#u>LC{pT5)6<29v+k3Ryves4U@`DIHDXG_iJsV`q9G8ouy$#B>3{(6$~Ju z)?G_#MFu0^x#)}*p*F0V5Z$6gMV%!J4DiG;{g>@Lbj|tZAONZ@Zz_K(yE8W~40|K0 zLL2XWPq(DWTC3j^9TA8942Kk7C=OK8?UfJW5G14Wkj6Wxc1Wf)bLeg*<(%>0V~OT7 zzba^mf&bB;BEZC!!7_+J8+WfZSYNl6!a1K@_8_ zieV}?P+W5e2hBqM(>l;rY+wUTLgsuB1AcNd0HMs_kxnf(g)8*~`ge&%{j;`4kedLy zzqZ&5?^$X@Qtf6w02h-NSA6^Am96Mv5T3PQshU&)(|txqI*el&S}KVR(d7iwR*_)X z^C^Mqcq7}QjLwT>K;=pJG|qji<|u@z_NaJuLN#NTl&|&rrS19kCE(%low?b4{lRTl zb%=dIC2qxoAKyQ*9g_$Kcw`wwxtTdA*HIWRhjde+L=Rhv$C3?;lH}pNy6jI0Gd=H;n8t zOOE{BR9G0%8VDw-7e1O02}B01v^Z*^?jY=-7!v~fPt&05r+BANo{zN4Q=9CY?1tLt zu!Cfe${n~ZNe*++ceNa7B5EbgYXSbmJj&&@%tDI_@M$G8DJdJN^RZOxcg}-86`PPQ z{3CMeWH&lg5NR4K;1sAx3coGM0bmTKh?bGjprVpIpjL2rnqJd3wr0zFi2%GUkU;cM zzgz=(5zRY+s`fsUpaSvA*cgv~h+xbO!7AS}FdQGE;=xy7EL7$j{ zrHu#ma!`YW%@}XyCcX7+Y!7)FK5L}aU;jc^^FplsL)dFP@vCeHeV8ADMCmsxT64v#wCc>3_rIC>_s7&~MTBu4`=+eZuewHpzJ*hx=Q>rH@OXIpM{3#D zQC?^&*KJAQUjN7MP zpUc|Xji6xC6XwbPa>at_*dk2-btZ$-+B<|qAjbeozm5AsKq?`Yz}k7XuK!OCW}xFH z_1ZDxN$({|V_>#x@F~%l-}MyPj1PwIQ6js@7u>tnt89tNk>xlKk$Ag~e9y zMDxk5EJgfFk1GY&8#jogBfA%@tEkwsxnnxP&ra?;pa&5$`JLiB?(HY*#F+J z%J!dz)$Vk+C=+hieS`kNX9HL?oztZ1fLlv0#Y=gE;gJZKMNn)S=*ob|yqk0RlC|b; z@$iHj9e-Z7{I_)WQ#-?hU_;1QBxFz*r0}XF?1=FjUtV~KY;vD6!6^TgB~R&}A@ZBplJ=c)sJ8j*B5Fv{?kC^J z?tz$0Xe6V zdNgJj{o-pMV53PhJPHbv!mwpQ!n}O}-GLB%WsW3epWNd#IcPqk?6HORmzG*y!oeQC zHR(ZpfZqREB!(Ll@_te>>z@T>+P*w=(I?WQTYoZ;T)lu2hmqFQO!Lq{9NLbXlFpOtX0Vkd=dy@psNJjhX&ek)fwr+h=rlv0WF$KW9lhEVC#L zT2e#~BL38w{HiV8Ou7>TdP>T(a|gZqoWE$Hbd}j_1q|G6$9HBZpJOm82EEYp+T1cO z)8zd5lPAm+<0!AFr7E8=OAsv#j;z!ffMWVe{s%?ok%|fs{>Y{|74; z{pc)b^C-sZngU)S<=4^?x{4cum<#P$rL;`BkKsAYxne!2$5x`CqY zl50|N)5LKasXf=^A3!kBy5QNY5`~<6%43~^dA`)e49x)gbtO`TgQn6eY>`k0_+VeK92#1ysQ_})MN*6tNh&=1pil&W&C){yg|5AMnp4`|02iqEE(dT>!9 zSKgfD9OUi!kiM6;eThC1G=sdj_F8C#rW9FujX2g>Je5~VNKveqJHE!iCk_UnFz+oq zJcb9$SQO}s8%%aXpUDh!{XK5muIPmV7N zT2)M23C%9zvpbPT6PZKsfAss1a60$J*%jO#tEG-IC2*{uDMkSy+j5nG>N#X6W+on@6Pg4>~^)mVsp zyP@FUKzlTsb*2Xy6W1Y9M7yXc1w#VWpeiixz~6akk_hj?axa4^Zxx& zTd%6Tl5lW(zti!rm!}2%D6eOSj$j6X|y%4wdGSD zG+!SXU{Lxz?-wvr{1AkgK9=86=cV0m+ zO4)*+EXm$f=)nrJ!?KaubJgUmHjv?#OW*4}uY`JVf{A`^45hy?Eh^oeyi5Bk9WmEG z=0ATAF}Yr`#2+qy!fdunFUk^7$+$RMyI*+0)x?JaJ+{sD4w8TOcNLl0j|VL(RCwK$gs{-{M}S-}UP%(v+R4f?qm(CDEwDm8GX%Bb{tk3*VF z9Vk-AY7p@BEG-@Q!M?Rvnp(t5zL4K%u*Vf<_Q-B?SM=H2L9ny$pB^g~{M@nS)Fw)p z-vWhZH%hOEy0MDy+cy#=4o?jwlTz>zijdfF{UYs{8sh#&4zjtGB0>L;zGcPcIChL4 z*zVcUWj;iep>VcwSy@^6*Le0tt)7l5u@)rR-y}H#FsM(15Rf1Xfd>9`{dd!9tt#xj z`Fa#_p|XYJf6N^#h2`z7$&KQS=>s zXp-DUB}pKa4VpyIsn?wF$!G`OxStlvD%PIg^P0jrYJa!7iJJ>cL6n)l(tvyldh9)< z^Wy!1)v*4u)?c>F>X>abmwTKtO2*!O3Q`QB-@g}N2>#$Q^o(pIQ~OR(%#i2$G%e(D zqt^L%bH$fh%cE5(*X1-7Bbn!8y)4@)vk$-hb_nTr!prnbPb+1=T#WHXV{%T;#@$!1 z<@swXH-k(OAX3kQfq%E-*lOgz1b{ng&UwGj16E6t-0Wk<2A|c8;>m>5BU$#NpG#6e zhGjuHmENOm0+$1sAYyO~QndXfwh-S~H~u%rkU{CP)Xh>)BhBf1eT0+-n~4YoIkih| zCc+PZ<8!6|CIJAL&`FM2o;MqJC`2Lv01ib8xI^3FfK?30RuZ*Gqny``9C!S*YoF0J z*1W%7H;H*RNbSt9)jw1Kuknc(R4qSCLXvQ@o0w5J?&yK^YqvkfLW-1k8u#djnMk5@g{5VW6Qe zx$G5sr|W0(HPl~F&>#h>m01Y zOz@y>q^qC$3&5nHm8h^i7-k&X_)8Kh5&+K}JpIFm3L%F4F}XP2%Ibe$_vuXkGQOAb z^*ehUha|=M>B(RJ@y!yJ-ex29SfUbS0l~2tt}vUwa&sno)p5vk(&_q(Ehjk()RFMELdCSGe~ zIDxvQr(GW~5c^$vFT8kcyg&NX^Quncx8T*<82s8JvfEw~4CU}znpH}4R1D_6<(I>m z-E?i3lY>p4g^z(C6m4*nS-dH%)?-Jq)kPULczj%UMW=gpwumxNVTxO22haXxsjdZ; zrinS+NzpU1Wv5;f87|A|>!0+tfUx?5VtO0N^AfxW_t=T>Q!4memMb5qD!mI7WiqbwrtGSugW&2gN`kF<%YJB^nd5eBn zbjk13xOk`>K9SjEsFHpke|iqQ5-E?lnuzAx}125;m0oI<5Bf+_Bhx$D{UIvEuT71Z)SB_{zW*yYb0_uPT8C zAKcUZL37ivH23VkQ7a}Ql`Q|}_5GKuW^aOGaM;6_==5rxjEIzmapz_?84CkZ zsj01{<(kXsCvuz1>-q)B+Y6r6iit?C)(QGwINw~}mU<{@Lb^#`n#&so#mUBk+|j}^ zby`?|!`a@+Ur?vT{;e&^k8>$t1`O7pIC%7rANTm=={pY}ewKPLJggX;z7=dreM1nR;E}>a`PFSPG~4wD!JkFF7kO ze5Wd~!#hl*f5gGjDTR5Ou?cFvV&ycS!)>+olPEzqmcNofrf2yTZH{g)DW-XjX%I4e zBJu?N*}Z%^^R^Mu(p2U&x5v3m}8E&4xM7(_{A19F9raB**C@AMwh4( z*V~#pd+8>{JryY=q8L`sH@A274s{KSwQ9rT7r*ljIlB*}-VZ>?h=ONPB8fgm`55I* z9-0?@SH)l7eCpY83srec1dr$m?V)3;+}4h&#*TBRo6F)ei}o94XxXNprfSd`6}j?K zqsi;@3qtus^v;Kk-!R0qcG{A(fjB*0e|vXtMzUsRB7(^{E>?W3&9Ba88KR7R2X>r4 zR?+mc#I*U42>RSyM*si-OT`o^n@}5LV1`@jTTNfC&}l?OMBz%E!f(3P++yb&5>?Y3 z7M(b^Ofo6U_nWQ#Vu>hX>`OeuGlWdsYAFrbP4=-JUPrstPfA(r?DmivhbN6wlFQcT z48Q*}Bf`h7g+LUWn)2el-DjUuyE^W6M5bG*NwBorB!Q z1ONaq*QOeZVWKlM{DxKwg(rM3FxxHF#fr|=S)2mZ0sXYI$Z&Bu&uzNe(iAH3l4Pwu ze%Jb|$LgzJWzw^gq~d#Sr^8_l)8?-WR%SFj*wSvB(3eyzoz9?X?532ohU&7Ce+Zw7 zj}7yfE;m$uAG07u!vN&T} zLsR2N#qarcC5pTbTTP9{Ll6F@4)59i?H`@|{MMbLdyV6};*Yk z_SJtgY+9B)4$JE zrtQu>#MG8usX5-jC|I>|bG)t0^6(k|a_Yc^$}2}+dp9|r)kMpQsOx6mzdEdI6QX~4 zAm`!Z7d-FaO-tW!@Kp_=PpH(9bJ+ZLoMjB-c6-OR-0r)?EaR%C=syjw6_i{LB*gBX z7VC<=FT5QXJzu{0rPsgvs0H5u0094=kdcuQu44c$HMjSed-`tJZWB?AYE@j6JVhTh z<)h}IOV!ouHf)-`^Y_|s$EhP{;lz%2^0VIGQT_gNG9se7!t|re>AII^V=bxK(i^E< zHT}Te`r!%ocfXo`c~j%%x-D5*j};mK04#O$p@ssul%kJXpgHJbA=|ohZSk3tf7|qO zwz%z$Q-5c6o=L*{5&!^}n0Zi<4zRSzpZWEPZPwzQhNz5#RqWx?mr~IS6aWC08jQvO za4l4}wV-rs0oP#w003|YDJcL1CEIcE`wad=006+8zzqigf|KdvxDEjT0G0}@B?SNg z00535B?SNg00535B?SNg00535B?SNg00535C51(T$HN~p006*3g}XQ`27aF}Oe8{5 z${`+)!bBpjqW}Pag$nm@SPb}~p<=Okz}X)X!gVN1Qb-65IQy3`U(R(P006L1Ati-H zLN1rz=T0+U!5 U#n0l_3IG5A07*qoM6N<$g1zwS7XSbN literal 0 HcmV?d00001 diff --git a/docs/zh/06-advanced/05-data-in/index.md b/docs/zh/06-advanced/05-data-in/index.md index 0dfa04db56..8f23fe2a81 100644 --- a/docs/zh/06-advanced/05-data-in/index.md +++ b/docs/zh/06-advanced/05-data-in/index.md @@ -294,9 +294,32 @@ let v3 = data["voltage"].split(","); 在任务列表页面，还可以对任务进行启动、停止、查看、删除、复制等操作，也可以查看各个任务的运行情况，包括写入的记录条数、流量等。 +### 健康状态 + +从 3.3.5.0 开始，在任务管理列表中，增加了一项 ”健康状态“，用于指示当前任务运行过程中的健康状态。 + +在数据源的”高级选项“列表中，增加了多项健康状态监测的配置项，包括： + +![health options](./health-options.png) + +1. 健康监测时段（Health Check Duration）：可选项，表示对最近多长时间的任务状态进行统计。 +2. Busy 状态阈值（Busy State Threshold）：百分比，表示写入队列中入队元素数量与队列长度之比，默认 100%。 +3. 写入队列长度（Max Write Queue Length）：表示对应的写入队列长度最大值。 +4. 写入错误阈值（Write Error Threshold）：数值类型，表示健康监测时段中允许写入错误的数量。超出阈值，则报错。 + +在任务管理列表展示中，有如下状态： + +- Ready：数据源和目标端健康检查通过，可以进行数据读取和写入。 +- Idle：表示监测时段内无数据处理（没有数据进入处理流程）。 +- Busy：表示写入队列已满（超出一定阈值，表示写入繁忙，在一定程度上意味着当前可能存在性能瓶颈，需要调整参数或配置等来进行改善，但并不说明存在错误）。 +- Bounce：数据源和目标端均正常，但在写入过程中存在错误，一定周期内超出阈值，可能意味着存在大量非正常数据或正在发生数据丢失。 +- SourceError: 数据源错误导致无法进行读取。此时工作负载将尝试重连数据源。 +- SinkError：写入端错误导致无法进行写入。此时工作负载将尝试重连数据库，恢复后进入 Ready 状态。 +- Fatal：严重或无法恢复的错误。 + ```mdx-code-block import DocCardList from '@theme/DocCardList'; import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; -``` \ No newline at end of file +``` From e5222818e7d3f064581726331980100a05c1d220 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 18 Dec 2024 14:08:05 +0800 Subject: [PATCH 093/108] Update 01-index.md --- docs/zh/01-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/01-index.md b/docs/zh/01-index.md index 1fda72024c..02c6105d43 100644 --- a/docs/zh/01-index.md +++ b/docs/zh/01-index.md @@ -16,7 +16,7 @@ TDengine 采用 SQL 作为查询语言，大大降低学习成本、降低迁移 如果你是系统管理员，关心安装、升级、容错灾备、关心数据导入、导出、配置参数，如何监测 TDengine 是否健康运行，如何提升系统运行的性能，请仔细参考[运维指南](./operation)一章。 -如果你对数据库内核设计感兴趣，或是开源爱好者，建议仔细阅读[技术内幕](./tdinterna)一章。该章从分布式架构到存储引擎、查询引擎、数据订阅，再到流计算引擎都做了详细阐述。建议对照文档，查看TDengine在GitHub的源代码，对TDengine的设计和编码做深入了解，更欢迎加入开源社区，贡献代码。 +如果你对数据库内核设计感兴趣，或是开源爱好者，建议仔细阅读[技术内幕](./tdinternal)一章。该章从分布式架构到存储引擎、查询引擎、数据订阅，再到流计算引擎都做了详细阐述。建议对照文档，查看TDengine在GitHub的源代码，对TDengine的设计和编码做深入了解，更欢迎加入开源社区，贡献代码。 最后，作为一个开源软件，欢迎大家的参与。如果发现文档有任何错误、描述不清晰的地方，请在每个页面的最下方，点击“编辑本文档”直接进行修改。 From ed523c9c139431013a8ef3130edc3cb1f27a0fa7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 18 Dec 2024 14:51:22 +0800 Subject: [PATCH 094/108] fix:memory leak --- source/libs/scalar/src/scalar.c | 12 ++++++++++-- source/libs/scalar/test/scalar/scalarTests.cpp | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index df875dffea..3552a21d4f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -244,6 +244,11 @@ void sclFreeParam(SScalarParam *param) { taosHashCleanup(param->pHashFilter); param->pHashFilter = NULL; } + + if (param->pHashFilterVar != NULL) { + taosHashCleanup(param->pHashFilterVar); + param->pHashFilterVar = NULL; + } } int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { @@ -409,6 +414,8 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pHashFilter); param->pHashFilter = NULL; + taosHashCleanup(param->pHashFilterVar); + param->pHashFilterVar = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); return terrno; } @@ -541,14 +548,15 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC } if (0 == *rowNum) { - taosMemoryFreeClear(paramList); + sclFreeParamList(paramList, *paramNum); + paramList = NULL; } *pParams = paramList; return TSDB_CODE_SUCCESS; _return: - taosMemoryFreeClear(paramList); + sclFreeParamList(paramList, *paramNum); SCL_RET(code); } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 3eae06d9bb..865fb30814 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -2106,7 +2106,7 @@ TEST(columnTest, int_column_in_double_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; int32_t leftv[5] = {1, 2, 3, 4, 5}; double rightv1 = 1.1, rightv2 = 2.2, rightv3 = 3.3; - bool eRes[5] = {true, true, true, false, false}; + bool eRes[5] = {false, false, false, false, false}; SSDataBlock *src = NULL; int32_t rowNum = sizeof(leftv) / sizeof(leftv[0]); int32_t code = TSDB_CODE_SUCCESS; From 7e2064d5c127bbe7a8124496bb5de936a91b8107 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 18 Dec 2024 15:39:47 +0800 Subject: [PATCH 095/108] Update index.md --- docs/zh/04-get-started/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/04-get-started/index.md b/docs/zh/04-get-started/index.md index 4d9f7ceae5..5a7192f2c6 100644 --- a/docs/zh/04-get-started/index.md +++ b/docs/zh/04-get-started/index.md @@ -10,7 +10,7 @@ import official_account from './official-account.webp' TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../reference/components/taosadapter) 提供 [RESTful 接口](../reference/connector/rest-api)。 -本章主要介绍如何利用 Docker 或者安装包快速设置 TDengine 环境并体验其高效写入和查询。 +本章主要介绍如何快速设置 TDengine 环境并体验其高效写入和查询。 ```mdx-code-block import DocCardList from '@theme/DocCardList'; @@ -34,4 +34,4 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; 关注 TDengine 视频号
收看技术直播与教学视频 关注 TDengine 公众号
阅读技术文章与行业案例 - \ No newline at end of file + From 290f8700162daa9e7b12cc74b9d4fbeb79f5b989 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Wed, 18 Dec 2024 16:29:13 +0800 Subject: [PATCH 096/108] fix TD-33219 --- include/libs/parser/parser.h | 2 ++ source/client/src/clientStmt2.c | 3 +- source/libs/parser/src/parInsertStmt.c | 42 ++++++++++++++++++++++++++ tests/script/api/stmt2-performance.c | 30 +++++++++++++----- 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 95f522f504..23ad151787 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -137,6 +137,8 @@ int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb); +int32_t qRebuildStmt2DataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, + bool rebuildCreateTb, const char* tName, const char* sTableName); void qDestroyStmtDataBlock(STableDataCxt* pBlock); STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index a479c367e1..f63d12e298 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -488,7 +488,8 @@ static int32_t stmtTryAddTableVgroupInfo(STscStmt2* pStmt, int32_t* vgId) { static int32_t stmtRebuildDataBlock(STscStmt2* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, uint64_t suid, int32_t vgId) { STMT_ERR_RET(stmtTryAddTableVgroupInfo(pStmt, &vgId)); - STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, suid, vgId, pStmt->sql.autoCreateTbl)); + STMT_ERR_RET(qRebuildStmt2DataBlock(newBlock, pDataBlock, uid, suid, vgId, pStmt->sql.autoCreateTbl, pStmt->bInfo.sname.tname, + pStmt->bInfo.stbFName)); STMT_DLOG("tableDataCxt rebuilt, uid:%" PRId64 ", vgId:%d", uid, vgId); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 4ecc18d189..cf476567d7 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -1212,6 +1212,48 @@ int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_ return TSDB_CODE_SUCCESS; } +int32_t qRebuildStmt2DataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, + bool rebuildCreateTb, const char* tName, const char* sTableName) { + int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); + if (code) { + return code; + } + + STableDataCxt* pBlock = (STableDataCxt*)*pDst; + if (pBlock->pMeta) { + pBlock->pMeta->uid = uid; + pBlock->pMeta->vgId = vgId; + pBlock->pMeta->suid = suid; + } + + pBlock->pData->suid = suid; + pBlock->pData->uid = uid; + + if (rebuildCreateTb && NULL == pBlock->pData->pCreateTbReq) { + pBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pBlock->pData->pCreateTbReq) { + return terrno; + } + SArray* pTagArray = taosArrayInit(0, sizeof(STagVal)); + SArray* tagName = taosArrayInit(0, TSDB_COL_NAME_LEN); + + STag* tag = NULL; + code = tTagNew(pTagArray, 1, false, &tag); + + code = insBuildCreateTbReq(pBlock->pData->pCreateTbReq, tName, tag, suid, sTableName, tagName, 0, + TSDB_DEFAULT_TABLE_TTL); + + taosArrayDestroy(pTagArray); + taosArrayDestroy(tagName); + taosMemoryFree(tag); + if (code) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } void qDestroyStmtDataBlock(STableDataCxt* pBlock) { diff --git a/tests/script/api/stmt2-performance.c b/tests/script/api/stmt2-performance.c index aa8e5b9450..a539affaf1 100644 --- a/tests/script/api/stmt2-performance.c +++ b/tests/script/api/stmt2-performance.c @@ -5,9 +5,9 @@ #include #include "taos.h" -int CTB_NUMS = 1000; -int ROW_NUMS = 10; -int CYC_NUMS = 5; +int CTB_NUMS = 2; +int ROW_NUMS = 2; +int CYC_NUMS = 2; void do_query(TAOS* taos, const char* sql) { TAOS_RES* result = taos_query(taos, sql); @@ -57,7 +57,7 @@ void do_stmt(TAOS* taos, const char* sql) { return; } int fieldNum = 0; - TAOS_FIELD_STB* pFields = NULL; + TAOS_FIELD_ALL* pFields = NULL; // code = taos_stmt2_get_stb_fields(stmt, &fieldNum, &pFields); // if (code != 0) { // printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); @@ -74,7 +74,7 @@ void do_stmt(TAOS* taos, const char* sql) { for (int i = 0; i < CTB_NUMS; i++) { tbs[i] = (char*)malloc(sizeof(char) * 20); sprintf(tbs[i], "ctb_%d", i); - createCtb(taos, tbs[i]); + // createCtb(taos, tbs[i]); } for (int r = 0; r < CYC_NUMS; r++) { // col params @@ -138,7 +138,24 @@ void do_stmt(TAOS* taos, const char* sql) { end = clock(); cpu_time_used = ((double)(end - start)) / CLOCKS_PER_SEC; printf("stmt2-exec [%s] insert Time used: %f seconds\n", sql, cpu_time_used); + + for (int i = 0; i < CTB_NUMS; i++) { + free(tags[i]); + free(paramv[i]); + free(ts[i]); + free(b[i]); + } + free(ts); + free(b); + free(ts_len); + free(b_len); + free(paramv); + free(tags); } + for (int i = 0; i < CTB_NUMS; i++) { + free(tbs[i]); + } + free(tbs); // taos_stmt2_free_fields(stmt, pFields); taos_stmt2_close(stmt); @@ -200,10 +217,9 @@ int main() { exit(1); } - sleep(3); do_stmt(taos, "insert into db.stb(tbname,ts,b,t1,t2) values(?,?,?,?,?)"); // do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); - do_taosc(taos); + // do_taosc(taos); taos_close(taos); taos_cleanup(); } From 5fe28f968e40c1f9d352b767f417abd6bc720520 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 18 Dec 2024 17:22:07 +0800 Subject: [PATCH 097/108] Update 03-package.md --- docs/zh/04-get-started/03-package.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/04-get-started/03-package.md b/docs/zh/04-get-started/03-package.md index 9724c0a1c9..62830b10b8 100644 --- a/docs/zh/04-get-started/03-package.md +++ b/docs/zh/04-get-started/03-package.md @@ -8,7 +8,7 @@ import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; import PkgListV3 from "/components/PkgListV3"; -您可以[用 Docker 立即体验](../../get-started/docker/) TDengine。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. +如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 TDinsight 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/components/taosadapter/) 提供 [RESTful 接口](../../reference/connector/rest-api/)。 @@ -319,4 +319,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1 SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s); ``` -在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。 \ No newline at end of file +在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。 From 23c86774da701aa4ec6350e38f79c3dd0211154d Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 18 Dec 2024 17:23:41 +0800 Subject: [PATCH 098/108] Update 03-package.md --- docs/zh/04-get-started/03-package.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/zh/04-get-started/03-package.md b/docs/zh/04-get-started/03-package.md index 62830b10b8..344b9412df 100644 --- a/docs/zh/04-get-started/03-package.md +++ b/docs/zh/04-get-started/03-package.md @@ -8,8 +8,6 @@ import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; import PkgListV3 from "/components/PkgListV3"; -如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. - TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 TDinsight 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/components/taosadapter/) 提供 [RESTful 接口](../../reference/connector/rest-api/)。 为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。 From e0c4cdf664677daa32066686952256599ddb5821 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Thu, 19 Dec 2024 10:15:02 +0800 Subject: [PATCH 099/108] fix TD-33219 2 --- include/libs/parser/parser.h | 2 -- source/client/src/clientMain.c | 5 +++ source/client/src/clientStmt2.c | 11 +++---- source/libs/parser/src/parInsertStmt.c | 42 -------------------------- 4 files changed, 10 insertions(+), 50 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 23ad151787..95f522f504 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -137,8 +137,6 @@ int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb); -int32_t qRebuildStmt2DataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, - bool rebuildCreateTb, const char* tName, const char* sTableName); void qDestroyStmtDataBlock(STableDataCxt* pBlock); STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index dfc18da8cd..ecd835c76f 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -2182,6 +2182,11 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col if (code) { return code; } + } else { + code = stmtSetTbTags2(stmt, NULL); + if (code) { + return code; + } } if (bindv->bind_cols && bindv->bind_cols[i]) { diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index f63d12e298..340bc299fd 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -488,8 +488,7 @@ static int32_t stmtTryAddTableVgroupInfo(STscStmt2* pStmt, int32_t* vgId) { static int32_t stmtRebuildDataBlock(STscStmt2* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, uint64_t suid, int32_t vgId) { STMT_ERR_RET(stmtTryAddTableVgroupInfo(pStmt, &vgId)); - STMT_ERR_RET(qRebuildStmt2DataBlock(newBlock, pDataBlock, uid, suid, vgId, pStmt->sql.autoCreateTbl, pStmt->bInfo.sname.tname, - pStmt->bInfo.stbFName)); + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, suid, vgId, pStmt->sql.autoCreateTbl)); STMT_DLOG("tableDataCxt rebuilt, uid:%" PRId64 ", vgId:%d", uid, vgId); @@ -1013,10 +1012,10 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { } SBoundColInfo* tags_info = (SBoundColInfo*)pStmt->bInfo.boundTags; - if (tags_info->numOfBound <= 0 || tags_info->numOfCols <= 0) { - tscWarn("no tags or cols bound in sql, will not bound tags"); - return TSDB_CODE_SUCCESS; - } + // if (tags_info->numOfBound <= 0 || tags_info->numOfCols <= 0) { + // tscWarn("no tags or cols bound in sql, will not bound tags"); + // return TSDB_CODE_SUCCESS; + // } STableDataCxt** pDataBlock = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index cf476567d7..4ecc18d189 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -1212,48 +1212,6 @@ int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_ return TSDB_CODE_SUCCESS; } -int32_t qRebuildStmt2DataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, - bool rebuildCreateTb, const char* tName, const char* sTableName) { - int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); - if (code) { - return code; - } - - STableDataCxt* pBlock = (STableDataCxt*)*pDst; - if (pBlock->pMeta) { - pBlock->pMeta->uid = uid; - pBlock->pMeta->vgId = vgId; - pBlock->pMeta->suid = suid; - } - - pBlock->pData->suid = suid; - pBlock->pData->uid = uid; - - if (rebuildCreateTb && NULL == pBlock->pData->pCreateTbReq) { - pBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); - if (NULL == pBlock->pData->pCreateTbReq) { - return terrno; - } - SArray* pTagArray = taosArrayInit(0, sizeof(STagVal)); - SArray* tagName = taosArrayInit(0, TSDB_COL_NAME_LEN); - - STag* tag = NULL; - code = tTagNew(pTagArray, 1, false, &tag); - - code = insBuildCreateTbReq(pBlock->pData->pCreateTbReq, tName, tag, suid, sTableName, tagName, 0, - TSDB_DEFAULT_TABLE_TTL); - - taosArrayDestroy(pTagArray); - taosArrayDestroy(tagName); - taosMemoryFree(tag); - if (code) { - return code; - } - } - - return TSDB_CODE_SUCCESS; -} - STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } void qDestroyStmtDataBlock(STableDataCxt* pBlock) { From 6fd86d2b1341f3df4732eb56640e2eee6aa02ab4 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 19 Dec 2024 11:00:04 +0800 Subject: [PATCH 100/108] fix: remove invalid badge --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 030be7bc3b..f827c38975 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,6 @@

-[![Build Status](https://cloud.drone.io/api/badges/taosdata/TDengine/status.svg?ref=refs/heads/master)](https://cloud.drone.io/taosdata/TDengine) -[![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
From 2f2de2978da0ade116ac6d9a5e5041c3c7d833a8 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 19 Dec 2024 14:12:55 +0800 Subject: [PATCH 101/108] Fix core at dmConfigThreadFp. --- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 3 ++- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 9 ++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 2108a097ee..bfe4cd165e 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -69,6 +69,7 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartConfigThread(SDnodeMgmt *pMgmt); int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); +void dmStopConfigThread(SDnodeMgmt *pMgmt); void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index b58c1a216d..d370c5ff67 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -52,6 +52,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { dmStopMonitorThread(pMgmt); dmStopAuditThread(pMgmt); dmStopStatusThread(pMgmt); + dmStopConfigThread(pMgmt); dmStopStatusInfoThread(pMgmt); #if defined(TD_ENTERPRISE) dmStopNotifyThread(pMgmt); @@ -99,7 +100,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static void dmCloseMgmt(SDnodeMgmt *pMgmt) { dmStopWorker(pMgmt); - taosMemoryFree(pMgmt); + taosMemoryFreeClear(pMgmt); } static int32_t dmRequireMgmt(const SMgmtInputOpt *pInput, bool *required) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 8f890f6805..ef4e76031d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -343,7 +343,7 @@ int32_t dmStartConfigThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); - (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pMgmt->configThread, &thAttr, dmConfigThreadFp, pMgmt) != 0) { code = TAOS_SYSTEM_ERROR(errno); dError("failed to create config thread since %s", tstrerror(code)); @@ -378,6 +378,13 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { } } +void dmStopConfigThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->configThread)) { + (void)taosThreadJoin(pMgmt->configThread, NULL); + taosThreadClear(&pMgmt->configThread); + } +} + void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->statusInfoThread)) { (void)taosThreadJoin(pMgmt->statusInfoThread, NULL); From 4edc23057db4b6e23860ad8ae0693a7f9e63bb47 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 19 Dec 2024 14:15:42 +0800 Subject: [PATCH 102/108] Fix err update at free pMgmt. --- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index d370c5ff67..b3b1df314a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -100,7 +100,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static void dmCloseMgmt(SDnodeMgmt *pMgmt) { dmStopWorker(pMgmt); - taosMemoryFreeClear(pMgmt); + taosMemoryFree(pMgmt); } static int32_t dmRequireMgmt(const SMgmtInputOpt *pInput, bool *required) { From df43f4cd59dc32a8fe4f772bc4c0653403259968 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Thu, 19 Dec 2024 14:46:14 +0800 Subject: [PATCH 103/108] fix TSDB_TABLE_FNAME_LEN mismatch tbname with ' --- source/libs/parser/src/parInsertUtil.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 502dbb57dd..ed1f498a32 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -101,10 +101,13 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c return buildInvalidOperationMsg(pMsgBuf, msg1); } } else { // get current DB name first, and then set it into path - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(tbname, pTableName->z, pTableName->n); + int32_t tbLen = strdequote(tbname); + if (tbLen >= TSDB_TABLE_NAME_LEN) { return buildInvalidOperationMsg(pMsgBuf, msg1); } - if (pTableName->n == 0) { + if (tbLen == 0) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "invalid table name"); } From 9eb2ba5269620a9f0805a7f05b9a78921887aff6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 19 Dec 2024 15:27:23 +0800 Subject: [PATCH 104/108] refactor:do some internal refactor. --- include/common/tanalytics.h | 4 +-- source/libs/executor/src/forecastoperator.c | 33 +++++++++++++-------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/include/common/tanalytics.h b/include/common/tanalytics.h index d0af84ecfb..6ebdb38fa6 100644 --- a/include/common/tanalytics.h +++ b/include/common/tanalytics.h @@ -28,8 +28,8 @@ extern "C" { #define ANAL_FORECAST_DEFAULT_ROWS 10 #define ANAL_FORECAST_DEFAULT_CONF 95 #define ANAL_FORECAST_DEFAULT_WNCHECK 1 -#define ANAL_FORECAST_MAX_ROWS 10000 -#define ANAL_ANOMALY_WINDOW_MAX_ROWS 10000 +#define ANAL_FORECAST_MAX_ROWS 40000 +#define ANAL_ANOMALY_WINDOW_MAX_ROWS 40000 typedef struct { EAnalAlgoType type; diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index a56b0dd214..2985e5e000 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -72,17 +72,20 @@ static FORCE_INLINE int32_t forecastEnsureBlockCapacity(SSDataBlock* pBlock, int return TSDB_CODE_SUCCESS; } -static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock) { - if (pSupp->cachedRows > ANAL_FORECAST_MAX_ROWS) { - return TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; - } - - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; +static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock, const char* id) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SAnalyticBuf* pBuf = &pSupp->analBuf; - qDebug("block:%d, %p rows:%" PRId64, pSupp->numOfBlocks, pBlock, pBlock->info.rows); + if (pSupp->cachedRows > ANAL_FORECAST_MAX_ROWS) { + code = TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; + qError("%s rows:%" PRId64 " for forecast cache, error happens, code:%s, upper limit:%d", id, pSupp->cachedRows, + tstrerror(code), ANAL_FORECAST_MAX_ROWS); + return code; + } + pSupp->numOfBlocks++; + qDebug("%s block:%d, %p rows:%" PRId64, id, pSupp->numOfBlocks, pBlock, pBlock->info.rows); for (int32_t j = 0; j < pBlock->info.rows; ++j) { SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pSupp->inputValSlot); @@ -98,10 +101,16 @@ static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock) { pSupp->numOfRows++; code = taosAnalBufWriteColData(pBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, &ts); - if (TSDB_CODE_SUCCESS != code) return code; + if (TSDB_CODE_SUCCESS != code) { + qError("%s failed to write ts in buf, code:%s", id, tstrerror(code)); + return code; + } code = taosAnalBufWriteColData(pBuf, 1, valType, val); - if (TSDB_CODE_SUCCESS != code) return code; + if (TSDB_CODE_SUCCESS != code) { + qError("%s failed to write val in buf, code:%s", id, tstrerror(code)); + return code; + } } return 0; @@ -394,7 +403,7 @@ static int32_t forecastNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pSupp->cachedRows += pBlock->info.rows; qDebug("%s group:%" PRId64 ", blocks:%d, rows:%" PRId64 ", total rows:%" PRId64, pId, pSupp->groupId, numOfBlocks, pBlock->info.rows, pSupp->cachedRows); - code = forecastCacheBlock(pSupp, pBlock); + code = forecastCacheBlock(pSupp, pBlock, pId); QUERY_CHECK_CODE(code, lino, _end); } else { qDebug("%s group:%" PRId64 ", read finish for new group coming, blocks:%d", pId, pSupp->groupId, numOfBlocks); @@ -405,7 +414,7 @@ static int32_t forecastNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pSupp->cachedRows = pBlock->info.rows; qDebug("%s group:%" PRId64 ", new group, rows:%" PRId64 ", total rows:%" PRId64, pId, pSupp->groupId, pBlock->info.rows, pSupp->cachedRows); - code = forecastCacheBlock(pSupp, pBlock); + code = forecastCacheBlock(pSupp, pBlock, pId); QUERY_CHECK_CODE(code, lino, _end); } From fc14fe4455a92ac11438ab4ef2a11f429d463674 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Dec 2024 18:23:34 +0800 Subject: [PATCH 105/108] fix:[TS-5761] error for in operator --- include/libs/function/function.h | 2 +- source/libs/scalar/src/filter.c | 33 +++++------ source/libs/scalar/src/scalar.c | 57 ++++++++----------- source/libs/scalar/src/sclvector.c | 53 ++++++++--------- source/util/src/thashutil.c | 8 +-- .../system-test/2-query/ts-5761-scalemode.py | 11 ++++ tests/system-test/2-query/ts-5761.py | 11 ++++ 7 files changed, 92 insertions(+), 83 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index e8d89e517c..126ed2c9b0 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -288,7 +288,7 @@ struct SScalarParam { bool colAlloced; SColumnInfoData *columnData; SHashObj *pHashFilter; - SHashObj *pHashFilterVar; + SHashObj *pHashFilterOthers; int32_t hashValueType; void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value int32_t numOfRows; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index eea86dbcc6..b329bbbd44 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1298,7 +1298,6 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree, if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { SNodeListNode *listNode = (SNodeListNode *)node->pRight; - SListCell *cell = listNode->pNodeList->pHead; SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; if (out.columnData == NULL) { @@ -1308,8 +1307,9 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree, out.columnData->info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; // reserved space for simple_copy int32_t overflowCount = 0; - for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; + SNode* nodeItem = NULL; + FOREACH(nodeItem, listNode->pNodeList) { + SValueNode *valueNode = (SValueNode *)nodeItem; if (valueNode->node.resType.type != type) { int32_t overflow = 0; code = sclConvertValueToSclParam(valueNode, &out, &overflow); @@ -1319,7 +1319,6 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree, } if (overflow) { - cell = cell->pNext; ++overflowCount; continue; } @@ -1358,8 +1357,6 @@ int32_t fltAddGroupUnitFromNode(void *pContext, SFilterInfo *info, SNode *tree, code = terrno; break; } - - cell = cell->pNext; } if(overflowCount == listNode->pNodeList->length) { ctx->ignore = true; @@ -4863,18 +4860,15 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (LIST_LENGTH(listNode->pNodeList) > 10 || OP_TYPE_NOT_IN == node->opType) { stat->scalarMode = true; } - int32_t type = -1; + int32_t type = refNode->node.resType.type; exprNode = &listNode->node; - SListCell *cell = listNode->pNodeList->pHead; - for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; - cell = cell->pNext; - int32_t tmp = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); - if (tmp != 0 && tmp != refNode->node.resType.type){ + SNode* nodeItem = NULL; + FOREACH(nodeItem, listNode->pNodeList) { + SValueNode *valueNode = (SValueNode *)nodeItem; + int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type); + if (tmp != 0){ stat->scalarMode = true; - if (IS_NUMERIC_TYPE(tmp) && tmp > type){ - type = tmp; - } + type = tmp; } } @@ -5031,11 +5025,11 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } case OP_TYPE_IN: { SNodeListNode *listNode = (SNodeListNode *)oper->valNode; - SListCell *cell = listNode->pNodeList->pHead; SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MAX, .type = oper->colNode->node.resType}; SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MIN, .type = oper->colNode->node.resType}; - for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; + SNode* nodeItem = NULL; + FOREACH(nodeItem, listNode->pNodeList) { + SValueNode *valueNode = (SValueNode *)nodeItem; SFltSclDatum valDatum; FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, valueNode)); if(valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) { @@ -5045,7 +5039,6 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { minDatum.i = TMIN(minDatum.i, valDatum.i); maxDatum.i = TMAX(maxDatum.i, valDatum.i); } - cell = cell->pNext; } SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum}; SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum}; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 3552a21d4f..9bab697772 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -116,7 +116,7 @@ _return: SCL_RET(code); } -// processType = 0 means all type. 1 means number, 2 means var +// processType = 0 means all type. 1 means number, 2 means var, 3 means float, 4 means var&integer int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { @@ -128,7 +128,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_ int32_t code = 0; SNodeListNode *nodeList = (SNodeListNode *)pNode; - SListCell *cell = nodeList->pNodeList->pHead; SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; if (out.columnData == NULL) { SCL_ERR_JRET(terrno); @@ -136,20 +135,13 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_ int32_t len = 0; void *buf = NULL; - for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; - - if (IS_VAR_DATA_TYPE(valueNode->node.resType.type)){ - if (processType == 1) { - cell = cell->pNext; - continue; - } - } else{ - if (processType == 2) - { - cell = cell->pNext; - continue; - } + SNode* nodeItem = NULL; + FOREACH(nodeItem, nodeList->pNodeList) { + SValueNode *valueNode = (SValueNode *)nodeItem; + if ((IS_VAR_DATA_TYPE(valueNode->node.resType.type) && (processType == 1 || processType == 3)) || + (IS_INTEGER_TYPE(valueNode->node.resType.type) && (processType == 2 || processType == 3)) || + (IS_FLOAT_TYPE(valueNode->node.resType.type) && (processType == 2 || processType == 4))) { + continue; } if (valueNode->node.resType.type != type) { @@ -172,7 +164,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_ } if (overflow) { - cell = cell->pNext; continue; } @@ -198,7 +189,6 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_ } colInfoDataCleanup(out.columnData, out.numOfRows); - cell = cell->pNext; } *data = pObj; @@ -245,9 +235,9 @@ void sclFreeParam(SScalarParam *param) { param->pHashFilter = NULL; } - if (param->pHashFilterVar != NULL) { - taosHashCleanup(param->pHashFilterVar); - param->pHashFilterVar = NULL; + if (param->pHashFilterOthers != NULL) { + taosHashCleanup(param->pHashFilterOthers); + param->pHashFilterOthers = NULL; } } @@ -388,15 +378,15 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t type = -1; - SListCell *cell = nodeList->pNodeList->pHead; - for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; - cell = cell->pNext; - int32_t tmp = vectorGetConvertType(ctx->type.selfType, valueNode->node.resType.type); - if (tmp != 0 && IS_NUMERIC_TYPE(tmp) && tmp > type){ - type = tmp; + int32_t type = ctx->type.selfType; + SNode* nodeItem = NULL; + FOREACH(nodeItem, nodeList->pNodeList) { + SValueNode *valueNode = (SValueNode *)nodeItem; + int32_t tmp = vectorGetConvertType(type, valueNode->node.resType.type); + if (tmp != 0){ + type = tmp; } + } if (IS_NUMERIC_TYPE(type)){ ctx->type.peerType = type; @@ -404,7 +394,10 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t type = ctx->type.peerType; if (IS_VAR_DATA_TYPE(ctx->type.selfType) && IS_NUMERIC_TYPE(type)){ SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 1)); - SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterVar, node, ctx->type.selfType, 2)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterOthers, node, ctx->type.selfType, 2)); + } else if (IS_INTEGER_TYPE(ctx->type.selfType) && IS_FLOAT_TYPE(type)){ + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 3)); + SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilterOthers, node, ctx->type.selfType, 4)); } else { SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type, 0)); } @@ -414,8 +407,8 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pHashFilter); param->pHashFilter = NULL; - taosHashCleanup(param->pHashFilterVar); - param->pHashFilterVar = NULL; + taosHashCleanup(param->pHashFilterOthers); + param->pHashFilterOthers = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); return terrno; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e759f35190..5b432535fd 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1009,28 +1009,29 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { - /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ - /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1, - /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, - /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, - /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1, - /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1, - /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1, - /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 0, 0, 0, -1, - /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 0, 0, 0, 20, - /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 0, 0, 0, -1, - /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 16, 0, 0, 0, -1, - /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, -1, 0, 0, 0, -1, - /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, -1, 0, 0, 0, -1, - /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, -1, 0, 0, 0, -1, - /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1,-1, -1, - /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, - /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0}; + /*NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ + /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 5, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1, + /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, + /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, + /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 5, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1, + /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 5, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1, + /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 6, 6, 6, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1, + /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 0, 0, 0, -1, + /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 0, 0, 0, 20, + /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 0, 0, 0, -1, + /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 16, 0, 0, 0, -1, + /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, -1, 0, 0, 0, -1, + /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, -1, 0, 0, 0, -1, + /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, -1, 0, 0, 0, -1, + /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, + /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0 +}; int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { /*NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIM NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ @@ -1071,7 +1072,7 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows) { - if (input->columnData == NULL && (input->pHashFilter != NULL || input->pHashFilterVar != NULL)){ + if (input->columnData == NULL && (input->pHashFilter != NULL || input->pHashFilterOthers != NULL)){ return TSDB_CODE_SUCCESS; } output->numOfRows = input->numOfRows; @@ -2008,7 +2009,7 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara char *pLeftData = colDataGetData(pLeft->columnData, i); bool res = filterDoCompare(fp, optr, pLeftData, pRight->pHashFilter); - if (pLeftVar != NULL && taosHashGetSize(pRight->pHashFilterVar) > 0){ + if (pLeftVar != NULL && taosHashGetSize(pRight->pHashFilterOthers) > 0){ do{ if (optr == OP_TYPE_IN && res){ break; @@ -2017,7 +2018,7 @@ int32_t doVectorCompare(SScalarParam *pLeft, SScalarParam *pLeftVar, SScalarPara break; } pLeftData = colDataGetData(pLeftVar->columnData, i); - res = filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterVar); + res = filterDoCompare(fpVar, optr, pLeftData, pRight->pHashFilterOthers); }while(0); } colDataSetInt8(pOut->columnData, i, (int8_t *)&res); @@ -2048,7 +2049,7 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara SCL_ERR_JRET(vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows)); param1 = (pLeftOut.columnData != NULL) ? &pLeftOut : pLeft; param2 = (pRightOut.columnData != NULL) ? &pRightOut : pRight; - if (pRight->pHashFilterVar != NULL){ + if (pRight->pHashFilterOthers != NULL){ param3 = pLeft; } } diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 67441f8bb0..b466e1b351 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -152,8 +152,8 @@ uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { if (FLT_EQUAL(f, 0.0)) { return 0; } - if (fabs(f) < INT32_MAX) { - int32_t t = (int32_t)(floor(f)); + if (fabs(f) < FLT_MAX / BASE - DLT) { + int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; } else { return 0x7fc00000; @@ -168,8 +168,8 @@ uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { if (FLT_EQUAL(f, 0.0)) { return 0; } - if (fabs(f) < INT32_MAX) { - int32_t t = (int32_t)(floor(f)); + if (fabs(f) < DBL_MAX / BASE - DLT) { + int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; } else { return 0x7fc00000; diff --git a/tests/system-test/2-query/ts-5761-scalemode.py b/tests/system-test/2-query/ts-5761-scalemode.py index 3fbc023eef..0eeabd3af6 100644 --- a/tests/system-test/2-query/ts-5761-scalemode.py +++ b/tests/system-test/2-query/ts-5761-scalemode.py @@ -33,7 +33,18 @@ class TDTestCase: tdSql.execute("INSERT INTO t2 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") tdSql.execute("INSERT INTO t3 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") + tdSql.execute("CREATE TABLE stt( time TIMESTAMP, c1 BIGINT, c2 timestamp, c3 int, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32)) tags(t1 binary(32), t2 nchar(32))") + tdSql.execute("create table tt1 using stt tags('1', '1.7')") + + # create index for all tags + tdSql.execute("INSERT INTO tt1 VALUES (1641024000000, 9223372036854775807, 1641024000000, 1, 1, 1, '1', '1.7')") + def check(self): + tdSql.query(f"SELECT * FROM tt1 WHERE c1 in (1.7, 9223372036854775803, '')") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM tt1 WHERE c1 = 9223372036854775803") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7") tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)") diff --git a/tests/system-test/2-query/ts-5761.py b/tests/system-test/2-query/ts-5761.py index de80835a00..5c8430d856 100644 --- a/tests/system-test/2-query/ts-5761.py +++ b/tests/system-test/2-query/ts-5761.py @@ -32,7 +32,18 @@ class TDTestCase: tdSql.execute("INSERT INTO t2 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") tdSql.execute("INSERT INTO t3 VALUES (1641024000002, 1, 1, 1, 1, 1, '1', 'er')") + tdSql.execute("CREATE TABLE stt( time TIMESTAMP, c1 BIGINT, c2 timestamp, c3 int, c4 int UNSIGNED, c5 bool, c6 binary(32), c7 nchar(32)) tags(t1 binary(32), t2 nchar(32))") + tdSql.execute("create table tt1 using stt tags('1', '1.7')") + + # create index for all tags + tdSql.execute("INSERT INTO tt1 VALUES (1641024000000, 9223372036854775807, 1641024000000, 1, 1, 1, '1', '1.7')") + def check(self): + tdSql.query(f"SELECT * FROM tt1 WHERE c1 in (1.7, 9223372036854775803, '')") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM tt1 WHERE c1 = 9223372036854775803") + tdSql.checkRows(0) + tdSql.query(f"SELECT * FROM t1 WHERE c1 = 1.7") tdSql.checkRows(0) tdSql.query(f"SELECT * FROM t1 WHERE c1 in (1.7, 2)") From ac4e4093bb09f3365f51d4ee5c1af8c22e935330 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 19 Dec 2024 18:54:16 +0800 Subject: [PATCH 106/108] enh: ignore some tanalytics files in Jenkinsfile2 --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 249a8d1c9d..6fa3483099 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -78,7 +78,7 @@ def check_docs(){ file_only_tdgpt_change_except = sh ( script: ''' cd ${WKC} - git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -v "forecastoperator.c\\|anomalywindowoperator.c" |grep -v "tsim/analytics" |grep -v "tdgpt_cases.task" || : + git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -v "forecastoperator.c\\|anomalywindowoperator.c\\|tanalytics.h\\|tanalytics.c" |grep -v "tsim/analytics" |grep -v "tdgpt_cases.task" || : ''', returnStdout: true ).trim() From 63dc6890f6214308b954ccbc9e82bdb60bccee71 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Thu, 19 Dec 2024 21:12:40 +0800 Subject: [PATCH 107/108] docs(agent): add in_memory_cache_capacity options in agent For [TS-5737](https://jira.taosdata.com:18080/browse/TS-5737) --- docs/en/14-reference/01-components/05-taosx-agent.md | 5 +++++ docs/zh/14-reference/01-components/05-taosx-agent.md | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/en/14-reference/01-components/05-taosx-agent.md b/docs/en/14-reference/01-components/05-taosx-agent.md index 3e8b4f4d63..d7d86a64cb 100644 --- a/docs/en/14-reference/01-components/05-taosx-agent.md +++ b/docs/en/14-reference/01-components/05-taosx-agent.md @@ -14,6 +14,7 @@ The default configuration file for `Agent` is located at `/etc/taos/agent.toml`, - `token`: Required, the Token generated when creating `Agent` in `Explorer`. - `instanceId`: The instance ID of the current taosx-agent service. If multiple taosx-agent instances are started on the same machine, it is necessary to ensure that the instance IDs of each instance are unique. - `compression`: Optional, can be configured as `true` or `false`, default is `false`. If set to `true`, it enables data compression in communication between `Agent` and `taosX`. +- `in_memory_cache_capacity`: Optional, signifies the maximum number of message batches that can be cached in memory and can be configured as a positive integer greater than zero. The default value is set at 64. - `log_level`: Optional, log level, default is `info`. Like `taosX`, it supports five levels: `error`, `warn`, `info`, `debug`, `trace`. Deprecated, please use `log.level` instead. - `log_keep_days`: Optional, the number of days to keep logs, default is `30` days. Deprecated, please use `log.keepDays` instead. - `log.path`: The directory where log files are stored. @@ -45,6 +46,10 @@ As shown below: # #compression = true +# In-memory cache capacity +# +#in_memory_cache_capacity = 64 + # log configuration [log] # All log files are stored in this directory diff --git a/docs/zh/14-reference/01-components/05-taosx-agent.md b/docs/zh/14-reference/01-components/05-taosx-agent.md index bf2e6f7e78..1f1276e834 100644 --- a/docs/zh/14-reference/01-components/05-taosx-agent.md +++ b/docs/zh/14-reference/01-components/05-taosx-agent.md @@ -12,7 +12,8 @@ sidebar_label: taosX-Agent - `endpoint`: 必填，`taosX` 的 GRPC 服务地址。 - `token`: 必填，在 `Explorer` 上创建 `Agent` 时，产生的 Token。 - `instanceId`：当前 taosx-agent 服务的实例 ID，如果同一台机器上启动了多个 taosx-agent 实例，必须保证各个实例的实例 ID 互不相同。 -- `compression`: 非必填，可配置为 `ture` 或 `false`, 默认为 `false`。配置为`true`, 则开启 `Agent` 和 `taosX` 通信数据压缩。 +- `compression`: 非必填，可配置为 `true` 或 `false`, 默认为 `false`。配置为`true`, 则开启 `Agent` 和 `taosX` 通信数据压缩。 +- `in_memory_cache_capacity`: 非必填，表示可在内存中缓存的最大消息批次数，可配置为大于 0 的整数。默认为 `64`。 - `log_level`: 非必填，日志级别，默认为 `info`, 同 `taosX` 一样，支持 `error`，`warn`，`info`，`debug`，`trace` 五级。已弃用，请使用 `log.level` 代替。 - `log_keep_days`：非必填，日志保存天数，默认为 `30` 天。已弃用，请使用 `log.keepDays` 代替。 - `log.path`：日志文件存放的目录。 @@ -44,6 +45,10 @@ sidebar_label: taosX-Agent # #compression = true +# In-memory cache capacity +# +#in_memory_cache_capacity = 64 + # log configuration [log] # All log files are stored in this directory From 8921bf854904a8bf4abd17ecd3441d9d2b071155 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 Dec 2024 00:29:24 +0800 Subject: [PATCH 108/108] fix:[TS-5761] error for in operator --- tests/parallel_test/cases.task | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3733075649..2f52c10abe 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -218,8 +218,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5761.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5761-scalemode.py +,,n,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5761.py +,,n,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5761-scalemode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-5712.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 2