diff --git a/.clang-format b/.clang-format index 3ddd8b43f6..e58d518b3b 100644 --- a/.clang-format +++ b/.clang-format @@ -5,6 +5,7 @@ AccessModifierOffset: -1 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: true +AlignConsecutiveMacros: true AlignEscapedNewlinesLeft: true AlignOperands: true AlignTrailingComments: true diff --git a/example/src/tmq.c b/example/src/tmq.c index 8757104ad9..efb4d1830e 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -28,7 +28,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -42,25 +42,33 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + pRes = + taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); - pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)"); + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000)"); if (taos_errno(pRes) != 0) { printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); - pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)"); + pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)"); if (taos_errno(pRes) != 0) { printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); return -1; } + + pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); return 0; } @@ -82,12 +90,40 @@ int32_t create_topic() { /*const char* sql = "select * from tu1";*/ /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/ - pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1"); + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1 from ct1"); if (taos_errno(pRes) != 0) { - printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); + printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); + +#if 0 + pRes = taos_query(pConn, "insert into tu1 values(now, 1, 1.0, 'bi1')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into tu1 values(now+1d, 1, 1.0, 'bi1')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into tu2 values(now, 2, 2.0, 'bi2')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into tu2 values(now+1d, 2, 2.0, 'bi2')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); +#endif + taos_close(pConn); return 0; } @@ -115,7 +151,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "test_stb_topic_1"); + tmq_list_append(topic_list, "topic_ctb_column"); return topic_list; } @@ -215,8 +251,8 @@ int main(int argc, char* argv[]) { if (argc > 1) { printf("env init\n"); code = init_env(); + create_topic(); } - create_topic(); tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); /*perf_loop(tmq, topic_list);*/ diff --git a/example/src/tstream.c b/example/src/tstream.c index 40d8ff9b0b..8ffa932bd2 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -20,7 +20,7 @@ #include "taos.h" int32_t init_env() { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { return -1; } @@ -65,7 +65,7 @@ int32_t init_env() { int32_t create_stream() { printf("create stream\n"); TAOS_RES* pRes; - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { return -1; } diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 51eabb7d61..8332e9d09a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -37,6 +37,14 @@ enum { TMQ_MSG_TYPE__EP_RSP, }; +enum { + STREAM_TRIGGER__AT_ONCE = 1, + STREAM_TRIGGER__WINDOW_CLOSE, + STREAM_TRIGGER__BY_COUNT, + STREAM_TRIGGER__BY_BATCH_COUNT, + STREAM_TRIGGER__BY_EVENT_TIME, +}; + typedef struct { uint32_t numOfTables; SArray* pGroupList; @@ -54,13 +62,16 @@ typedef struct SColumnDataAgg { } SColumnDataAgg; typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t rowSize; - int16_t numOfCols; - int16_t hasVarCol; - union {int64_t uid; int64_t blockId;}; - int64_t groupId; // no need to serialize + STimeWindow window; + int32_t rows; + int32_t rowSize; + int16_t numOfCols; + int16_t hasVarCol; + union { + int64_t uid; + int64_t blockId; + }; + int64_t groupId; // no need to serialize } SDataBlockInfo; typedef struct SSDataBlock { @@ -93,7 +104,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); int32_t tEncodeDataBlocks(void** buf, const SArray* blocks); void* tDecodeDataBlocks(const void* buf, SArray** blocks); -void colDataDestroy(SColumnInfoData* pColData) ; +void colDataDestroy(SColumnInfoData* pColData); static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { // WARNING: do not use info.numOfCols, @@ -198,8 +209,8 @@ typedef struct SGroupbyExpr { } SGroupbyExpr; enum { - FUNC_PARAM_TYPE_VALUE = 0, - FUNC_PARAM_TYPE_COLUMN, + FUNC_PARAM_TYPE_VALUE = 0x1, + FUNC_PARAM_TYPE_COLUMN= 0x2, }; typedef struct SFunctParam { @@ -239,7 +250,7 @@ typedef struct SSessionWindow { SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/trow.h b/include/common/trow.h index abbe55c3a3..01f4076382 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -47,21 +47,21 @@ extern "C" { #define TD_VTYPE_NONE 0x0U // none or unknown/undefined #define TD_VTYPE_NULL 0x01U // null val #define TD_VTYPE_NORM 0x02U // normal val: not none, not null -#define TD_VTYPE_MAX 0x03U // +#define TD_VTYPE_MAX 0x03U // #define TD_VTYPE_NONE_BYTE 0x0U #define TD_VTYPE_NULL_BYTE 0x55U #define TD_VTYPE_NORM_BYTE 0xAAU -#define TD_ROWS_ALL_NORM 0x01U +#define TD_ROWS_ALL_NORM 0x01U #define TD_ROWS_NULL_NORM 0x0U -#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM +#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v)) -#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) -#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) +#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) +#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) -#define KvConvertRatio (0.9f) +#define KvConvertRatio (0.9f) #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio)) #ifdef TD_SUPPORT_BITMAP @@ -98,7 +98,7 @@ typedef void *SRow; typedef struct { TDRowValT valType; - void * val; + void *val; } SCellVal; typedef struct { @@ -158,43 +158,43 @@ typedef struct { int16_t nBitmaps; int16_t nBoundBitmaps; int32_t offset; - void * pBitmap; - void * pOffset; + void *pBitmap; + void *pOffset; int32_t extendedRowSize; } SRowBuilder; -#define TD_ROW_HEAD_LEN (sizeof(STSRow)) +#define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) -#define TD_ROW_INFO(r) ((r)->info) -#define TD_ROW_TYPE(r) ((r)->type) -#define TD_ROW_DELETE(r) ((r)->del) -#define TD_ROW_ENDIAN(r) ((r)->endian) -#define TD_ROW_SVER(r) ((r)->sver) -#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow -#define TD_ROW_DATA(r) ((r)->data) -#define TD_ROW_LEN(r) ((r)->len) -#define TD_ROW_KEY(r) ((r)->ts) +#define TD_ROW_INFO(r) ((r)->info) +#define TD_ROW_TYPE(r) ((r)->type) +#define TD_ROW_DELETE(r) ((r)->del) +#define TD_ROW_ENDIAN(r) ((r)->endian) +#define TD_ROW_SVER(r) ((r)->sver) +#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow +#define TD_ROW_DATA(r) ((r)->data) +#define TD_ROW_LEN(r) ((r)->len) +#define TD_ROW_KEY(r) ((r)->ts) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) -#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i)) -#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) -#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) -#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) -#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) +#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i)) +#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) +#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) +#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) +#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) #define TD_ROW_SET_NCOLS(r, n) (*(col_id_t *)TD_ROW_NCOLS(r) = (n)) #define TD_ROW_IS_DELETED(r) (TD_ROW_DELETE(r) == 1) -#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) -#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) -#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) -#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) +#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) +#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) +#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) +#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) -#define TD_BOOL_STR(b) ((b) ? "true" : "false") +#define TD_BOOL_STR(b) ((b) ? "true" : "false") #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) #define TD_ROW_COL_IDX(r) POINTER_SHIFT(TD_ROW_DATA(r), sizeof(col_id_t)) @@ -275,7 +275,7 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); @@ -313,7 +313,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pValType = (((*pDestByte) & 0xC0) >> 6); @@ -620,7 +620,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa if (tdValIsNorm(valType, val, colType)) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN @@ -638,7 +638,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa // NULL/None value else { SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN const void *nullVal = getNullValue(colType); @@ -775,8 +775,8 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v typedef struct { STSchema *pSchema; - STSRow * pRow; - void * pBitmap; + STSRow *pRow; + void *pBitmap; uint32_t offset; col_id_t maxColId; col_id_t colIdx; // [PRIMARYKEY_TIMESTAMP_COL_ID, nSchemaCols], PRIMARYKEY_TIMESTAMP_COL_ID equals 1 @@ -881,7 +881,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy // internal static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { - STSRow * pRow = pIter->pRow; + STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; col_id_t kvNCols = tdRowGetNCols(pRow); @@ -1076,7 +1076,7 @@ typedef struct { typedef struct { STSchema *pSchema; - STSRow * pRow; + STSRow *pRow; } STSRowReader; typedef struct { diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 8b1ac3ed8d..e635227eaa 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -61,16 +61,16 @@ extern "C" { } \ } -#define WAL_HEAD_VER 0 +#define WAL_HEAD_VER 0 #define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) -#define WAL_LOG_SUFFIX "log" +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) +#define WAL_LOG_SUFFIX "log" #define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_MAGIC 0xFAFBFCFDULL +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_MAGIC 0xFAFBFCFDULL #define WAL_CUR_FAILED 1 @@ -150,14 +150,15 @@ typedef struct SWal { } SWal; // WAL HANDLE typedef struct SWalReadHandle { - SWal *pWal; - TdFilePtr pReadLogTFile; - TdFilePtr pReadIdxTFile; - int64_t curFileFirstVer; - int64_t curVersion; - int64_t capacity; - int64_t status; // if cursor valid - SWalHead *pHead; + SWal *pWal; + TdFilePtr pReadLogTFile; + TdFilePtr pReadIdxTFile; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + int64_t status; // if cursor valid + TdThreadMutex mutex; + SWalHead *pHead; } SWalReadHandle; #pragma pack(pop) @@ -191,6 +192,7 @@ int32_t walEndSnapshot(SWal *); SWalReadHandle *walOpenReadHandle(SWal *); void walCloseReadHandle(SWalReadHandle *); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); +int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead); // deprecated #if 0 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f0d5941141..8cd53f18b0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -181,6 +181,11 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { } int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { + // drop table if exists not_exists_table + if (NULL == pQuery->pCmdMsg) { + return TSDB_CODE_SUCCESS; + } + SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg; pRequest->type = pMsgInfo->msgType; pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL}; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 1fa267ae7e..5d1e95be7c 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -255,6 +255,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { break; } + msg = NULL; taosReadAllQitems(tmq->mqueue, tmq->qall); while (1) { taosGetQitem(tmq->qall, (void**)&msg); @@ -787,7 +788,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) { static bool noPrintSchema; char pBuf[128]; SMqPollRsp* pRsp = &tmq_message->msg; - int32_t colNum = pRsp->schema->nCols; + int32_t colNum = 2; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { @@ -838,6 +839,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { + /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ tsem_post(&tmq->rspSem); tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); return 0; @@ -886,6 +888,9 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { goto WRITE_QUEUE_FAIL; } + tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset, + pRsp->msg.rspOffset); + pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); @@ -902,6 +907,7 @@ WRITE_QUEUE_FAIL: bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ + /*printf("tmq update ep epoch %d to epoch %d\n", tmq->epoch, epoch);*/ bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; @@ -932,6 +938,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); + /*printf("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);*/ taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -945,9 +952,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; + /*printf("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);*/ if (pOffset != NULL) { offset = *pOffset; + /*printf("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);*/ } + /*printf("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);*/ SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, @@ -1195,6 +1205,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { + /*printf("skip vg %d\n", pVg->vgId);*/ continue; } SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); @@ -1238,6 +1249,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ atomic_add_fetch_32(&tmq->waitingRequest, 1); + /*tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);*/ + /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; tmq->pollCnt++; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 6976d83abd..7891c690e3 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -735,6 +735,9 @@ typedef struct { int8_t createdBy; // STREAM_CREATED_BY__USER or SMA int32_t fixedSinkVgId; // 0 for shuffle int64_t smaId; // 0 for unused + int8_t trigger; + int32_t triggerParam; + int64_t waterMark; char* sql; char* logicalPlan; char* physicalPlan; diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index ce08dfaaa3..b46920bc5e 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -89,7 +89,6 @@ static const SInfosTableSchema userStbsSchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT}, }; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 02e09252f9..196767462e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1652,10 +1652,6 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32 *(int32_t *)pWrite = pStb->numOfTags; cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = 0; // number of tables - cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int64_t *)pWrite = pStb->updateTime; // number of tables cols++; diff --git a/source/dnode/vnode/src/inc/tsdbCommit.h b/source/dnode/vnode/src/inc/tsdbCommit.h index 699aeaa133..2c6dd75e03 100644 --- a/source/dnode/vnode/src/inc/tsdbCommit.h +++ b/source/dnode/vnode/src/inc/tsdbCommit.h @@ -74,4 +74,4 @@ int tsdbApplyRtn(STsdbRepo *pRepo); } #endif -#endif /* _TD_TSDB_COMMIT_H_ */ \ No newline at end of file +#endif /* _TD_TSDB_COMMIT_H_ */ diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 02aba95517..3e42cb627a 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -79,4 +79,4 @@ static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, } #endif -#endif /*_TD_TSDB_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TSDB_DEF_H_*/ diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e5d1f952a8..aa4a9fc1de 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -194,7 +194,7 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqCommit(STQ*); -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); diff --git a/source/dnode/vnode/src/meta/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c index cad1eba134..1f57d1396a 100644 --- a/source/dnode/vnode/src/meta/metaTbUid.c +++ b/source/dnode/vnode/src/meta/metaTbUid.c @@ -27,5 +27,5 @@ void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */ tb_uid_t metaGenerateUid(SMeta *pMeta) { // Generate a new table UID - return ++(pMeta->uidGnrt.nextUid); + return tGenIdPI32(); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 06a2350aab..82300f82cb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -250,7 +250,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu return 0; } -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; int64_t fetchOffset; @@ -264,6 +264,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { fetchOffset = pReq->currentOffset + 1; } + /*printf("tmq poll vg %d req %ld %ld\n", pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);*/ + SMqPollRsp rsp = { /*.consumerId = consumerId,*/ .numOfTopics = 0, @@ -288,62 +290,77 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; - SWalHead* pHead; while (1) { /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ - if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + SWalReadHead* pHead; + if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) { // TODO: no more log, set timer to wait blocking time // if data inserted during waiting, launch query and // response to user break; } - int8_t pos = fetchOffset % TQ_BUFFER_SIZE; - pHead = pTopic->pReadhandle->pHead; - if (pHead->head.msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; - qTaskInfo_t task = pTopic->buffer.output[pos].task; + /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType, + * pReq->epoch);*/ + /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ + /*pHead = pTopic->pReadhandle->pHead;*/ + if (pHead->msgType == TDMT_VND_SUBMIT) { + SSubmitReq* pCont = (SSubmitReq*)&pHead->body; + /*printf("from topic %s from consumer\n", pTopic->topicName, consumerId);*/ + qTaskInfo_t task = pTopic->buffer.output[workerId].task; + ASSERT(task); qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); while (1) { - SSDataBlock* pDataBlock; + SSDataBlock* pDataBlock = NULL; uint64_t ts; if (qExecTask(task, &pDataBlock, &ts) < 0) { ASSERT(false); } if (pDataBlock == NULL) { - fetchOffset++; - pos = fetchOffset % TQ_BUFFER_SIZE; - rsp.skipLogNum++; + /*pos = fetchOffset % TQ_BUFFER_SIZE;*/ break; } taosArrayPush(pRes, pDataBlock); - rsp.schema = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; - rsp.rspOffset = fetchOffset; - - rsp.numOfTopics = 1; - rsp.pBlockData = pRes; - - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - pMsg->code = -1; - return -1; - } - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRsp(&abuf, &rsp); - /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - tmsgSendRsp(pMsg); - return 0; } + + if (taosArrayGetSize(pRes) == 0) { + fetchOffset++; + rsp.skipLogNum++; + taosArrayDestroy(pRes); + continue; + } + rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper; + rsp.rspOffset = fetchOffset; + + rsp.numOfTopics = 1; + rsp.pBlockData = pRes; + + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + taosMemoryFree(pHead); + return -1; + } + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqPollRsp(&abuf, &rsp); + /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + /*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset, + * pHead->msgType,*/ + /*pReq->epoch);*/ + tmsgSendRsp(pMsg); + taosMemoryFree(pHead); + return 0; } else { + taosMemoryFree(pHead); fetchOffset++; rsp.skipLogNum++; } @@ -368,6 +385,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { pMsg->contLen = tlen; pMsg->code = 0; tmsgSendRsp(pMsg); + /*printf("vg %d offset %ld from epoch %d not rsp\n", pTq->pVnode->vgId, fetchOffset, pReq->epoch);*/ /*}*/ return 0; @@ -432,7 +450,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { }; pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); + ASSERT(pTopic->buffer.output[i].task); } + printf("set topic %s to consumer %ld\n", pTopic->topicName, req.consumerId); taosArrayPush(pConsumer->topics, pTopic); tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); tqHandleCommit(pTq->tqMeta, req.consumerId); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 344b4fa655..61538055f2 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -168,6 +168,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { break; } if (colDataAppend(pColData, curRow, sVal.val, false) < 0) { + /*if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {*/ taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); return NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3e0b03f331..d5e9b55a71 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1394,7 +1394,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, + REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bac5255d17..cae6d5ec1a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4068,4 +4068,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d377ec2039..30c2fe84bf 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -66,7 +66,7 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: - return tqProcessPollReq(pVnode->pTq, pMsg); + return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId); case TDMT_VND_TASK_PIPE_EXEC: case TDMT_VND_TASK_MERGE_EXEC: return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index d6f9f0da0b..38eed46c9d 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -165,6 +165,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // } break; case TDMT_VND_SUBMIT: + /*printf("vnode %d write data %ld\n", pVnode->vgId, ver);*/ if (pVnode->config.streamMode == 0) { if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) { // TODO: handle error diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index e87fdba71d..a3019d8d56 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2304,8 +2304,6 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_RET(ctgStartUpdateThread()); - tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); @@ -2316,6 +2314,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; + CTG_ERR_RET(ctgStartUpdateThread()); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 5d5e0cae88..3faf208c0e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -73,27 +73,12 @@ typedef struct SResultRowPosition { } SResultRowPosition; typedef struct SResultRowInfo { - SList *pRows; SResultRowPosition *pPosition; - SResultRow **pResult; // result list int32_t size; // number of result set int32_t capacity; // max capacity int32_t curPos; // current active result row index of pResult list } SResultRowInfo; -typedef struct SResultRowPool { - int32_t elemSize; - int32_t blockSize; - int32_t numOfElemPerBlock; - - struct { - int32_t blockIndex; - int32_t pos; - } position; - - SArray* pData; // SArray -} SResultRowPool; - struct STaskAttr; struct STaskRuntimeEnv; struct SUdfInfo; @@ -109,25 +94,33 @@ void resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); -int32_t initResultRow(SResultRow *pResultRow); -void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); -bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow); +void initResultRow(SResultRow *pResultRow); +void closeResultRow(SResultRow* pResultRow); +bool isResultRowClosed(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); -void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); int32_t getRowNumForMultioutput(struct STaskAttr* pQueryAttr, bool topBottomQuery, bool stable); -static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { - assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); - return pResultRowInfo->pResult[slot]; +static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo *pResultRowInfo, int32_t slot) { + ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); + SResultRowPosition* pos = &pResultRowInfo->pPosition[slot]; + + SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); + return pRow; +} + +static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { + SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); + return pRow; } static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); - + ASSERT(0); // int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); // return ((char *)page->data) + rowOffset + offset * numOfRows; } @@ -139,22 +132,11 @@ static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffs return (char*) page + rowOffset + offset * numOfRows; } -//bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); -//bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); - -__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr); - -SResultRow* getNewResultRow(SResultRowPool* p); - typedef struct { SArray* pResult; // SArray int32_t colId; } SStddevInterResult; -void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen); -SArray* interResFromBinary(const char* data, int32_t len); -void freeInterResult(void* param); - void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ce0069fdfa..a92813812a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -69,7 +69,7 @@ enum { typedef struct SResultRowCell { uint64_t groupId; - SResultRow* pRow; + SResultRowPosition pos; } SResultRowCell; /** @@ -277,8 +277,6 @@ typedef struct STaskRuntimeEnv { char* keyBuf; // window key buffer // The window result objects pool, all the resultRow Objects are allocated and managed by this object. char** prevRow; - SResultRowPool* pool; - SArray* prevResult; // intermediate result, SArray STSBuf* pTsBuf; // timestamp filter list STSCursor cur; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 93fbfe43ce..40e86c7840 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -53,15 +53,13 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->size = 0; - pResultRowInfo->curPos = -1; + pResultRowInfo->curPos = -1; pResultRowInfo->capacity = size; - - pResultRowInfo->pResult = taosMemoryCalloc(pResultRowInfo->capacity, POINTER_BYTES); pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); - if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) { + + if (pResultRowInfo->pPosition == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - return TSDB_CODE_SUCCESS; } @@ -71,17 +69,17 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { } if (pResultRowInfo->capacity == 0) { - assert(pResultRowInfo->pResult == NULL); +// assert(pResultRowInfo->pResult == NULL); return; } for(int32_t i = 0; i < pResultRowInfo->size; ++i) { - if (pResultRowInfo->pResult[i]) { - taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); - } +// if (pResultRowInfo->pResult[i]) { +// taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); +// } } - taosMemoryFreeClear(pResultRowInfo->pResult); + taosMemoryFreeClear(pResultRowInfo->pPosition); } void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { @@ -90,8 +88,8 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow } for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow *pWindowRes = pResultRowInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes); +// SResultRow *pWindowRes = pResultRowInfo->pResult[i]; +// clearResultRow(pRuntimeEnv, pWindowRes); int32_t groupIndex = 0; int64_t uid = 0; @@ -101,14 +99,13 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow } pResultRowInfo->size = 0; - pResultRowInfo->curPos = -1; } int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { int32_t i = 0; - while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { - ++i; - } +// while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { +// ++i; +// } return i; } @@ -117,21 +114,22 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) { assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - SResultRow* pRow = pResultRowInfo->pResult[i]; - if (pRow->closed) { - continue; - } +// ASSERT(0); +// SResultRow* pRow = pResultRowInfo->pResult[i]; +// if (pRow->closed) { +// continue; +// } - pRow->closed = true; +// pRow->closed = true; } } -bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { - return (getResultRow(pResultRowInfo, slot)->closed == true); +bool isResultRowClosed(SResultRow* pRow) { + return (pRow->closed == true); } -void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { - getResultRow(pResultRowInfo, slot)->closed = true; +void closeResultRow(SResultRow* pResultRow) { + pResultRow->closed = true; } void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { @@ -181,29 +179,6 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return rowSize; } -SResultRow* getNewResultRow(SResultRowPool* p) { - if (p == NULL) { - return NULL; - } - - void* ptr = NULL; - if (p->position.pos == 0) { - ptr = taosMemoryCalloc(1, p->blockSize); - taosArrayPush(p->pData, &ptr); - - } else { - size_t last = taosArrayGetSize(p->pData); - - void** pBlock = taosArrayGet(p->pData, last - 1); - ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos; - } - - p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock; - initResultRow(ptr); - - return ptr; -} - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); @@ -261,8 +236,9 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); } -static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { +static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRowPosition *pos, int32_t* rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + ASSERT(0); for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; @@ -305,25 +281,26 @@ static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void * return -1; } + ASSERT(0); STableQueryInfo** pList = supporter->pTableQueryInfo; - SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos]; +// SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos]; // SResultRow * pWindowRes1 = getResultRow(&(pList[left]->resInfo), leftPos); - TSKEY leftTimestamp = pWindowRes1->win.skey; +// TSKEY leftTimestamp = pWindowRes1->win.skey; // SResultRowInfo *pWindowResInfo2 = &(pList[right]->resInfo); // SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); - SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos]; - TSKEY rightTimestamp = pWindowRes2->win.skey; +// SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos]; +// TSKEY rightTimestamp = pWindowRes2->win.skey; - if (leftTimestamp == rightTimestamp) { +// if (leftTimestamp == rightTimestamp) { return 0; - } +// } - if (supporter->order == TSDB_ORDER_ASC) { - return (leftTimestamp > rightTimestamp)? 1:-1; - } else { - return (leftTimestamp < rightTimestamp)? 1:-1; - } +// if (supporter->order == TSDB_ORDER_ASC) { +// return (leftTimestamp > rightTimestamp)? 1:-1; +// } else { +// return (leftTimestamp < rightTimestamp)? 1:-1; +// } } int32_t tsAscOrder(const void* p1, const void* p2) { @@ -331,11 +308,12 @@ int32_t tsAscOrder(const void* p1, const void* p2) { SResultRowCell* pc2 = (SResultRowCell*) p2; if (pc1->groupId == pc2->groupId) { - if (pc1->pRow->win.skey == pc2->pRow->win.skey) { - return 0; - } else { - return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1; - } + ASSERT(0); +// if (pc1->pRow->win.skey == pc2->pRow->win.skey) { +// return 0; +// } else { +// return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1; +// } } else { return (pc1->groupId < pc2->groupId)? -1:1; } @@ -346,11 +324,12 @@ int32_t tsDescOrder(const void* p1, const void* p2) { SResultRowCell* pc2 = (SResultRowCell*) p2; if (pc1->groupId == pc2->groupId) { - if (pc1->pRow->win.skey == pc2->pRow->win.skey) { - return 0; - } else { - return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1; - } + ASSERT(0); +// if (pc1->pRow->win.skey == pc2->pRow->win.skey) { +// return 0; +// } else { +// return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1; +// } } else { return (pc1->groupId < pc2->groupId)? -1:1; } @@ -384,13 +363,13 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe break; } - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pResultRowCell->pRow, rowCellInfoOffset); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset); if (num <= 0) { continue; } - taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pRow); - pResultRowCell->pRow->numOfRows = (uint32_t) num; + taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pos); +// pResultRowCell->pRow->numOfRows = (uint32_t) num; } return TSDB_CODE_SUCCESS; @@ -449,9 +428,10 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int32_t tableIndex = tMergeTreeGetChosenIndex(pTree); SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo; - SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); + ASSERT(0); + SResultRow *pWindowRes = NULL;//getResultRow(pBuf, pWindowResInfo, cs.rowIndex[tableIndex]); - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset); + int64_t num = 0;//getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset); if (num <= 0) { cs.rowIndex[tableIndex] += 1; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6265fe6123..169e47eb33 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -426,18 +426,10 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) newCapacity += 4; } - char* t = taosMemoryRealloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - if (t == NULL) { - longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - pResultRowInfo->pPosition = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); - pResultRowInfo->pResult = (SResultRow**)t; int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); memset(&pResultRowInfo->pPosition[pResultRowInfo->capacity], 0, sizeof(SResultRowPosition)); - pResultRowInfo->capacity = (int32_t)newCapacity; } @@ -458,9 +450,8 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR if (p1 != NULL) { if (pResultRowInfo->size == 0) { existed = false; - assert(pResultRowInfo->curPos == -1); } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); +// existed = (pResultRowInfo->pResult[0] == (*p1)); } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); int64_t* index = @@ -479,6 +470,7 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR return p1 != NULL; } +#if 0 static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { bool existed = false; @@ -496,16 +488,16 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR if (p1 != NULL) { if (pResultRowInfo->size == 0) { existed = false; - assert(pResultRowInfo->curPos == -1); +// assert(pResultRowInfo->curPos == -1); } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - pResultRowInfo->curPos = 0; +// existed = (pResultRowInfo->pResult[0] == (*p1)); +// pResultRowInfo->curPos = 0; } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { - pResultRowInfo->curPos = (int32_t)*index; +// pResultRowInfo->curPos = (int32_t)*index; existed = true; } else { existed = false; @@ -555,6 +547,7 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR return pResultRowInfo->pResult[pResultRowInfo->curPos]; } +#endif SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) { SFilePage* pData = NULL; @@ -599,65 +592,75 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { - bool existed = false; + bool existInCurrentResusltRowInfo = false; SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, tableGroupId); - SResultRow** p1 = (SResultRow**)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); // in case of repeat scan/reverse scan, no new time window added. if (isIntervalQuery) { if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - return (p1 != NULL) ? *p1 : NULL; + if (p1 != NULL) { + return getResultRowByPos(pResultBuf, p1); + } else { + return NULL; + } } if (p1 != NULL) { if (pResultRowInfo->size == 0) { - existed = false; + existInCurrentResusltRowInfo = false; // this time window created by other timestamp that does not belongs to current table. assert(pResultRowInfo->curPos == -1); } else if (pResultRowInfo->size == 1) { - existed = (pResultRowInfo->pResult[0] == (*p1)); - pResultRowInfo->curPos = 0; - } else { // check if current pResultRowInfo contains the existed pResultRow + ASSERT(0); +// existInCurrentResusltRowInfo = (pResultRowInfo->pResult[0] == (*p1)); + } else { // check if current pResultRowInfo contains the existInCurrentResusltRowInfo pResultRow SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { - pResultRowInfo->curPos = (int32_t)*index; - existed = true; + // TODO check the scan order for current opened time window +// pResultRowInfo->curPos = (int32_t)*index; + existInCurrentResusltRowInfo = true; } else { - existed = false; + existInCurrentResusltRowInfo = false; } } } } else { - // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. + // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the pResultRowInfo object. if (p1 != NULL) { - return *p1; + return getResultRowByPos(pResultBuf, p1); } } - if (!existed) { - prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); - - SResultRow* pResult = NULL; - if (p1 == NULL) { - pResult = getNewResultRow_rv(pResultBuf, tableGroupId, pSup->resultRowSize); - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - // add a new result set for a new group - taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); - SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; - taosArrayPush(pSup->pResultRowArrayList, &cell); - } else { - pResult = *p1; + SResultRow* pResult = NULL; + if (!existInCurrentResusltRowInfo) { + // 1. close current opened time window + if (pResultRowInfo->curPos != -1) { // todo extract function + SResultRowPosition* pos = &pResultRowInfo->pPosition[pResultRowInfo->curPos]; + SFilePage* pPage = getBufPage(pResultBuf, pos->pageId); + SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset); + closeResultRow(pRow); + releaseBufPage(pResultBuf, pPage); } + prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); + if (p1 == NULL) { + pResult = getNewResultRow_rv(pResultBuf, tableGroupId, pSup->resultRowSize); + initResultRow(pResult); + + // add a new result set for a new group + SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, POINTER_BYTES); + SResultRowCell cell = {.groupId = tableGroupId, .pos = pos}; + taosArrayPush(pSup->pResultRowArrayList, &cell); + } else { + pResult = getResultRowByPos(pResultBuf, p1); + } + + // 2. set the new time window to be the new active time window pResultRowInfo->curPos = pResultRowInfo->size; - pResultRowInfo->pPosition[pResultRowInfo->size] = - (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + pResultRowInfo->pPosition[pResultRowInfo->size++] = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; int64_t index = pResultRowInfo->curPos; SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); @@ -669,7 +672,7 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return pResultRowInfo->pResult[pResultRowInfo->curPos]; + return pResult; } static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, TSKEY ekey, @@ -693,7 +696,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, +static STimeWindow getActiveTimeWindow(SDiskbasedBuf * pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t precision, STimeWindow* win) { STimeWindow w = {0}; @@ -701,7 +704,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t t getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + w = getResultRow(pBuf, pResultRowInfo, pResultRowInfo->curPos)->win; } if (w.skey > ts || w.ekey < ts) { @@ -730,7 +733,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t t // get the correct time window according to the handled timestamp static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, STaskAttr* pQueryAttr) { STimeWindow w = {0}; - +#if 0 if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value // getInitialStartTimeWindow(pQueryAttr, ts, &w); @@ -742,7 +745,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, in w.ekey = w.skey + pQueryAttr->interval.interval - 1; } } else { - w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + w = pRow->win; } /* @@ -752,6 +755,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, in if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { w.ekey = pQueryAttr->window.ekey; } +#endif return w; } @@ -816,8 +820,8 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowI assert(win->skey <= win->ekey); SDiskbasedBuf* pResultBuf = pRuntimeEnv->pResultBuf; - SResultRow* pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&win->skey, TSDB_KEYSIZE, - masterscan, tableGroupId); + SResultRow* pResultRow = NULL;//doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&win->skey, TSDB_KEYSIZE, +// masterscan, tableGroupId); if (pResultRow == NULL) { *pResult = NULL; return TSDB_CODE_SUCCESS; @@ -909,9 +913,9 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, - bool timeWindowInterpo) { +static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; +#if 0 int32_t i = 0; for (i = pResultRowInfo->size - 1; i >= 0; --i) { SResultRow* pResult = pResultRowInfo->pResult[i]; @@ -963,6 +967,7 @@ static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey pResultRowInfo->curPos = i + 1; // current not closed result object } } +#endif } static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, @@ -1253,8 +1258,8 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, ASSERT(pCtx[i].input.pData[j] != NULL); } } - // setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns); + // setBlockStatisInfo(&pCtx[i], pBlock, pOperator->pExpr[i].base.pColumns); // uint32_t flag = pOperator->pExpr[i].base.pParam[0].pCol->flag; // if (TSDB_COL_IS_NORMAL_COL(flag) /*|| (pCtx[i].functionId == FUNCTION_BLKINFO) || // (TSDB_COL_IS_TAG(flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)*/) { @@ -1551,14 +1556,14 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); tsCols = (int64_t*)pColDataInfo->pData; -// assert(tsCols[0] == pSDataBlock->info.window.skey && -// tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); +// assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] == +// pSDataBlock->info.window.ekey); } int32_t startPos = ascScan? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); - STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); bool masterScan = true; SResultRow* pResult = NULL; @@ -1581,6 +1586,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // prev time window not interpolation yet. int32_t curIndex = pResultRowInfo->curPos; + +#if 0 if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. SResultRow* pRes = getResultRow(pResultRowInfo, j); @@ -1615,6 +1622,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } +#endif // window start key interpolation doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, @@ -3430,10 +3438,8 @@ void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -// TODO fix this bug. -int32_t initResultRow(SResultRow* pResultRow) { +void initResultRow(SResultRow* pResultRow) { pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); - return TSDB_CODE_SUCCESS; } /* @@ -3449,7 +3455,9 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; + initResultRowInfo(pResultRowInfo, 16); int64_t tid = 0; int64_t groupId = 0; @@ -3610,9 +3618,11 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD SFilePage* bufPage = getBufPage(pBuf, pPos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); - if (!isResultRowClosed(pResultRowInfo, i)) { - continue; - } + + // TODO ignore the close status anyway. +// if (!isResultRowClosed(pRow)) { +// continue; +// } for (int32_t j = 0; j < numOfOutput; ++j) { pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); @@ -3622,7 +3632,7 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD continue; } - if (pCtx[j].fpSet.process) { // TODO set the dummy function. + if (pCtx[j].fpSet.process) { // TODO set the dummy function, to avoid the check for null ptr. pCtx[j].fpSet.finalize(&pCtx[j]); } @@ -4132,7 +4142,7 @@ static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutpu // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { // return; // } - +#if 0 for (int32_t i = 0; i < pResultRowInfo->size; ++i) { SResultRow* pResult = pResultRowInfo->pResult[i]; @@ -4146,6 +4156,8 @@ static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutpu pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); } } +#endif + } static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { @@ -7585,10 +7597,10 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); + pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); + pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); - pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || @@ -8761,8 +8773,8 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; - pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType); +// pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; +// pExp->base.pParam[0].pCol = createColumn(pTargetNode->dataBlockId, pTargetNode->slotId, pType); } else { ASSERT(0); } diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 50b9cae7ff..8fb5455336 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -63,9 +63,10 @@ typedef struct { FstRegex *regexCreate(const char *str); -void regexSetup(FstRegex *regex, uint32_t size, const char *str); - -// uint32_t regexStart() +uint32_t regexAutomStart(FstRegex *regex); +bool regexAutomIsMatch(FstRegex *regex, uint32_t state); +bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null); +bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result); #ifdef __cplusplus } diff --git a/source/libs/index/inc/indexFstSparse.h b/source/libs/index/inc/indexFstSparse.h index 69b33c82d9..665fb2ba5c 100644 --- a/source/libs/index/inc/indexFstSparse.h +++ b/source/libs/index/inc/indexFstSparse.h @@ -23,9 +23,9 @@ extern "C" { #endif typedef struct FstSparseSet { - SArray *dense; - SArray *sparse; - int32_t size; + uint32_t *dense; + uint32_t *sparse; + int32_t size; } FstSparseSet; FstSparseSet *sparSetCreate(int32_t sz); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d3ca3a1acf..7d52abcd1b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -27,7 +27,7 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 200 +#define INDEX_QUEUE_SIZE 200 void* indexQhandle = NULL; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index ca26cf38e5..df3c0b6e7b 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -21,8 +21,8 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 1024 * 1024 +#define MEM_TERM_LIMIT 10 * 10000 +#define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 static void indexMemRef(MemTable* tbl); diff --git a/source/libs/index/src/indexFstCommon.c b/source/libs/index/src/indexFstCommon.c index e2544c7ac3..902e68ce09 100644 --- a/source/libs/index/src/indexFstCommon.c +++ b/source/libs/index/src/indexFstCommon.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 TAOS Data, Inc. +YAML:9:25: error: unknown key 'AlignConsecutiveMacros' * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index ec41a7f58e..33eeae802e 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -14,6 +14,7 @@ */ #include "indexFstRegex.h" +#include "indexFstDfa.h" #include "indexFstSparse.h" FstRegex *regexCreate(const char *str) { @@ -26,9 +27,35 @@ FstRegex *regexCreate(const char *str) { memcpy(orig, str, sz); regex->orig = orig; + + // construct insts based on str + SArray *insts = NULL; + + FstDfaBuilder *builder = dfaBuilderCreate(insts); + regex->dfa = dfaBuilderBuild(builder); + return regex; } -void regexSetup(FstRegex *regex, uint32_t size, const char *str) { - // return - // return; +uint32_t regexAutomStart(FstRegex *regex) { + ///// no nothing + return 0; +} +bool regexAutomIsMatch(FstRegex *regex, uint32_t state) { + if (regex->dfa != NULL && dfaIsMatch(regex->dfa, state)) { + return true; + } else { + return false; + } +} + +bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null) { + // make frame happy + return null; +} + +bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result) { + if (regex->dfa == NULL) { + return false; + } + return dfaAccept(regex->dfa, state, byte, result); } diff --git a/source/libs/index/src/indexSparse.c b/source/libs/index/src/indexFstSparse.c similarity index 65% rename from source/libs/index/src/indexSparse.c rename to source/libs/index/src/indexFstSparse.c index 9d228e71ff..e8ab3be2fe 100644 --- a/source/libs/index/src/indexSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -21,47 +21,44 @@ FstSparseSet *sparSetCreate(int32_t sz) { return NULL; } - ss->dense = taosArrayInit(sz, sizeof(uint32_t)); - ss->sparse = taosArrayInit(sz, sizeof(uint32_t)); - ss->size = sz; + ss->dense = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t)); + ss->sparse = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t)); + ss->size = 0; return ss; } void sparSetDestroy(FstSparseSet *ss) { if (ss == NULL) { return; } - taosArrayDestroy(ss->dense); - taosArrayDestroy(ss->sparse); + taosMemoryFree(ss->dense); + taosMemoryFree(ss->sparse); taosMemoryFree(ss); } -uint32_t sparSetLen(FstSparseSet *ss) { return ss == NULL ? 0 : ss->size; } +uint32_t sparSetLen(FstSparseSet *ss) { + // Get occupied size + return ss == NULL ? 0 : ss->size; +} uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) { if (ss == NULL) { return 0; } uint32_t i = ss->size; - taosArraySet(ss->dense, i, &ip); - taosArraySet(ss->sparse, ip, &i); + ss->dense[i] = ip; + ss->sparse[ip] = i; ss->size += 1; return i; } uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) { - if (i >= taosArrayGetSize(ss->dense)) { - return 0; - } - uint32_t *v = taosArrayGet(ss->dense, i); - return *v; + // check later + return ss->dense[i]; } bool sparSetContains(FstSparseSet *ss, uint32_t ip) { - if (ip >= taosArrayGetSize(ss->sparse)) { + uint32_t i = ss->sparse[ip]; + if (i < ss->size && ss->dense[i] == ip) { + return true; + } else { return false; } - uint32_t i = *(uint32_t *)taosArrayGet(ss->sparse, ip); - if (i >= taosArrayGetSize(ss->dense)) { - return false; - } - uint32_t v = *(uint32_t *)taosArrayGet(ss->dense, i); - return v == ip; } void sparSetClear(FstSparseSet *ss) { if (ss == NULL) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index ed67de17e0..e94dfd9e0a 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -65,7 +65,7 @@ typedef struct SInsertParseContext { SVnodeModifOpStmt* pOutput; } SInsertParseContext; -typedef int32_t (*_row_append_fn_t)(const void *value, int32_t len, void *param); +typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param); static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; @@ -444,26 +444,26 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (isNullStr(pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { int64_t tmpVal = 0; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } - return func(NULL, 0, param); + return func(pMsgBuf, NULL, 0, param); } switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - return func(&TRUE_VALUE, pSchema->bytes, param); + return func(pMsgBuf, &TRUE_VALUE, pSchema->bytes, param); } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - return func(&FALSE_VALUE, pSchema->bytes, param); + return func(pMsgBuf, &FALSE_VALUE, pSchema->bytes, param); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else if (pToken->type == TK_NK_FLOAT) { - return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } @@ -477,7 +477,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } uint8_t tmpVal = (uint8_t)iv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_UTINYINT:{ @@ -487,7 +487,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); } uint8_t tmpVal = (uint8_t)iv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_SMALLINT: { @@ -497,7 +497,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); } int16_t tmpVal = (int16_t)iv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_USMALLINT: { @@ -507,7 +507,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); } uint16_t tmpVal = (uint16_t)iv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_INT: { @@ -517,7 +517,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); } int32_t tmpVal = (int32_t)iv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_UINT: { @@ -527,7 +527,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); } uint32_t tmpVal = (uint32_t)iv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_BIGINT: { @@ -536,7 +536,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int } else if (!IS_VALID_BIGINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); } - return func(&iv, pSchema->bytes, param); + return func(pMsgBuf, &iv, pSchema->bytes, param); } case TSDB_DATA_TYPE_UBIGINT: { @@ -546,7 +546,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); } uint64_t tmpVal = (uint64_t)iv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_FLOAT: { @@ -558,7 +558,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } float tmpVal = (float)dv; - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } case TSDB_DATA_TYPE_DOUBLE: { @@ -569,7 +569,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); } - return func(&dv, pSchema->bytes, param); + return func(pMsgBuf, &dv, pSchema->bytes, param); } case TSDB_DATA_TYPE_BINARY: { @@ -578,11 +578,11 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "string data overflow", pToken->z); } - return func(pToken->z, pToken->n, param); + return func(pMsgBuf, pToken->z, pToken->n, param); } case TSDB_DATA_TYPE_NCHAR: { - return func(pToken->z, pToken->n, param); + return func(pMsgBuf, pToken->z, pToken->n, param); } case TSDB_DATA_TYPE_TIMESTAMP: { @@ -591,7 +591,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); } - return func(&tmpVal, pSchema->bytes, param); + return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } } @@ -605,7 +605,7 @@ typedef struct SMemParam { col_id_t colIdx; } SMemParam; -static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* param) { +static FORCE_INLINE int32_t MemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { SMemParam* pa = (SMemParam*)param; SRowBuilder* rb = pa->rb; if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { @@ -617,7 +617,9 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p int32_t output = 0; const char* rowEnd = tdRowEnd(rb->pBuf); if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(pMsgBuf, buf, value); } varDataSetLen(rowEnd, output); tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); @@ -714,7 +716,7 @@ typedef struct SKvParam { char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; -static int32_t KvRowAppend(const void *value, int32_t len, void *param) { +static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*) param; int8_t type = pa->schema->type; @@ -727,7 +729,9 @@ static int32_t KvRowAppend(const void *value, int32_t len, void *param) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t output = 0; if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(pMsgBuf, buf, value);; } varDataSetLen(pa->buf, output); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 65e248a2bc..a8ef762a73 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1217,6 +1217,9 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt SName tableName; int32_t code = getTableMetaImpl( pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); + if ((TSDB_CODE_TDB_INVALID_TABLE_ID == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) { + return TSDB_CODE_SUCCESS; + } if (TSDB_CODE_SUCCESS == code) { if (TSDB_SUPER_TABLE == pTableMeta->tableType) { code = doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); @@ -2167,7 +2170,7 @@ typedef struct SVgroupTablesBatch { static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { pSchema->colId = colId; pSchema->type = pCol->dataType.type; - pSchema->bytes = pCol->dataType.bytes; + pSchema->bytes = calcTypeBytes(pCol->dataType); strcpy(pSchema->name, pCol->colName); } @@ -2592,8 +2595,10 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { break; default: pQuery->directRpc = true; - TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg, SCmdMsgInfo*); - pQuery->msgType = pQuery->pCmdMsg->msgType; + if (NULL != pCxt->pCmdMsg) { + TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg, SCmdMsgInfo*); + pQuery->msgType = pQuery->pCmdMsg->msgType; + } break; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index c46f98ef55..11baeff04a 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -719,6 +719,11 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc if (NULL == pScan->pScanCols) { code = TSDB_CODE_OUT_OF_MEMORY; } + + if (TSDB_CODE_SUCCESS == code) { + code = sortScanCols(pScan->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { code = sortScanCols(pScan->pScanCols); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f22f9a5c3c..33be65832c 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -720,7 +720,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); - if (compareDoubleVal(&zero, &rx)) { + if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { colDataAppend(pOutputCol, i, NULL, true); continue; } @@ -729,7 +729,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pLeft->numOfRows == 1) { double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0); - if (colDataIsNull_f(pLeftCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value // TODO set numOfRows NULL value } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { @@ -739,7 +739,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } double rx = getVectorDoubleValueFnRight(pRightCol->pData, i); - if (compareDoubleVal(&zero, &rx)) { + if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppend(pOutputCol, i, NULL, true); continue; } @@ -749,17 +749,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pRight->numOfRows == 1) { double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0); - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value // TODO set numOfRows NULL value } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_f(pRightCol->nullbitmap, i)) { + if (colDataIsNull_f(pLeftCol->nullbitmap, i)) { colDataAppend(pOutputCol, i, NULL, true); continue; } - double lx = getVectorDoubleValueFnLeft(pRightCol->pData, i); - if (compareDoubleVal(&zero, &lx)) { + double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i); + if (isnan(lx) || isinf(lx)) { colDataAppend(pOutputCol, i, NULL, true); continue; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 5af13d97ca..61462f1dc3 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -75,10 +75,10 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { SRpcCtxVal *ctxVal = (SRpcCtxVal *)pIter; (*ctxVal->freeFunc)(ctxVal->val); - + pIter = taosHashIterate(pCtx->args, pIter); } - + taosHashCleanup(pCtx->args); if (pCtx->brokenVal.freeFunc) { @@ -86,7 +86,7 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { } } -void schFreeTask(SSchTask* pTask) { +void schFreeTask(SSchTask *pTask) { if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } @@ -125,41 +125,47 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m case TDMT_SCH_LINK_BROKEN: return TSDB_CODE_SUCCESS; case TDMT_VND_QUERY_RSP: // query_rsp may be processed later than ready_rsp - if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { - SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { + SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); } - + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_RES_READY_RSP: reqMsgType = TDMT_VND_QUERY; if (lastMsgType != reqMsgType && -1 != lastMsgType) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", (lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", + (lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_FETCH_RSP: - if (lastMsgType != reqMsgType && -1 != lastMsgType) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + if (lastMsgType != reqMsgType && -1 != lastMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; case TDMT_VND_CREATE_TABLE_RSP: @@ -171,12 +177,14 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m } if (lastMsgType != reqMsgType) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), + TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -357,7 +365,7 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, void *handle) { SSchNodeInfo nodeInfo = {.addr = *addr, .handle = handle}; - + if (NULL == taosArrayPush(pTask->execNodes, &nodeInfo)) { SCH_TASK_ELOG("taosArrayPush nodeInfo to execNodes list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -587,7 +595,7 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_WLOG("task already in failTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -612,7 +620,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != code) { if (HASH_NODE_EXIST(code)) { *moved = true; - + SCH_TASK_ELOG("task already in execTask list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -631,7 +639,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { int8_t status = 0; ++pTask->tryTimes; - + if (schJobNeedToStop(pJob, &status)) { *needRetry = false; SCH_TASK_DLOG("task no more retry cause of job status, job status:%s", jobTaskStatusStr(status)); @@ -643,7 +651,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo SCH_TASK_DLOG("task no more retry since reach max try times, tryTimes:%d", pTask->tryTimes); return TSDB_CODE_SUCCESS; } - + if (!NEED_SCHEDULER_RETRY_ERROR(errCode)) { *needRetry = false; SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode)); @@ -654,7 +662,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo if (SCH_IS_DATA_SRC_TASK(pTask)) { if (pTask->tryTimes >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes, SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); + SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes, + SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); return TSDB_CODE_SUCCESS; } } else { @@ -662,14 +671,15 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo if ((pTask->candidateIdx + 1) >= candidateNum) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", pTask->candidateIdx, candidateNum); + SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", + pTask->candidateIdx, candidateNum); return TSDB_CODE_SUCCESS; } } *needRetry = true; SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->tryTimes, errCode, tstrerror(errCode)); - + return TSDB_CODE_SUCCESS; } @@ -706,9 +716,8 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { memcpy(&hb->trans, trans, sizeof(*trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); - qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", - schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, - trans->transHandle); + qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", schMgmt.sId, + epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, trans->transHandle); return TSDB_CODE_SUCCESS; } @@ -730,15 +739,15 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { if (NEED_CLIENT_HANDLE_ERROR(origCode)) { return; } - + if (NEED_CLIENT_HANDLE_ERROR(errCode)) { atomic_store_32(&pJob->errCode, errCode); goto _return; } return; - -_return: + +_return: SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode)); } @@ -825,7 +834,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); - + if (SCH_IS_WAIT_ALL_JOB(pJob)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskFailed++; @@ -833,9 +842,9 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); schUpdateJobErrCode(pJob, errCode); - + if (taskDone < pTask->level->taskNum) { - SCH_TASK_DLOG("need to wait other tasks, doneNum:%d, allNum:%d", taskDone, pTask->level->taskNum); + SCH_TASK_DLOG("need to wait other tasks, doneNum:%d, allNum:%d", taskDone, pTask->level->taskNum); SCH_RET(errCode); } } @@ -867,7 +876,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { - int32_t taskDone = 0; + int32_t taskDone = 0; if (SCH_IS_WAIT_ALL_JOB(pJob)) { SCH_LOCK(SCH_WRITE, &pTask->level->lock); pTask->level->taskSucceed++; @@ -965,7 +974,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { - SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), rspCode); + SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), + rspCode); SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -985,11 +995,11 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(rsp->code); } } - + taosArrayDestroy(batchRsp.rspList); } - } - + } + SCH_ERR_JRET(rspCode); SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); break; @@ -1011,21 +1021,21 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } case TDMT_VND_QUERY_RSP: { - SQueryTableRsp rsp = {0}; - if (msg) { - tDeserializeSQueryTableRsp(msg, msgSize, &rsp); - SCH_ERR_JRET(rsp.code); - } - - SCH_ERR_JRET(rspCode); - - if (NULL == msg) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - //SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); - - break; + SQueryTableRsp rsp = {0}; + if (msg) { + tDeserializeSQueryTableRsp(msg, msgSize, &rsp); + SCH_ERR_JRET(rsp.code); + } + + SCH_ERR_JRET(rspCode); + + if (NULL == msg) { + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + // SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); + + break; } case TDMT_VND_RES_READY_RSP: { SResReadyRsp *rsp = (SResReadyRsp *)msg; @@ -1088,14 +1098,14 @@ _return: } int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { - int32_t code = 0; + int32_t code = 0; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SSchTask *pTask = NULL; + SSchTask *pTask = NULL; SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, - pParam->queryId, pParam->taskId, pParam->refId); + pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -1114,7 +1124,7 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in pTask = *task; SCH_TASK_DLOG("rsp msg received, type:%s, handle:%p, code:%s", TMSG_INFO(msgType), pMsg->handle, tstrerror(rspCode)); - SCH_SET_TASK_HANDLE(pTask, pMsg->handle); + SCH_SET_TASK_HANDLE(pTask, pMsg->handle); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: @@ -1173,7 +1183,8 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans)); int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); - qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, rsp.epId.ep.port); + qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, + rsp.epId.ep.port); for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); @@ -1187,8 +1198,9 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { } // TODO - - SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, jobTaskStatusStr(taskStatus->status)); + + SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, + jobTaskStatusStr(taskStatus->status)); schReleaseJob(taskStatus->refId); } @@ -1208,7 +1220,7 @@ int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t c if (head->isHbParam) { SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; - SSchTrans trans = {.transInst = hbParam->transport, .transHandle = NULL}; + SSchTrans trans = {.transInst = hbParam->transport, .transHandle = NULL}; SCH_ERR_RET(schUpdateHbConnection(&hbParam->nodeEpId, &trans)); SCH_ERR_RET(schBuildAndSendHbMsg(&hbParam->nodeEpId)); @@ -1219,7 +1231,6 @@ int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t c return TSDB_CODE_SUCCESS; } - int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { switch (msgType) { case TDMT_VND_CREATE_TABLE: @@ -1258,8 +1269,8 @@ void schFreeRpcCtxVal(const void *arg) { if (NULL == arg) { return; } - - SMsgSendInfo* pMsgSendInfo = (SMsgSendInfo *)arg; + + SMsgSendInfo *pMsgSendInfo = (SMsgSendInfo *)arg; taosMemoryFreeClear(pMsgSendInfo->param); taosMemoryFreeClear(pMsgSendInfo); } @@ -1301,11 +1312,10 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { return TSDB_CODE_SUCCESS; } - int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) { - int32_t code = 0; - SMsgSendInfo* pMsgSendInfo = NULL; - + int32_t code = 0; + SMsgSendInfo *pMsgSendInfo = NULL; + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); @@ -1318,17 +1328,17 @@ int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal * SCH_ERR_JRET(schMakeTaskCallbackParam(pJob, pTask, &pMsgSendInfo->param)); } - int32_t msgType = TDMT_SCH_LINK_BROKEN; + int32_t msgType = TDMT_SCH_LINK_BROKEN; __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); - + pMsgSendInfo->fp = fp; brokenVal->msgType = msgType; brokenVal->val = pMsgSendInfo; brokenVal->clone = schCloneSMsgSendInfo; brokenVal->freeFunc = schFreeRpcCtxVal; - + return TSDB_CODE_SUCCESS; _return: @@ -1340,16 +1350,16 @@ _return: } int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { - int32_t code = 0; + int32_t code = 0; SSchTaskCallbackParam *param = NULL; - SMsgSendInfo* pMsgSendInfo = NULL; + SMsgSendInfo *pMsgSendInfo = NULL; pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); @@ -1362,7 +1372,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgType = TDMT_VND_RES_READY_RSP; + int32_t msgType = TDMT_VND_RES_READY_RSP; __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_RES_READY, &fp)); @@ -1370,7 +1380,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { param->refId = pJob->refId; param->taskId = SCH_TASK_ID(pTask); param->transport = pJob->transport; - + pMsgSendInfo->param = param; pMsgSendInfo->fp = fp; @@ -1394,11 +1404,11 @@ _return: } int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { - int32_t code = 0; + int32_t code = 0; SSchHbCallbackParam *param = NULL; - SMsgSendInfo* pMsgSendInfo = NULL; - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SQueryNodeEpId epId = {0}; + SMsgSendInfo *pMsgSendInfo = NULL; + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); @@ -1408,7 +1418,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); @@ -1421,13 +1431,13 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgType = TDMT_VND_QUERY_HEARTBEAT_RSP; + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT_RSP; __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_QUERY_HEARTBEAT, &fp)); param->nodeEpId = epId; param->transport = pJob->transport; - + pMsgSendInfo->param = param; pMsgSendInfo->fp = fp; @@ -1450,19 +1460,18 @@ _return: SCH_RET(code); } - int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { - int32_t code = 0; + int32_t code = 0; SSchHbTrans hb = {0}; hb.trans.transInst = pJob->transport; - + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); if (code) { schFreeRpcCtx(&hb.rpcCtx); - + if (HASH_NODE_EXIST(code)) { *exist = true; return TSDB_CODE_SUCCESS; @@ -1475,8 +1484,6 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId * return TSDB_CODE_SUCCESS; } - - int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHeader **pDst) { if (pSrc->isHbParam) { SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); @@ -1496,16 +1503,16 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe qError("malloc SSchTaskCallbackParam failed"); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + memcpy(dst, pSrc, sizeof(*dst)); *pDst = (SSchCallbackParamHeader *)dst; - + return TSDB_CODE_SUCCESS; } int32_t schCloneSMsgSendInfo(void *src, void **dst) { SMsgSendInfo *pSrc = src; - int32_t code = 0; + int32_t code = 0; SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == pDst) { qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); @@ -1520,7 +1527,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { *dst = pDst; return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(pDst); @@ -1531,7 +1538,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { int32_t code = 0; memcpy(&pDst->brokenVal, &pSrc->brokenVal, sizeof(pSrc->brokenVal)); pDst->brokenVal.val = NULL; - + SCH_ERR_RET(schCloneSMsgSendInfo(pSrc->brokenVal.val, &pDst->brokenVal.val)); pDst->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); @@ -1541,16 +1548,16 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { } SRpcCtxVal dst = {0}; - void *pIter = taosHashIterate(pSrc->args, NULL); + void *pIter = taosHashIterate(pSrc->args, NULL); while (pIter) { SRpcCtxVal *pVal = (SRpcCtxVal *)pIter; - int32_t *msgType = taosHashGetKey(pIter, NULL); + int32_t *msgType = taosHashGetKey(pIter, NULL); dst = *pVal; dst.val = NULL; - + SCH_ERR_JRET(schCloneSMsgSendInfo(pVal->val, &dst.val)); - + if (taosHashPut(pDst->args, msgType, sizeof(*msgType), &dst, sizeof(dst))) { qError("taosHashPut msg %d to rpcCtx failed", *msgType); (*dst.freeFunc)(dst.val); @@ -1568,8 +1575,8 @@ _return: SCH_RET(code); } - -int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) { +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet *epSet, int32_t msgType, void *msg, + uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; @@ -1601,11 +1608,11 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", - TMSG_INFO(msgType), ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, - pJob->refId, trans->transInst, trans->transHandle); - - int64_t transporterId = 0; + qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", TMSG_INFO(msgType), + ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, pJob->refId, + trans->transInst, trans->transHandle); + + int64_t transporterId = 0; code = asyncSendMsgToServerExt(trans->transInst, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx); if (code) { SCH_ERR_JRET(code); @@ -1623,18 +1630,19 @@ _return: int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { SSchedulerHbReq req = {0}; - int32_t code = 0; - SRpcCtx rpcCtx = {0}; - SSchTrans trans = {0}; - int32_t msgType = TDMT_VND_QUERY_HEARTBEAT; + int32_t code = 0; + SRpcCtx rpcCtx = {0}; + SSchTrans trans = {0}; + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT; - req.header.vgId = htonl(nodeEpId->nodeId); + req.header.vgId = nodeEpId->nodeId; req.sId = schMgmt.sId; memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); if (NULL == hb) { - qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, nodeEpId->ep.port); + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, + nodeEpId->ep.port); SCH_ERR_RET(code); } @@ -1642,9 +1650,9 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); memcpy(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); - + SCH_ERR_RET(code); - + int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); @@ -1655,7 +1663,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { qError("calloc hb req %d failed", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1684,17 +1692,18 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { pMsgSendInfo->msgInfo.handle = trans.transHandle; pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - - int64_t transporterId = 0; - SEpSet epSet = {.inUse = 0, .numOfEps = 1}; + + int64_t transporterId = 0; + SEpSet epSet = {.inUse = 0, .numOfEps = 1}; memcpy(&epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); - qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port); - + qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle, + nodeEpId->ep.fqdn, nodeEpId->ep.port); + code = asyncSendMsgToServerExt(trans.transInst, &epSet, &transporterId, pMsgSendInfo, true, &rpcCtx); if (code) { - qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", - trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code)); + qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", trans.transInst, + trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code)); SCH_ERR_JRET(code); } @@ -1712,12 +1721,12 @@ _return: int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void *msg = NULL; - int32_t code = 0; - bool isCandidateAddr = false; - bool persistHandle = false; - SRpcCtx rpcCtx = {0}; - + void *msg = NULL; + int32_t code = 0; + bool isCandidateAddr = false; + bool persistHandle = false; + SRpcCtx rpcCtx = {0}; + if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; @@ -1741,7 +1750,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_QUERY: { SCH_ERR_RET(schMakeQueryRpcCtx(pJob, pTask, &rpcCtx)); - + uint32_t len = strlen(pJob->sql); msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; msg = taosMemoryCalloc(1, msgSize); @@ -1822,7 +1831,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_QUERY_HEARTBEAT: { SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &rpcCtx)); - + SSchedulerHbReq req = {0}; req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; @@ -1856,7 +1865,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType); SSchTrans trans = {.transInst = pJob->transport, .transHandle = SCH_GET_TASK_HANDLE(pTask)}; - SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL))); + SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle, + (rpcCtx.args ? &rpcCtx : NULL))); if (msgType == TDMT_VND_QUERY) { SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr, trans.transHandle)); @@ -1902,7 +1912,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no need to launch task cause of job status, job status:%s", jobTaskStatusStr(status)); - + SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -1911,7 +1921,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schPushTaskToExecList(pJob, pTask)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); } - + SSubplan *plan = pTask->plan; if (NULL == pTask->msg) { // TODO add more detailed reason for failure @@ -2071,7 +2081,7 @@ void schFreeJobImpl(void *job) { taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); - + taosMemoryFreeClear(pJob->resData); taosMemoryFreeClear(pJob); @@ -2212,7 +2222,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; - + schReleaseJob(*pJob); return TSDB_CODE_SUCCESS; @@ -2400,7 +2410,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_JOB_ELOG("job failed or dropping, status:%s", jobTaskStatusStr(status)); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } - + if (pJob->resData && ((SRetrieveTableRsp *)pJob->resData)->completed) { SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 5223aa1e3a..8aaaa414ca 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -38,7 +38,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* req.taskId = pTask->fixedEpDispatcher.taskId; } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - // TODO fix tbname issue + // TODO use general name rule of schemaless char ctbName[TSDB_TABLE_FNAME_LEN + 22]; // all groupId must be the same in an array SSDataBlock* pBlock = taosArrayGet(data, 0); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 5296a16703..2545eeab4b 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -30,6 +30,9 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { pRead->curFileFirstVer = -1; pRead->capacity = 0; pRead->status = 0; + + taosThreadMutexInit(&pRead->mutex, NULL); + pRead->pHead = taosMemoryMalloc(sizeof(SWalHead)); if (pRead->pHead == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; @@ -135,6 +138,22 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { return 0; } +int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead) { + taosThreadMutexLock(&pRead->mutex); + if (walReadWithHandle(pRead, ver) < 0) { + taosThreadMutexUnlock(&pRead->mutex); + return -1; + } + *ppHead = taosMemoryMalloc(sizeof(SWalReadHead) + pRead->pHead->head.len); + if (*ppHead == NULL) { + taosThreadMutexUnlock(&pRead->mutex); + return -1; + } + memcpy(*ppHead, &pRead->pHead->head, sizeof(SWalReadHead) + pRead->pHead->head.len); + taosThreadMutexUnlock(&pRead->mutex); + return 0; +} + int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int code; // TODO: check wal life @@ -145,7 +164,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - if (!taosValidFile(pRead->pReadLogTFile)) return -1; + if (!taosValidFile(pRead->pReadLogTFile)) { + return -1; + } code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index c98f6eb9be..7f9a3ff593 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -173,6 +173,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { if (isnan(p2)) { return 1; } + if (FLT_EQUAL(p1, p2)) { return 0; } diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0efcf517a9..93a843fee8 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -121,7 +121,8 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { return tjsonAddItemToArray(pJson, pJobj); } -int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, int32_t num) { +int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, + int32_t num) { if (num > 0) { SJson* pJsonArray = tjsonAddArrayToObject(pJson, pName); if (NULL == pJsonArray) { @@ -168,7 +169,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal } *pVal = strtol(p, NULL, 10); - return (errno == EINVAL || errno == ERANGE) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal) { @@ -199,19 +200,19 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV } *pVal = strtoul(p, NULL, 10); - return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) { uint64_t val = 0; - int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); *pVal = val; return code; } int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) { uint64_t val = 0; - int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); *pVal = val; return code; } @@ -264,7 +265,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize) { const cJSON* jArray = tjsonGetObjectItem(pJson, pName); - int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray)); + int32_t size = (NULL == jArray ? 0 : tjsonGetArraySize(jArray)); for (int32_t i = 0; i < size; ++i) { int32_t code = func(tjsonGetArrayItem(jArray, i), (char*)pArray + itemSize * i); if (TSDB_CODE_SUCCESS != code) { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 37233c288f..811c9941f7 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -5,6 +5,7 @@ ./test.sh -f tsim/user/basic1.sim # ---- db +./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 5c2848883b..0fc7175cb1 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -59,22 +59,12 @@ endi print ============= create database #database_option: { # BLOCKS value [3~1000, default: 6] -# | CACHE value [default: 16M] # | CACHELAST value [0, 1, 2, 3] -# | COMP [0 | 1 | 2] -# | DAYS value [unit is minutes] # | FSYNC value [0 ~ 180000 ms] -# | MAXROWS value [default: 4096] -# | MINROWS value [default: 100] # | KEEP value [days, 365000] -# | PRECISION ['ms' | 'us' | 'ns'] # | QUORUM value [1 | 2] # | REPLICA value [1 | 3] -# | TTL value [unit is day, min=1] # | WAL value [1 | 2] -# | VGROUPS value [default: 2] -# | SINGLE_STABLE [0 | 1] -# | STREAM_MODE [0 | 1] sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 sql show databases @@ -187,22 +177,10 @@ sql_error alter database db quorum 4 sql_error alter database db quorum 5 #print ============== modify days -#sql alter database db days 480 -#sql show databases -#print days $data6_db -#if $data6_db != 480 then # days -# return -1 -#endi -#sql alter database db days 360 -#sql show databases -#print days $data6_db -#if $data6_db != 360 then # days -# return -1 -#endi - +sql_error alter database db days 480 +sql_error alter database db days 360 sql_error alter database db days 0 -#sql_error alter database db days 14400 # set over than keep - +sql_error alter database db days 14400 # set over than keep print ============== modify keep sql alter database db keep 2000 @@ -230,25 +208,14 @@ sql_error alter database db keep -1 #sql_error alter database db keep 365001 print ============== modify cache -#sql alter database db cache 12 -#sql show databases -#print cache $data8_db -#if $data8_db != 12 then -# return -1 -#endi -#sql alter database db cache 1 -#sql show databases -#print cache $data8_db -#if $data8_db != 6 then -# return -1 -#endi -# -#sql_error alter database db cache 60 -#sql_error alter database db cache 50 -#sql_error alter database db cache 20 -#sql_error alter database db cache 3 -#sql_error alter database db cache 129 -#sql_error alter database db cache 300 +sql_error alter database db cache 12 +sql_error alter database db cache 1 +sql_error alter database db cache 60 +sql_error alter database db cache 50 +sql_error alter database db cache 20 +sql_error alter database db cache 3 +sql_error alter database db cache 129 +sql_error alter database db cache 300 sql_error alter database db cache 0 sql_error alter database db cache -1 @@ -276,45 +243,18 @@ sql_error alter database db blocks 0 sql_error alter database db blocks -1 sql_error alter database db blocks 10001 -#print ============== modify minrows -#sql alter database db minrows 8 -#sql show databases -#print minrows $data10_db -#if $data10_db != 8 then -# return -1 -#endi -#sql alter database db minrows 200 -#sql show databases -#print minrows $data10_db -#if $data10_db != 200 then -# return -1 -#endi -# -#sql alter database db minrows 11 -#sql show databases -#print minrows $data10_db -#if $data10_db != 11 then -# return -1 -#endi -#sql_error alter database db minrows 8000 -#sql_error alter database db minrows 8001 +print ============== modify minrows +sql_error alter database db minrows 8 +sql_error alter database db minrows 200 +sql_error alter database db minrows 11 +sql_error alter database db minrows 8000 +sql_error alter database db minrows 8001 -#print ============== modify maxrows -#sql alter database db maxrows 1000 -#sql show databases -#print maxrows $data11_db -#if $data11_db != 1000 then -# return -1 -#endi -#sql alter database db maxrows 2000 -#sql show databases -#print maxrows $data11_db -#if $data11_db != 2000 then -# return -1 -#endi -# -#sql_error alter database db maxrows 11 # equal minrows -#sql_error alter database db maxrows 10 # little than minrows +print ============== modify maxrows +sql_error alter database db maxrows 1000 +sql_error alter database db maxrows 2000 +sql_error alter database db maxrows 11 # equal minrows +sql_error alter database db maxrows 10 # little than minrows print ============== step wal sql alter database db wal 1 @@ -330,7 +270,7 @@ if $data12_db != 2 then return -1 endi -sql_error alter database db wal 0 +#sql_error alter database db wal 0 # TD-14436 sql_error alter database db wal 3 sql_error alter database db wal 100 sql_error alter database db wal -1 @@ -348,35 +288,20 @@ print fsync $data13_db if $data13_db != 500 then return -1 endi -sql_error alter database db fsync 0 +sql alter database db fsync 0 +sql show databases +print fsync $data13_db +if $data13_db != 0 then + return -1 +endi +sql_error alter database db fsync 180001 sql_error alter database db fsync -1 print ============== modify comp -sql alter database db comp 1 -sql show databases -print comp $data14_db -if $data14_db != 1 then - return -1 -endi -sql alter database db comp 2 -sql show databases -print comp $data14_db -if $data14_db != 2 then - return -1 -endi -sql alter database db comp 1 -sql show databases -print comp $data14_db -if $data14_db != 1 then - return -1 -endi -sql alter database db comp 0 -sql show databases -print comp $data14_db -if $data14_db != 0 then - return -1 -endi - +sql_error alter database db comp 1 +sql_error alter database db comp 2 +sql_error alter database db comp 1 +sql_error alter database db comp 0 sql_error alter database db comp 3 sql_error alter database db comp 4 sql_error alter database db comp 5 @@ -414,30 +339,15 @@ if $data15_db != 3 then return -1 endi -sql_error alter database db comp 4 -sql_error alter database db comp 10 -sql_error alter database db comp -1 +sql_error alter database db cachelast 4 +sql_error alter database db cachelast 10 +sql_error alter database db cachelast -1 print ============== modify precision -sql alter database db precision 'ms' -sql show databases -print precision $data16_db -if $data16_db != ms then - return -1 -endi -sql alter database db precision 'us' -sql show databases -print precision $data16_db -if $data16_db != us then - return -1 -endi -sql alter database db precision 'ns' -sql show databases -print precision $data16_db -if $data16_db != ns then - return -1 -endi - +sql_error alter database db precision 'ms' +sql_error alter database db precision 'us' +sql_error alter database db precision 'ns' +sql_error alter database db precision 'ys' sql_error alter database db prec 'xs' #system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 4c4ebc6670..ffb13979cd 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -7,7 +7,7 @@ sql connect print =============== create database sql create database d0 sql show databases -if $rows != 2 then +if $rows != 2 then return -1 endi @@ -17,7 +17,7 @@ print =============== create super table and child table sql create table stb (ts timestamp, tbcol int) tags (t1 int) sql show stables print $rows $data00 $data01 $data02 -if $rows != 1 then +if $rows != 1 then return -1 endi @@ -29,7 +29,7 @@ sql show tables print $rows $data00 $data10 $data20 if $rows != 4 then return -1 -endi +endi print =============== insert data into child table ct1 (s) sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1 ) @@ -83,13 +83,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 if $rows != 5 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data40 != 1 then return -1 -endi +endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) @@ -101,13 +101,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 if $rows != 5 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data40 != 1 then return -1 -endi +endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) @@ -123,16 +123,16 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74 print ===> rows8: $data80 $data81 $data82 $data83 $data84 if $rows != 9 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data70 != 2 then return -1 -endi +endi if $data80 != 1 then return -1 -endi +endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) @@ -144,10 +144,10 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 if $rows != 4 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data10 != 2 then return -1 endi @@ -155,27 +155,54 @@ endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) print ===> rows: $rows -print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 -print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 -print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 -print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 -print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 -print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 -print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 -print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 +print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 if $rows != 8 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data10 != 2 then return -1 -endi +endi if $data70 != 1 then return -1 endi - + +sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from ct2 interval(1d, 2h) sliding(12h) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 +if $rows != 8 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data75 != 1 then + return -1 +endi +if $data02 != 86400000 then + return -1 +endi + sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) print ===> rows: $rows @@ -184,15 +211,15 @@ print ===> rows1: $data10 $data11 $data12 $data13 $data14 print ===> rows2: $data20 $data21 $data22 $data23 $data24 print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 -#if $rows != 5 then -# return -1 -#endi -#if $data00 != 1 then -# return -1 -#endi -#if $data40 != 1 then -# return -1 -#endi +# if $rows != 5 then +# return -1 +# endi +# f $data00 != 1 then +# return -1 +# endi +# if $data40 != 1 then +# return -1 +# endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) @@ -204,13 +231,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 #if $rows != 5 then # return -1 -#endi +#endi #if $data00 != 1 then # return -1 -#endi +#endi #if $data40 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) @@ -225,13 +252,13 @@ print ===> rows6: $data60 $data61 $data62 $data63 $data64 print ===> rows7: $data70 $data71 $data72 $data73 $data74 #if $rows != 8 then # return -1 -#endi +#endi #if $data00 != 2 then # return -1 -#endi +#endi #if $data70 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) @@ -243,13 +270,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 #if $rows != 5 then # return -1 -#endi +#endi #if $data00 != 1 then # return -1 -#endi +#endi #if $data40 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) @@ -261,13 +288,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 #if $rows != 5 then # return -1 -#endi +#endi #if $data00 != 1 then # return -1 -#endi +#endi #if $data40 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) @@ -282,13 +309,13 @@ print ===> rows6: $data60 $data61 $data62 $data63 $data64 print ===> rows7: $data70 $data71 $data72 $data73 $data74 #if $rows != 8 then # return -1 -#endi +#endi #if $data00 != 2 then # return -1 -#endi +#endi #if $data70 != 1 then # return -1 -#endi +#endi #================================================= print =============== stop and restart taosd @@ -322,9 +349,4 @@ endi #sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) - - - - - #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 6dd0a9537e..384008c887 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -29,18 +29,18 @@ $i = 0 while $i < $tbNum $tb = $tbPrefix . $i sql create table $tb using $mt tags( $i ) - + $x = 0 while $x < $rowNum $cc = $x * 60000 $ms = 1601481600000 + $cc - sql insert into $tb values ($ms , $x ) + sql insert into $tb values ($ms , $x ) $x = $x + 1 - endw - + endw + $i = $i + 1 -endw +endw print =============== step2 $i = 1 @@ -49,51 +49,51 @@ $tb = $tbPrefix . $i sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) print ===> $rows $data01 $data05 -if $rows != $rowNum then +if $rows != $rowNum then return -1 endi -if $data00 != 1 then +if $data00 != 1 then return -1 endi -if $data04 != 1 then +if $data04 != 1 then return -1 endi -#print =============== step3 +print =============== step3 #$cc = 4 * 60000 #$ms = 1601481600000 + $cc #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) #print ===> $rows $data01 $data05 -#if $rows != 5 then +#if $rows != 5 then # return -1 #endi -#if $data00 != 1 then +#if $data00 != 1 then # return -1 #endi -#if $data04 != 1 then +#if $data04 != 1 then # return -1 #endi -#print =============== step4 +print =============== step4 #$cc = 40 * 60000 #$ms = 1601481600000 + $cc #$cc = 1 * 60000 #$ms2 = 1601481600000 - $cc -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> $rows $data01 $data05 -#if $rows != 20 then -# return -1 -#endi -#if $data00 != 1 then -# return -1 -#endi -#if $data04 != 1 then -# return -1 -#endi +sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from $tb interval(1m) +print ===> select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from $tb interval(1m) +print ===> $rows $data01 $data05 +if $rows != $rowNum then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data02 != 60000 then + return -1 +endi #print =============== step5 #$cc = 40 * 60000 @@ -105,13 +105,13 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) #print ===> $rows $data21 $data25 -#if $rows != 42 then +#if $rows != 42 then # return -1 #endi -#if $data20 != 1 then +#if $data20 != 1 then # return -1 #endi -#if $data24 != 1 then +#if $data24 != 1 then # return -1 #endi @@ -119,10 +119,10 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) #print ===> $rows $data11 -#if $rows != 20 then +#if $rows != 20 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi @@ -132,10 +132,10 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) #print ===> $rows $data11 -#if $rows != 5 then +#if $rows != 5 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi @@ -149,10 +149,10 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) #print ===> $rows $data11 -#if $rows != 20 then +#if $rows != 20 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi # @@ -166,18 +166,18 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) #print ===> $rows $data11 -#if $rows != 42 then +#if $rows != 42 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi print =============== clear #sql drop database $db #sql show databases -#if $rows != 0 then +#if $rows != 0 then # return -1 #endi -#system sh/exec.sh -n dnode1 -s stop -x SIGINT +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/testCaseSuite.sim b/tests/script/tsim/testCaseSuite.sim index 0e17ed9141..4245529343 100644 --- a/tests/script/tsim/testCaseSuite.sim +++ b/tests/script/tsim/testCaseSuite.sim @@ -1,6 +1,7 @@ run tsim/user/basic1.sim +run tsim/db/alter_option.sim run tsim/db/basic1.sim run tsim/db/basic2.sim run tsim/db/basic3.sim diff --git a/tests/script/tsim/tmq/multiTopic.sim b/tests/script/tsim/tmq/multiTopic.sim new file mode 100644 index 0000000000..cd977e5909 --- /dev/null +++ b/tests/script/tsim/tmq/multiTopic.sim @@ -0,0 +1,224 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene2 and scene4 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 1 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 1 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== insert data +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + + $binary = ' . binary + $binary = $binary . $c + $binary = $binary . ' + + sql insert into $tb values ($tstart , $c , $x , $binary ) + sql insert into ntb values ($tstart , $c , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$totalMsgCnt = $rowNum * $tbNum +print inserted totalMsgCnt: $totalMsgCnt + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$numOfTopics = 2 +$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCntOfmultiTopics +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 20000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$numOfTopics = 3 +$totalMsgCntOfmultiTopics = $rowNum * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCntOfmultiTopics +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 300, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$numOfTopics = 3 +$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCntOfmultiTopics +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 30000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/oneTopic.sim b/tests/script/tsim/tmq/oneTopic.sim new file mode 100644 index 0000000000..8e8d00977c --- /dev/null +++ b/tests/script/tsim/tmq/oneTopic.sim @@ -0,0 +1,264 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 1 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 1 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== insert data +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + + $binary = ' . binary + $binary = $binary . $c + $binary = $binary . ' + + sql insert into $tb values ($tstart , $c , $x , $binary ) + sql insert into ntb values ($tstart , $c , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$totalMsgCnt = $rowNum * $tbNum +print inserted totalMsgCnt: $totalMsgCnt + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content != $expect_result then +# return -1 +#endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $rowNum +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 100, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 100, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 100, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content != $expect_result then + return -1 +endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 78f1c7e015..38264331c1 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -226,7 +226,7 @@ void loop_consume(tmq_t* tmq) { int32_t totalRows = 0; int32_t skipLogNum = 0; while (running) { - tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 6000); + tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 3000); if (tmqMsg) { totalMsgs++;