From cccd5dbd8bda0ad56d736add278419754a30aefa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 05:28:08 +0000 Subject: [PATCH 001/103] refact data --- include/common/tdataformat.h | 3 + include/common/tmsg.h | 13 ++- source/common/src/tdataformat.c | 165 ++++++++++++++++++++++++-------- 3 files changed, 138 insertions(+), 43 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2e8a214a9d..b01b7ac240 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -38,6 +38,8 @@ typedef struct SKVIdx SKVIdx; // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, + int8_t *flags); // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); @@ -86,6 +88,7 @@ struct STSRowBuilder { uint8_t *pKVBuf; int32_t szTPBuf; uint8_t *pTPBuf; + uint8_t *pBitBuf; int32_t nCols; int32_t kvVLen; int32_t tpVLen; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7f17c1673b..6294e14adb 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -271,12 +271,13 @@ typedef struct { int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); -void tFreeSSubmitRsp(SSubmitRsp *pRsp); - -#define COL_SMA_ON ((int8_t)0x1) -#define COL_IDX_ON ((int8_t)0x2) -#define COL_VAL_SET ((int8_t)0x4) +void tFreeSSubmitRsp(SSubmitRsp* pRsp); +#define COL_SMA_ON ((int8_t)0x1) +#define COL_IDX_ON ((int8_t)0x2) +#define COL_SET_VAL ((int8_t)0x10) +#define COL_SET_NONE ((int8_t)0x20) +#define COL_SET_NULL ((int8_t)0x40) typedef struct SSchema { int8_t type; int8_t flags; @@ -285,6 +286,8 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0) + #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) #define SSCHMEA_TYPE(s) ((s)->type) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad020fe7d9..d492d98b63 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,13 +19,17 @@ #include "tdatablock.h" #include "tlog.h" -#define TD_KV_ROW 0x1U +#define TD_HAS_NONE 0x1U +#define TD_HAS_NULL 0x2U +#define TD_HAS_VAL 0x4U +#define TD_KV_ROW 0x10U struct SKVIdx { int32_t cid; int32_t offset; }; +// STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; @@ -50,6 +54,59 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { return 0; } +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, + int8_t *flags) { + if (cid == 0) { + *ppData = (uint8_t *)&pRow->ts; + *nData = sizeof(TSKEY); + *flags = 0; + } else { + uint32_t tflags = pRow->flags & 0xf; + *ppData = NULL; + *nData = 0; + + switch (tflags) { + case TD_HAS_NONE: + *flags = -1; + break; + case TD_HAS_NULL: + *flags = 1; + break; + case TD_HAS_VAL: + *flags = 0; + // find the row + break; + case TD_HAS_NULL | TD_HAS_NONE: + // read bit map (todo) + if (0) { + *flags = 1; + } else { + *flags = -1; + } + break; + case TD_HAS_VAL | TD_HAS_NONE: + case TD_HAS_VAL | TD_HAS_NULL: + // read bitmap (todo) + if (0) { + if (tflags & TD_HAS_NONE) { + *flags = -1; + } else { + *flags = 1; + } + } else { + // get value (todo) + } + break; + case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: + break; + default: + return -1; + } + } + return 0; +} + +// STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); if (*ppTSchema == NULL) { @@ -87,6 +144,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema * void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } +// STSRowBuilder int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { int32_t kvBufLen; int32_t tpBufLen; @@ -131,7 +189,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) { for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - pBuilder->pTColumn->flags &= (~COL_VAL_SET); + pBuilder->pTColumn->flags &= 0xf; } pBuilder->nCols = 0; @@ -144,7 +202,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD int32_t iCol; uint8_t *p; - // search column + // search column (todo: make here faster) if (pBuilder->pTColumn->colId < cid) { iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { @@ -160,7 +218,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD } // check - if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) { + if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) { return -1; } @@ -168,9 +226,12 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD if (cid == 0) { ASSERT(pData && nData == sizeof(TSKEY)); pBuilder->row.ts = *(TSKEY *)pData; + + pBuilder->pTColumn->flags |= COL_SET_VAL; } else { if (pData) { - // ASSERT(!IS_NULL(pData)); + pBuilder->row.flags |= TD_HAS_VAL; + pBuilder->pTColumn->flags |= COL_SET_VAL; // set tuple data p = pBuilder->pTPBuf + pBuilder->pTColumn->offset; @@ -197,51 +258,79 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD pBuilder->kvVLen += nData; } } else { - // set NULL val + pBuilder->row.flags |= TD_HAS_NULL; + pBuilder->pTColumn->flags |= COL_SET_NULL; + + p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; + ((SKVIdx *)p)->cid = cid; + ((SKVIdx *)p)->offset = -1; } + + pBuilder->nCols++; } - pBuilder->pTColumn->flags |= COL_VAL_SET; - pBuilder->nCols++; return 0; } int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { - if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) { + int32_t tpDataLen, kvDataLen; + uint32_t flags; + + // error not set ts + if (!COL_IS_SET(pBuilder->pTSchema->columns)) { return -1; } - if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) { - // encode as TD_KV_ROW - pBuilder->row.flags |= TD_KV_ROW; - pBuilder->row.ncols = pBuilder->nCols; - pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen; - pBuilder->row.pData = pBuilder->pKVBuf; + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { + pBuilder->row.flags |= TD_HAS_NONE; + } - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { - memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, - pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen); - } - } else { - // encode as TD_TUPLE_ROW - pBuilder->row.flags &= (~TD_KV_ROW); - pBuilder->row.sver = pBuilder->pTSchema->version; - pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->row.pData = pBuilder->pTPBuf; - - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { - // set non-set cols as None - for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pBuilder->pTColumn->flags & COL_VAL_SET) continue; - - { - // set None (todo) - } - - pBuilder->pTColumn->flags |= COL_VAL_SET; + flags = pBuilder->row.flags & 0xf; + switch (flags) { + case TD_HAS_NONE: + case TD_HAS_NULL: + pBuilder->row.sver = pBuilder->pTSchema->version; + pBuilder->row.nData = 0; + pBuilder->row.pData = NULL; + break; + case TD_HAS_VAL: + pBuilder->row.sver = pBuilder->pTSchema->version; + pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; + pBuilder->row.pData = pBuilder->pTPBuf; + break; + case TD_HAS_NULL | TD_HAS_NONE: + pBuilder->row.sver = pBuilder->pTSchema->version; + // set bitmap (todo) + pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1; + pBuilder->row.pData = pBuilder->pBitBuf; + break; + case TD_HAS_VAL | TD_HAS_NONE: + case TD_HAS_VAL | TD_HAS_NULL: + case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: + if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) { + tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; + } else { + tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; } - } + kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen; + + if (kvDataLen < tpDataLen) { + pBuilder->row.flags |= TD_KV_ROW; + pBuilder->row.ncols = pBuilder->nCols; + pBuilder->row.nData = kvDataLen; + pBuilder->row.pData = pBuilder->pKVBuf; + // memmove(); todo + // qsort + } else { + pBuilder->row.sver = pBuilder->pTSchema->numOfCols; + // set bitmap etc (todo) + pBuilder->row.nData = tpDataLen; + pBuilder->row.pData = pBuilder->pTPBuf; + } + break; + default: + ASSERT(0); + return -1; } *ppRow = &pBuilder->row; From 793b49dbf61c29fe6802e7d7658223e7442b94d9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 10:38:35 +0000 Subject: [PATCH 002/103] feat: data format --- include/common/tdataformat.h | 12 +- include/common/tmsg.h | 8 +- source/common/src/tdataformat.c | 194 +++++++++++++++++++------------- 3 files changed, 126 insertions(+), 88 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index b01b7ac240..de7c2a49f6 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -33,9 +33,9 @@ typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; typedef struct SKVIdx SKVIdx; -// STSchema - // STSRow2 +#define TSROW_SVER(r) (((r)->flags & TSROW_KV_ROW) ? -1 : (r)->sver) + int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, @@ -70,9 +70,13 @@ struct STSchema { STColumn columns[]; }; +#define TSROW_HAS_NONE ((uint8_t)0x1) +#define TSROW_HAS_NULL ((uint8_t)0x2U) +#define TSROW_HAS_VAL ((uint8_t)0x4U) +#define TSROW_KV_ROW ((uint8_t)0x10U) struct STSRow2 { - TSKEY ts; - uint32_t flags; + TSKEY ts; + uint8_t flags; union { int32_t sver; int32_t ncols; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8eb9ae62d8..d860ce5b99 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -275,9 +275,8 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SMA_ON ((int8_t)0x1) #define COL_IDX_ON ((int8_t)0x2) -#define COL_SET_VAL ((int8_t)0x10) -#define COL_SET_NONE ((int8_t)0x20) -#define COL_SET_NULL ((int8_t)0x40) +#define COL_SET_NULL ((int8_t)0x10) +#define COL_SET_VAL ((int8_t)0x20) typedef struct SSchema { int8_t type; int8_t flags; @@ -286,7 +285,8 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0) +#define COL_IS_SET(FLG) ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0) +#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d492d98b63..0fed7c86fe 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,38 +19,66 @@ #include "tdatablock.h" #include "tlog.h" -#define TD_HAS_NONE 0x1U -#define TD_HAS_NULL 0x2U -#define TD_HAS_VAL 0x4U -#define TD_KV_ROW 0x10U - struct SKVIdx { int32_t cid; int32_t offset; }; +#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) + // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; - if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; - if (pRow->flags & TD_KV_ROW) { - if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; - } else { - if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; + if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1; + + ASSERT(pRow->flags & 0xf != 0); + + switch (pRow->flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + ASSERT(TSROW_IS_KV_ROW(pRow) && pRow->nData == 0 && pRow->pData == NULL); + return 0; + case TSROW_HAS_VAL: + ASSERT(!TSROW_IS_KV_ROW(pRow)); + default: + ASSERT(pRow->nData && pRow->pData); + if (TSROW_IS_KV_ROW(pRow)) { + if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; + } else { + if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; + } + if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1; + break; } - if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData) < 0) return -1; + return 0; } int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1; - if (tDecodeU32v(pDecoder, &pRow->flags) < 0) return -1; - if (pRow->flags & TD_KV_ROW) { - if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; - } else { - if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; + if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1; + + ASSERT(pRow->flags & 0xf != 0); + + switch (pRow->flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + ASSERT(TSROW_IS_KV_ROW(pRow)); + pRow->nData = 0; + pRow->pData = NULL; + return 0; + case TSROW_HAS_VAL: + ASSERT(!TSROW_IS_KV_ROW(pRow)); + default: + if (TSROW_IS_KV_ROW(pRow)) { + if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; + } else { + if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; + } + if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1; + break; } - if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData) < 0) return -1; + return 0; } @@ -66,17 +94,17 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui *nData = 0; switch (tflags) { - case TD_HAS_NONE: + case TSROW_HAS_NONE: *flags = -1; break; - case TD_HAS_NULL: + case TSROW_HAS_NULL: *flags = 1; break; - case TD_HAS_VAL: + case TSROW_HAS_VAL: *flags = 0; // find the row break; - case TD_HAS_NULL | TD_HAS_NONE: + case TSROW_HAS_NULL | TSROW_HAS_NONE: // read bit map (todo) if (0) { *flags = 1; @@ -84,11 +112,11 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui *flags = -1; } break; - case TD_HAS_VAL | TD_HAS_NONE: - case TD_HAS_VAL | TD_HAS_NULL: + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: // read bitmap (todo) if (0) { - if (tflags & TD_HAS_NONE) { + if (tflags & TSROW_HAS_NONE) { *flags = -1; } else { *flags = 1; @@ -97,7 +125,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui // get value (todo) } break; - case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: break; default: return -1; @@ -199,18 +227,15 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) { } int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { - int32_t iCol; - uint8_t *p; - - // search column (todo: make here faster) + // search column (TODO: bsearch with interp) if (pBuilder->pTColumn->colId < cid) { - iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; + int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; if (pBuilder->pTColumn->colId == cid) break; } } else if (pBuilder->pTColumn->colId > cid) { - iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; + int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; for (; iCol >= 0; iCol--) { pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; if (pBuilder->pTColumn->colId == cid) break; @@ -218,19 +243,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD } // check - if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) { + if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn->flags)) { return -1; } // set value + uint8_t *p; if (cid == 0) { ASSERT(pData && nData == sizeof(TSKEY)); pBuilder->row.ts = *(TSKEY *)pData; pBuilder->pTColumn->flags |= COL_SET_VAL; } else { - if (pData) { - pBuilder->row.flags |= TD_HAS_VAL; + if (pData) { // set val + pBuilder->row.flags |= TSROW_HAS_VAL; pBuilder->pTColumn->flags |= COL_SET_VAL; // set tuple data @@ -250,20 +276,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD ((SKVIdx *)p)->cid = cid; ((SKVIdx *)p)->offset = pBuilder->kvVLen; - p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols + pBuilder->kvVLen; + p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->kvVLen; if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { pBuilder->kvVLen += tPutBinary(p, pData, nData); } else { memcpy(p, pData, nData); pBuilder->kvVLen += nData; } - } else { - pBuilder->row.flags |= TD_HAS_NULL; + } else { // set NULL + pBuilder->row.flags |= TSROW_HAS_NULL; pBuilder->pTColumn->flags |= COL_SET_NULL; p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; ((SKVIdx *)p)->cid = cid; - ((SKVIdx *)p)->offset = -1; + ((SKVIdx *)p)->offset = -1; // for TSROW_KV_ROW, offset -1 means NULL } pBuilder->nCols++; @@ -272,68 +298,76 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD return 0; } +static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) { + SKVIdx *pKVIdx1 = (SKVIdx *)p1; + SKVIdx *pKVIdx2 = (SKVIdx *)p2; + if (pKVIdx1->cid > pKVIdx2->cid) { + return 1; + } else if (pKVIdx1->cid < pKVIdx2->cid) { + return -1; + } + return 0; +} int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { int32_t tpDataLen, kvDataLen; uint32_t flags; // error not set ts - if (!COL_IS_SET(pBuilder->pTSchema->columns)) { + if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) { return -1; } + ASSERT(pBuilder->nCols < pBuilder->pTSchema->numOfCols); if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { - pBuilder->row.flags |= TD_HAS_NONE; + pBuilder->row.flags |= TSROW_HAS_NONE; } - flags = pBuilder->row.flags & 0xf; - switch (flags) { - case TD_HAS_NONE: - case TD_HAS_NULL: - pBuilder->row.sver = pBuilder->pTSchema->version; + ASSERT(pBuilder->row.flags & 0xf != 0); + *(ppRow) = &pBuilder->row; + switch (pBuilder->row.flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + pBuilder->row.flags |= TSROW_KV_ROW; pBuilder->row.nData = 0; pBuilder->row.pData = NULL; + return 0; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + tpDataLen = (pBuilder->pTSchema->numOfCols - 1) / 8; break; - case TD_HAS_VAL: - pBuilder->row.sver = pBuilder->pTSchema->version; - pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->row.pData = pBuilder->pTPBuf; + case TSROW_HAS_VAL: + tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen; break; - case TD_HAS_NULL | TD_HAS_NONE: - pBuilder->row.sver = pBuilder->pTSchema->version; - // set bitmap (todo) - pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1; - pBuilder->row.pData = pBuilder->pBitBuf; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 8; break; - case TD_HAS_VAL | TD_HAS_NONE: - case TD_HAS_VAL | TD_HAS_NULL: - case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: - if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) { - tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; - } else { - tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; - } - kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen; - - if (kvDataLen < tpDataLen) { - pBuilder->row.flags |= TD_KV_ROW; - pBuilder->row.ncols = pBuilder->nCols; - pBuilder->row.nData = kvDataLen; - pBuilder->row.pData = pBuilder->pKVBuf; - // memmove(); todo - // qsort - } else { - pBuilder->row.sver = pBuilder->pTSchema->numOfCols; - // set bitmap etc (todo) - pBuilder->row.nData = tpDataLen; - pBuilder->row.pData = pBuilder->pTPBuf; - } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 4; break; default: - ASSERT(0); - return -1; + // decide chose tuple or kv + kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen; + break; + } + + if (kvDataLen < tpDataLen) { + pBuilder->row.flags |= TSROW_KV_ROW; + pBuilder->row.ncols = pBuilder->nCols; + + pBuilder->row.nData = kvDataLen; + pBuilder->row.pData = pBuilder->pKVBuf; + qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { + memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, + pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->kvVLen); + } + } else { + pBuilder->row.sver = pBuilder->pTSchema->version; + + pBuilder->row.nData = tpDataLen; + pBuilder->row.pData } - *ppRow = &pBuilder->row; return 0; } From 6972b1d6d14ac119589434b64a3cdd43beececa0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 10:41:29 +0000 Subject: [PATCH 003/103] make compile --- source/common/src/tdataformat.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 0fed7c86fe..efc8819435 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -365,7 +365,6 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { pBuilder->row.sver = pBuilder->pTSchema->version; pBuilder->row.nData = tpDataLen; - pBuilder->row.pData } return 0; From 038e9558bbf40a030deb95bc25e18f975d1e5c70 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 May 2022 01:19:34 +0000 Subject: [PATCH 004/103] more data --- include/common/tdataformat.h | 9 +- source/common/src/tdataformat.c | 222 +++++++++++++++++++------------- 2 files changed, 140 insertions(+), 91 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index aed7f6ddc6..ad47b82e7f 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -86,16 +86,17 @@ struct STSRow2 { }; struct STSRowBuilder { - STColumn *pTColumn; STSchema *pTSchema; + int32_t szBitMap1; + int32_t szBitMap2; int32_t szKVBuf; uint8_t *pKVBuf; int32_t szTPBuf; uint8_t *pTPBuf; - uint8_t *pBitBuf; + int32_t iCol; int32_t nCols; - int32_t kvVLen; - int32_t tpVLen; + int32_t vlenKV; + int32_t vlenTP; STSRow2 row; }; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index efc8819435..a951efbe20 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -25,6 +25,8 @@ struct SKVIdx { }; #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) +#define SET_BIT1(p, i, v) +#define SET_BIT2(p, i, v) // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { @@ -170,126 +172,135 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema * return 0; } -void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } +void tTSchemaDestroy(STSchema *pTSchema) { + if (pTSchema) taosMemoryFree(pTSchema); +} // STSRowBuilder -int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { - int32_t kvBufLen; - int32_t tpBufLen; - uint8_t *p; +static int32_t tTSRowBitMapLen1(int32_t nCols) { return nCols / 8 + ((nCols % 8) == 0 ? 0 : 1); } +static int32_t tTSRowBitMapLen2(int32_t nCols) { return nCols / 4 + ((nCols % 4) == 0 ? 0 : 1); } +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; - kvBufLen = sizeof(SKVIdx) * nCols + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; - tpBufLen = pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; - - if (pBuilder->szKVBuf < kvBufLen) { - p = taosMemoryRealloc(pBuilder->pKVBuf, kvBufLen); - if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pBuilder->pKVBuf = p; - pBuilder->szKVBuf = kvBufLen; + pBuilder->szBitMap1 = tTSRowBitMapLen1(nCols - 1); + pBuilder->szBitMap2 = tTSRowBitMapLen2(nCols - 1); + pBuilder->szKVBuf = sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf); + if (pBuilder->pKVBuf == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; } - - if (pBuilder->szTPBuf < tpBufLen) { - p = taosMemoryRealloc(pBuilder->pTPBuf, tpBufLen); - if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pBuilder->pTPBuf = p; - pBuilder->szTPBuf = tpBufLen; + pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf); + if (pBuilder->pTPBuf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - tTSRowBuilderReset(pBuilder); - return 0; } void tTSRowBuilderClear(STSRowBuilder *pBuilder) { - taosMemoryFree(pBuilder->pKVBuf); - taosMemoryFree(pBuilder->pTPBuf); + tTSchemaDestroy(pBuilder->pTSchema); + pBuilder->pTSchema = NULL; + if (pBuilder->pKVBuf) { + taosMemoryFree(pBuilder->pKVBuf); + pBuilder->pKVBuf = NULL; + } + if (pBuilder->pTPBuf) { + taosMemoryFree(pBuilder->pTPBuf); + pBuilder->pTPBuf = NULL; + } } void tTSRowBuilderReset(STSRowBuilder *pBuilder) { for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - - pBuilder->pTColumn->flags &= 0xf; + STColumn *pTColumn = &pBuilder->pTSchema->columns[iCol]; + COL_CLR_SET(pTColumn->flags); } + pBuilder->iCol = 0; pBuilder->nCols = 0; - pBuilder->kvVLen = 0; - pBuilder->tpVLen = 0; + pBuilder->vlenKV = 0; + pBuilder->vlenTP = 0; pBuilder->row.flags = 0; } int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { - // search column (TODO: bsearch with interp) - if (pBuilder->pTColumn->colId < cid) { - int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; - for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pBuilder->pTColumn->colId == cid) break; + STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol]; + uint8_t *p; + int32_t iCol; + + // use interp search (todo) + if (pTColumn->colId > cid) { + for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pTColumn->colId == cid) break; } - } else if (pBuilder->pTColumn->colId > cid) { - int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; - for (; iCol >= 0; iCol--) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pBuilder->pTColumn->colId == cid) break; + } else if (pTColumn->colId < cid) { + for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) { + pTColumn = &pBuilder->pTSchema->columns[iCol]; + if (pTColumn->colId == cid) break; } } - // check - if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn->flags)) { + if (pTColumn->colId != cid || COL_IS_SET(pTColumn->flags)) { return -1; } + pBuilder->iCol = iCol; + // set value - uint8_t *p; if (cid == 0) { - ASSERT(pData && nData == sizeof(TSKEY)); + ASSERT(pData && nData == sizeof(TSKEY) && iCol == 0); pBuilder->row.ts = *(TSKEY *)pData; - - pBuilder->pTColumn->flags |= COL_SET_VAL; + pTColumn->flags |= COL_SET_VAL; } else { - if (pData) { // set val + if (pData) { + // set VAL + pBuilder->row.flags |= TSROW_HAS_VAL; - pBuilder->pTColumn->flags |= COL_SET_VAL; + pTColumn->flags |= COL_SET_VAL; - // set tuple data - p = pBuilder->pTPBuf + pBuilder->pTColumn->offset; - if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { - *(int32_t *)p = pBuilder->tpVLen; + /* KV */ + if (1) { // avoid KV at some threshold (todo) + p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; + ((SKVIdx *)p)->cid = cid; + ((SKVIdx *)p)->offset = pBuilder->vlenKV; - // encode the variant-length data - p = pBuilder->pTPBuf + pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->tpVLen += tPutBinary(p, pData, nData); - } else { - memcpy(p, pData, nData); + p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->vlenKV; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ASSERT(nData <= pTColumn->bytes); + pBuilder->vlenKV += tPutBinary(p, pData, nData); + } else { + ASSERT(nData == pTColumn->bytes); + memcpy(p, pData, nData); + pBuilder->vlenKV += nData; + } } - // set kv data - p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; - ((SKVIdx *)p)->cid = cid; - ((SKVIdx *)p)->offset = pBuilder->kvVLen; + /* TUPLE */ + p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pTColumn->offset; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ASSERT(nData <= pTColumn->bytes); + *(int32_t *)p = pBuilder->vlenTP; - p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->kvVLen; - if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { - pBuilder->kvVLen += tPutBinary(p, pData, nData); + p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen; + pBuilder->vlenTP += tPutBinary(p, pData, nData); } else { + ASSERT(nData == pTColumn->bytes); memcpy(p, pData, nData); - pBuilder->kvVLen += nData; } - } else { // set NULL + } else { + // set NULL + pBuilder->row.flags |= TSROW_HAS_NULL; - pBuilder->pTColumn->flags |= COL_SET_NULL; + pTColumn->flags |= COL_SET_NULL; p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; ((SKVIdx *)p)->cid = cid; - ((SKVIdx *)p)->offset = -1; // for TSROW_KV_ROW, offset -1 means NULL + ((SKVIdx *)p)->offset = -1; } pBuilder->nCols++; @@ -309,7 +320,7 @@ static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) { return 0; } int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { - int32_t tpDataLen, kvDataLen; + int32_t nDataTP, nDataKV; uint32_t flags; // error not set ts @@ -332,39 +343,76 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { pBuilder->row.pData = NULL; return 0; case TSROW_HAS_NULL | TSROW_HAS_NONE: - tpDataLen = (pBuilder->pTSchema->numOfCols - 1) / 8; + nDataTP = pBuilder->szBitMap1; break; case TSROW_HAS_VAL: - tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen; + nDataTP = pBuilder->pTSchema->flen + pBuilder->vlenTP; break; case TSROW_HAS_VAL | TSROW_HAS_NONE: case TSROW_HAS_VAL | TSROW_HAS_NULL: - tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 8; + nDataTP = pBuilder->szBitMap1 + pBuilder->pTSchema->flen + pBuilder->vlenTP; break; case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 4; + nDataTP = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP; break; default: - // decide chose tuple or kv - kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen; - break; + ASSERT(0); } - if (kvDataLen < tpDataLen) { + nDataKV = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->vlenKV; + ASSERT(pBuilder->row.flags & TSROW_KV_ROW == 0); + if (nDataKV < nDataTP) { + // generate KV row + pBuilder->row.flags |= TSROW_KV_ROW; pBuilder->row.ncols = pBuilder->nCols; - - pBuilder->row.nData = kvDataLen; + pBuilder->row.nData = nDataKV; pBuilder->row.pData = pBuilder->pKVBuf; + qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, - pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->kvVLen); + pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->vlenKV); } } else { - pBuilder->row.sver = pBuilder->pTSchema->version; + // generate TUPLE row - pBuilder->row.nData = tpDataLen; + uint8_t *p; + STColumn *pTColumn; + pBuilder->row.sver = pBuilder->pTSchema->version; + pBuilder->row.nData = nDataTP; + if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL) { + // no bitmap + pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2; + } else if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) { + // bitmap2 + p = pBuilder->pTPBuf; + + for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pTColumn = &pBuilder->pTSchema->columns[iCol]; + + if (pTColumn->flags & COL_SET_VAL) { + SET_BIT2(p, iCol - 1, 0x2); + } else if (pTColumn->flags & COL_SET_VAL) { + SET_BIT2(p, iCol - 1, 0x1); + } + } + + pBuilder->row.pData = p; + } else { + // bitmap1 + p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; + + for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { + pTColumn = &pBuilder->pTSchema->columns[iCol]; + + if (1) { + SET_BIT1(p, iCol - 1, 0x1); + } + } + + pBuilder->row.pData = p; + } } return 0; From e9ffb086a5443082e2f768dce2dac95352838c81 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 May 2022 15:36:26 +0000 Subject: [PATCH 005/103] more data format --- include/common/tdataformat.h | 12 +-- source/common/src/tdataformat.c | 171 +++++++++++++++++--------------- 2 files changed, 96 insertions(+), 87 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ad47b82e7f..b8a4f6ac9d 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -34,8 +34,6 @@ typedef struct STSRowBuilder STSRowBuilder; typedef struct SKVIdx SKVIdx; // STSRow2 -#define TSROW_SVER(r) (((r)->flags & TSROW_KV_ROW) ? -1 : (r)->sver) - int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, @@ -75,12 +73,9 @@ struct STSchema { #define TSROW_HAS_VAL ((uint8_t)0x4U) #define TSROW_KV_ROW ((uint8_t)0x10U) struct STSRow2 { - TSKEY ts; - uint8_t flags; - union { - int32_t sver; - int32_t ncols; - }; + TSKEY ts; + uint8_t flags; + int32_t sver; uint32_t nData; const uint8_t *pData; }; @@ -94,7 +89,6 @@ struct STSRowBuilder { int32_t szTPBuf; uint8_t *pTPBuf; int32_t iCol; - int32_t nCols; int32_t vlenKV; int32_t vlenTP; STSRow2 row; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a951efbe20..055eb9df80 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -24,31 +24,33 @@ struct SKVIdx { int32_t offset; }; +#pragma pack(push, 1) +typedef struct { + int16_t nCols; + SKVIdx idx[]; +} STSKVRow; +#pragma pack(pop) + #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) -#define SET_BIT1(p, i, v) -#define SET_BIT2(p, i, v) +#define SET_BIT1(p, i, v) ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8))) +#define SET_BIT2(p, i, v) ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4))) // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1; + if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; ASSERT(pRow->flags & 0xf != 0); switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: - ASSERT(TSROW_IS_KV_ROW(pRow) && pRow->nData == 0 && pRow->pData == NULL); return 0; case TSROW_HAS_VAL: ASSERT(!TSROW_IS_KV_ROW(pRow)); default: ASSERT(pRow->nData && pRow->pData); - if (TSROW_IS_KV_ROW(pRow)) { - if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; - } else { - if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; - } if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1; break; } @@ -59,24 +61,19 @@ int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1; if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1; + if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; ASSERT(pRow->flags & 0xf != 0); switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: - ASSERT(TSROW_IS_KV_ROW(pRow)); pRow->nData = 0; pRow->pData = NULL; return 0; case TSROW_HAS_VAL: ASSERT(!TSROW_IS_KV_ROW(pRow)); default: - if (TSROW_IS_KV_ROW(pRow)) { - if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; - } else { - if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; - } if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1; break; } @@ -86,6 +83,7 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, int8_t *flags) { +#if 0 if (cid == 0) { *ppData = (uint8_t *)&pRow->ts; *nData = sizeof(TSKEY); @@ -133,6 +131,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui return -1; } } +#endif return 0; } @@ -177,24 +176,25 @@ void tTSchemaDestroy(STSchema *pTSchema) { } // STSRowBuilder -static int32_t tTSRowBitMapLen1(int32_t nCols) { return nCols / 8 + ((nCols % 8) == 0 ? 0 : 1); } -static int32_t tTSRowBitMapLen2(int32_t nCols) { return nCols / 4 + ((nCols % 4) == 0 ? 0 : 1); } - int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; - pBuilder->szBitMap1 = tTSRowBitMapLen1(nCols - 1); - pBuilder->szBitMap2 = tTSRowBitMapLen2(nCols - 1); - pBuilder->szKVBuf = sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; + pBuilder->szBitMap1 = (nCols - 2) / 8 + 1; + pBuilder->szBitMap2 = (nCols - 2) / 4 + 1; + pBuilder->szKVBuf = + sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf); if (pBuilder->pKVBuf == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + tTSchemaDestroy(pBuilder->pTSchema); return -1; } pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf); if (pBuilder->pTPBuf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pBuilder->pKVBuf); + tTSchemaDestroy(pBuilder->pTSchema); return -1; } @@ -202,16 +202,16 @@ int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchem } void tTSRowBuilderClear(STSRowBuilder *pBuilder) { - tTSchemaDestroy(pBuilder->pTSchema); - pBuilder->pTSchema = NULL; - if (pBuilder->pKVBuf) { - taosMemoryFree(pBuilder->pKVBuf); - pBuilder->pKVBuf = NULL; - } if (pBuilder->pTPBuf) { taosMemoryFree(pBuilder->pTPBuf); pBuilder->pTPBuf = NULL; } + if (pBuilder->pKVBuf) { + taosMemoryFree(pBuilder->pKVBuf); + pBuilder->pKVBuf = NULL; + } + tTSchemaDestroy(pBuilder->pTSchema); + pBuilder->pTSchema = NULL; } void tTSRowBuilderReset(STSRowBuilder *pBuilder) { @@ -221,7 +221,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) { } pBuilder->iCol = 0; - pBuilder->nCols = 0; + ((STSKVRow *)pBuilder->pKVBuf)->nCols = 0; pBuilder->vlenKV = 0; pBuilder->vlenTP = 0; pBuilder->row.flags = 0; @@ -231,6 +231,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol]; uint8_t *p; int32_t iCol; + STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; // use interp search (todo) if (pTColumn->colId > cid) { @@ -265,11 +266,11 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD /* KV */ if (1) { // avoid KV at some threshold (todo) - p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; - ((SKVIdx *)p)->cid = cid; - ((SKVIdx *)p)->offset = pBuilder->vlenKV; + pTSKVRow->idx[pTSKVRow->nCols].cid = cid; + pTSKVRow->idx[pTSKVRow->nCols].offset = pBuilder->vlenKV; - p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->vlenKV; + p = pBuilder->pKVBuf + sizeof(STSKVRow) + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + + pBuilder->vlenKV; if (IS_VAR_DATA_TYPE(pTColumn->type)) { ASSERT(nData <= pTColumn->bytes); pBuilder->vlenKV += tPutBinary(p, pData, nData); @@ -286,7 +287,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD ASSERT(nData <= pTColumn->bytes); *(int32_t *)p = pBuilder->vlenTP; - p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen; + p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP; pBuilder->vlenTP += tPutBinary(p, pData, nData); } else { ASSERT(nData == pTColumn->bytes); @@ -298,12 +299,11 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD pBuilder->row.flags |= TSROW_HAS_NULL; pTColumn->flags |= COL_SET_NULL; - p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; - ((SKVIdx *)p)->cid = cid; - ((SKVIdx *)p)->offset = -1; + pTSKVRow->idx[pTSKVRow->nCols].cid = cid; + pTSKVRow->idx[pTSKVRow->nCols].offset = -1; } - pBuilder->nCols++; + pTSKVRow->nCols++; } return 0; @@ -319,17 +319,54 @@ static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) { } return 0; } +static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) { + int32_t bidx; + STColumn *pTColumn; + + for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { + pTColumn = &pTSchema->columns[iCol]; + bidx = iCol - 1; + + switch (flags) { + case TSROW_HAS_NULL | TSROW_HAS_NONE: + if (pTColumn->flags & COL_SET_NULL) { + SET_BIT1(p, bidx, (uint8_t)1); + } else { + SET_BIT1(p, bidx, (uint8_t)0); + } + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + if (pTColumn->flags & COL_SET_NULL) { + SET_BIT2(p, bidx, (uint8_t)1); + } else if (pTColumn->flags & COL_SET_NULL) { + SET_BIT2(p, bidx, (uint8_t)2); + } else { + SET_BIT2(p, bidx, (uint8_t)0); + } + break; + default: + if (pTColumn->flags & COL_SET_VAL) { + SET_BIT1(p, bidx, (uint8_t)1); + } else { + SET_BIT1(p, bidx, (uint8_t)0); + } + + break; + } + } +} int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { - int32_t nDataTP, nDataKV; - uint32_t flags; + int32_t nDataTP, nDataKV; + uint32_t flags; + STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; // error not set ts if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) { return -1; } - ASSERT(pBuilder->nCols < pBuilder->pTSchema->numOfCols); - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { + ASSERT(pTSKVRow->nCols < pBuilder->pTSchema->numOfCols); + if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) { pBuilder->row.flags |= TSROW_HAS_NONE; } @@ -338,7 +375,6 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { switch (pBuilder->row.flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: - pBuilder->row.flags |= TSROW_KV_ROW; pBuilder->row.nData = 0; pBuilder->row.pData = NULL; return 0; @@ -359,58 +395,37 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { ASSERT(0); } - nDataKV = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->vlenKV; - ASSERT(pBuilder->row.flags & TSROW_KV_ROW == 0); + nDataKV = sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pBuilder->vlenKV; + pBuilder->row.sver = pBuilder->pTSchema->version; if (nDataKV < nDataTP) { // generate KV row pBuilder->row.flags |= TSROW_KV_ROW; - pBuilder->row.ncols = pBuilder->nCols; pBuilder->row.nData = nDataKV; pBuilder->row.pData = pBuilder->pKVBuf; - qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { - memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, - pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->vlenKV); + qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); + if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) { + memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[pBuilder->pTSchema->numOfCols - 1], pBuilder->vlenKV); } } else { // generate TUPLE row - uint8_t *p; - STColumn *pTColumn; - pBuilder->row.sver = pBuilder->pTSchema->version; pBuilder->row.nData = nDataTP; - if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL) { - // no bitmap + + uint8_t *p; + uint8_t flags = pBuilder->row.flags & 0xf; + + if (flags == TSROW_HAS_VAL) { pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2; - } else if (pBuilder->row.flags & 0xf == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) { - // bitmap2 - p = pBuilder->pTPBuf; - - for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pTColumn = &pBuilder->pTSchema->columns[iCol]; - - if (pTColumn->flags & COL_SET_VAL) { - SET_BIT2(p, iCol - 1, 0x2); - } else if (pTColumn->flags & COL_SET_VAL) { - SET_BIT2(p, iCol - 1, 0x1); - } - } - - pBuilder->row.pData = p; } else { - // bitmap1 - p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; - - for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pTColumn = &pBuilder->pTSchema->columns[iCol]; - - if (1) { - SET_BIT1(p, iCol - 1, 0x1); - } + if (flags == TSROW_HAS_VAL) { + p = pBuilder->pTPBuf; + } else { + p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; } + setBitMap(p, pBuilder->pTSchema, flags); pBuilder->row.pData = p; } } From 34a219b5fc5589ed45bbc6434d0138edb5e35233 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 May 2022 06:18:07 +0000 Subject: [PATCH 006/103] feat: impl tsrow read --- include/common/tdataformat.h | 52 +++++++--- source/common/src/tdataformat.c | 163 +++++++++++++++++++++----------- 2 files changed, 150 insertions(+), 65 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index b8a4f6ac9d..ab0415d5d8 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -29,20 +29,41 @@ extern "C" { typedef struct SSchema SSchema; typedef struct STColumn STColumn; typedef struct STSchema STSchema; +typedef struct SColVal SColVal; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; -typedef struct SKVIdx SKVIdx; - -// STSRow2 -int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); -int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); -int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, - int8_t *flags); // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); +// SColVal +#define COL_VAL_SET_NONE(CV) \ + do { \ + (CV)->type = COL_NONE; \ + (CV)->nData = 0; \ + (CV)->pData = NULL; \ + } while (0) + +#define COL_VAL_SET_NULL(CV) \ + do { \ + (CV)->type = COL_NULL; \ + (CV)->nData = 0; \ + (CV)->pData = NULL; \ + } while (0) + +#define COL_VAL_SET_VAL(CV, PDATA, NDATA) \ + do { \ + (CV)->type = COL_VAL; \ + (CV)->nData = (NDATA); \ + (CV)->pData = (PDATA); \ + } while (0) + +// STSRow2 +int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); +int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); + // STSRowBuilder int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols); void tTSRowBuilderClear(STSRowBuilder *pBuilder); @@ -73,11 +94,11 @@ struct STSchema { #define TSROW_HAS_VAL ((uint8_t)0x4U) #define TSROW_KV_ROW ((uint8_t)0x10U) struct STSRow2 { - TSKEY ts; - uint8_t flags; - int32_t sver; - uint32_t nData; - const uint8_t *pData; + TSKEY ts; + uint8_t flags; + int32_t sver; + uint32_t nData; + uint8_t *pData; }; struct STSRowBuilder { @@ -94,6 +115,13 @@ struct STSRowBuilder { STSRow2 row; }; +typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT; +struct SColVal { + EColValT type; + uint32_t nData; + uint8_t *pData; +}; + #if 1 //==================================== // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 055eb9df80..4b1df237da 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,10 +19,10 @@ #include "tdatablock.h" #include "tlog.h" -struct SKVIdx { +typedef struct SKVIdx { int32_t cid; int32_t offset; -}; +} SKVIdx; #pragma pack(push, 1) typedef struct { @@ -34,6 +34,8 @@ typedef struct { #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) #define SET_BIT1(p, i, v) ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8))) #define SET_BIT2(p, i, v) ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4))) +#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) +#define GET_BIT2(p, i) (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3)) // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { @@ -81,57 +83,112 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { return 0; } -int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, - int8_t *flags) { -#if 0 - if (cid == 0) { - *ppData = (uint8_t *)&pRow->ts; - *nData = sizeof(TSKEY); - *flags = 0; - } else { - uint32_t tflags = pRow->flags & 0xf; - *ppData = NULL; - *nData = 0; +static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) { + col_id_t cid = *(col_id_t *)p1; + SKVIdx *pKVIdx = (SKVIdx *)p2; - switch (tflags) { - case TSROW_HAS_NONE: - *flags = -1; - break; - case TSROW_HAS_NULL: - *flags = 1; - break; - case TSROW_HAS_VAL: - *flags = 0; - // find the row - break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - // read bit map (todo) - if (0) { - *flags = 1; - } else { - *flags = -1; - } - break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - // read bitmap (todo) - if (0) { - if (tflags & TSROW_HAS_NONE) { - *flags = -1; - } else { - *flags = 1; - } - } else { - // get value (todo) - } - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - break; - default: - return -1; - } + if (cid < pKVIdx->cid) { + return -1; + } else if (cid > pKVIdx->cid) { + return 1; } -#endif + return 0; +} + +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { + uint32_t n; + uint8_t *p; + uint8_t v; + int32_t bidx = iCol - 1; + STColumn *pTColumn = &pTSchema->columns[iCol]; + STSKVRow *pTSKVRow; + SKVIdx *pKVIdx; + + ASSERT(pTColumn->colId != 0); + + ASSERT(pRow->flags & 0xf != 0); + switch (pRow->flags & 0xf) { + case TSROW_HAS_NONE: + COL_VAL_SET_NONE(pColVal); + return 0; + case TSROW_HAS_NULL: + COL_VAL_SET_NULL(pColVal); + return 0; + } + + if (TSROW_IS_KV_ROW(pRow)) { + ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL); + + pTSKVRow = (STSKVRow *)pRow->pData; + pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn); + if (pKVIdx == NULL) { + COL_VAL_SET_NONE(pColVal); + } else if (pKVIdx->offset < 0) { + COL_VAL_SET_NULL(pColVal); + } else { + p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset; + tGetBinary(p, &p, &n); + COL_VAL_SET_VAL(pColVal, p, n); + } + } else { + // get bitmap + switch (pRow->flags & 0xf) { + p = pRow->pData; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + v = GET_BIT1(p, bidx); + if (v == 0) { + COL_VAL_SET_NONE(pColVal); + } else { + COL_VAL_SET_NULL(pColVal); + } + return 0; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + v = GET_BIT1(p, bidx); + if (v == 1) { + p = p + (pTSchema->numOfCols - 2) / 8 + 1; + break; + } else { + COL_VAL_SET_NONE(pColVal); + return 0; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL: + v = GET_BIT1(p, bidx); + if (v == 1) { + p = p + (pTSchema->numOfCols - 2) / 8 + 1; + break; + } else { + COL_VAL_SET_NULL(pColVal); + return 0; + } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + v = GET_BIT2(p, bidx); + if (v == 0) { + COL_VAL_SET_NONE(pColVal); + return 0; + } else if (v == 1) { + COL_VAL_SET_NULL(pColVal); + return 0; + } else if (v == 2) { + p = p + (pTSchema->numOfCols - 2) / 4 + 1; + break; + } else { + ASSERT(0); + } + default: + break; + } + + // get real value + p = p + pTColumn->offset; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + p = pTSchema->flen + *(int32_t *)p; + tGetBinary(p, &p, &n); + } else { + n = pTColumn->bytes; + } + COL_VAL_SET_VAL(pColVal, p, n); + } + return 0; } @@ -445,8 +502,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += (int)nBitmapBytes; // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered. - // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove - // the additional space + // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus + // remove the additional space #endif if (pCol->spaceSize < spaceNeeded) { From acf1b14b21e1612097b4f3fd294f9cc7842ed218 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 May 2022 01:47:35 +0000 Subject: [PATCH 007/103] fix compile problem --- include/common/tdataformat.h | 16 ++++++++-------- source/common/src/tdataformat.c | 19 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ab0415d5d8..1a31b384d8 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -94,11 +94,11 @@ struct STSchema { #define TSROW_HAS_VAL ((uint8_t)0x4U) #define TSROW_KV_ROW ((uint8_t)0x10U) struct STSRow2 { - TSKEY ts; - uint8_t flags; - int32_t sver; - uint32_t nData; - uint8_t *pData; + TSKEY ts; + uint8_t flags; + int32_t sver; + uint32_t nData; + const uint8_t *pData; }; struct STSRowBuilder { @@ -117,9 +117,9 @@ struct STSRowBuilder { typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT; struct SColVal { - EColValT type; - uint32_t nData; - uint8_t *pData; + EColValT type; + uint32_t nData; + const uint8_t *pData; }; #if 1 //==================================== diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 4b1df237da..d421c452ca 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -96,13 +96,13 @@ static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) { } int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - uint32_t n; - uint8_t *p; - uint8_t v; - int32_t bidx = iCol - 1; - STColumn *pTColumn = &pTSchema->columns[iCol]; - STSKVRow *pTSKVRow; - SKVIdx *pKVIdx; + uint32_t n; + const uint8_t *p; + uint8_t v; + int32_t bidx = iCol - 1; + STColumn *pTColumn = &pTSchema->columns[iCol]; + STSKVRow *pTSKVRow; + SKVIdx *pKVIdx; ASSERT(pTColumn->colId != 0); @@ -132,8 +132,8 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal } } else { // get bitmap + p = pRow->pData; switch (pRow->flags & 0xf) { - p = pRow->pData; case TSROW_HAS_NULL | TSROW_HAS_NONE: v = GET_BIT1(p, bidx); if (v == 0) { @@ -181,8 +181,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal // get real value p = p + pTColumn->offset; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - p = pTSchema->flen + *(int32_t *)p; - tGetBinary(p, &p, &n); + tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); } else { n = pTColumn->bytes; } From de5828e4856c8d02251ea1b6c8598cd2113d7c11 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 May 2022 11:35:37 +0000 Subject: [PATCH 008/103] refact more --- source/common/test/dataformatTest.cpp | 0 source/common/type/type.c | 14 -------------- source/common/type/typeBigint.c | 14 -------------- source/common/type/typeBinary.c | 14 -------------- source/common/type/typeBlob.c | 14 -------------- source/common/type/typeBool.c | 14 -------------- source/common/type/typeDecimal.c | 14 -------------- source/common/type/typeDouble.c | 14 -------------- source/common/type/typeFloat.c | 14 -------------- source/common/type/typeInt.c | 14 -------------- source/common/type/typeJson.c | 14 -------------- source/common/type/typeLongblob.c | 14 -------------- source/common/type/typeNchar.c | 14 -------------- source/common/type/typeNull.c | 14 -------------- source/common/type/typeSmallint.c | 14 -------------- source/common/type/typeTimestamp.c | 14 -------------- source/common/type/typeTinyint.c | 14 -------------- source/common/type/typeUBigint.c | 14 -------------- source/common/type/typeUSmallint.c | 14 -------------- source/common/type/typeUTinyint.c | 14 -------------- source/common/type/typeUint.c | 14 -------------- source/common/type/typeVarchar.c | 14 -------------- 22 files changed, 294 deletions(-) create mode 100644 source/common/test/dataformatTest.cpp delete mode 100644 source/common/type/type.c delete mode 100644 source/common/type/typeBigint.c delete mode 100644 source/common/type/typeBinary.c delete mode 100644 source/common/type/typeBlob.c delete mode 100644 source/common/type/typeBool.c delete mode 100644 source/common/type/typeDecimal.c delete mode 100644 source/common/type/typeDouble.c delete mode 100644 source/common/type/typeFloat.c delete mode 100644 source/common/type/typeInt.c delete mode 100644 source/common/type/typeJson.c delete mode 100644 source/common/type/typeLongblob.c delete mode 100644 source/common/type/typeNchar.c delete mode 100644 source/common/type/typeNull.c delete mode 100644 source/common/type/typeSmallint.c delete mode 100644 source/common/type/typeTimestamp.c delete mode 100644 source/common/type/typeTinyint.c delete mode 100644 source/common/type/typeUBigint.c delete mode 100644 source/common/type/typeUSmallint.c delete mode 100644 source/common/type/typeUTinyint.c delete mode 100644 source/common/type/typeUint.c delete mode 100644 source/common/type/typeVarchar.c diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/common/type/type.c b/source/common/type/type.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/type.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBigint.c b/source/common/type/typeBigint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBigint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBinary.c b/source/common/type/typeBinary.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBinary.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBlob.c b/source/common/type/typeBlob.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBlob.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeBool.c b/source/common/type/typeBool.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeBool.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeDecimal.c b/source/common/type/typeDecimal.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeDecimal.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeDouble.c b/source/common/type/typeDouble.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeDouble.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeFloat.c b/source/common/type/typeFloat.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeFloat.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeInt.c b/source/common/type/typeInt.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeInt.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeJson.c b/source/common/type/typeJson.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeJson.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeLongblob.c b/source/common/type/typeLongblob.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeLongblob.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeNchar.c b/source/common/type/typeNchar.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeNchar.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeNull.c b/source/common/type/typeNull.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeNull.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeSmallint.c b/source/common/type/typeSmallint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeSmallint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeTimestamp.c b/source/common/type/typeTimestamp.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeTimestamp.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeTinyint.c b/source/common/type/typeTinyint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeTinyint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUBigint.c b/source/common/type/typeUBigint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUBigint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUSmallint.c b/source/common/type/typeUSmallint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUSmallint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUTinyint.c b/source/common/type/typeUTinyint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUTinyint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeUint.c b/source/common/type/typeUint.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeUint.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/common/type/typeVarchar.c b/source/common/type/typeVarchar.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/common/type/typeVarchar.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file From 96eecc4d3ee21be96f8641b6fd9454acdcd178ed Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 May 2022 11:43:48 +0000 Subject: [PATCH 009/103] more refact --- source/common/test/CMakeLists.txt | 9 +++++++++ source/common/test/dataformatTest.cpp | 1 + source/common/test/trowTest.cpp | 23 ----------------------- 3 files changed, 10 insertions(+), 23 deletions(-) delete mode 100644 source/common/test/trowTest.cpp diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index a0406e099c..0535b08be7 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -17,6 +17,15 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/libs/common/inc" ) +# dataformatTest.cpp +add_executable(dataformatTest "") +target_sources( + dataformatTest + PRIVATE + "dataformatTest.cpp" +) +target_link_libraries(dataformatTest gtest gtest_main util) + # tmsg test # add_executable(tmsgTest "") # target_sources(tmsgTest diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index e69de29bb2..3497014c22 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -0,0 +1 @@ +#include "gtest/gtest.h" \ No newline at end of file diff --git a/source/common/test/trowTest.cpp b/source/common/test/trowTest.cpp deleted file mode 100644 index d7f0783d4a..0000000000 --- a/source/common/test/trowTest.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -#include "trow.h" - -TEST(td_row_test, build_row_to_target) { -#if 0 - char dst[1024]; - SRow* pRow = (SRow*)dst; - int ncols = 10; - col_id_t cid; - void* pData; - SRowBuilder rb = trbInit(TD_OR_ROW_BUILDER, NULL, 0, pRow, 1024); - - trbSetRowInfo(&rb, false, 0); - trbSetRowTS(&rb, 1637550210000); - for (int c = 0; c < ncols; c++) { - cid = c; - if (trbWriteCol(&rb, pData, cid) < 0) { - // TODO - } - } -#endif -} \ No newline at end of file From 93c095b1a56e515be082a7b5ff1b6fed4aa9f58d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 17 May 2022 05:46:22 +0000 Subject: [PATCH 010/103] make compile --- include/common/tdataformat.h | 4 +- include/util/tencode.h | 205 ++++++++++++++++++++++++++------ source/common/src/tdataformat.c | 6 +- 3 files changed, 177 insertions(+), 38 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 1a31b384d8..1097f62798 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -62,13 +62,15 @@ void tTSchemaDestroy(STSchema *pTSchema); // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); +int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols); void tTSRowBuilderClear(STSRowBuilder *pBuilder); void tTSRowBuilderReset(STSRowBuilder *pBuilder); -int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData); +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); // STRUCT ================= diff --git a/include/util/tencode.h b/include/util/tencode.h index e49429f865..484c9443f7 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -456,52 +456,189 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { return p; } -static FORCE_INLINE int32_t tPutBinary(uint8_t* p, const uint8_t* pData, uint32_t nData) { - int n = 0; - uint32_t v = nData; +// =========================================== +#define tPutV(p, v) \ + do { \ + int32_t n = 0; \ + for (;;) { \ + if (v <= 0x7f) { \ + if (p) p[n] = v; \ + n++; \ + break; \ + } \ + if (p) p[n] = (v & 0x7f) | 0x80; \ + n++; \ + v >>= 7; \ + } \ + return n; \ + } while (0) - for (;;) { - if (v <= 0x7f) { - if (p) p[n] = v; - n++; - break; - } +#define tGetV(p, v) \ + do { \ + int32_t n = 0; \ + if (v) *v = 0; \ + for (;;) { \ + if (p[n] <= 0x7f) { \ + if (v) (*v) |= (p[n] << (7 * n)); \ + n++; \ + break; \ + } \ + if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ + n++; \ + } \ + return n; \ + } while (0) - if (p) p[n] = (v & 0x7f) | 0x80; - n++; - v >>= 7; - } +// PUT +static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { + if (p) ((uint8_t*)p)[0] = v; + return sizeof(uint8_t); +} - if (p) { - memcpy(p + n, pData, nData); - } +static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { + if (p) ((int8_t*)p)[0] = v; + return sizeof(int8_t); +} + +static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) { + if (p) ((uint16_t*)p)[0] = v; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) { + if (p) ((int16_t*)p)[0] = v; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) { + if (p) ((uint32_t*)p)[0] = v; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) { + if (p) ((int32_t*)p)[0] = v; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) { + if (p) ((uint64_t*)p)[0] = v; + return sizeof(uint64_t); +} + +static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { + if (p) ((int64_t*)p)[0] = v; + return sizeof(int64_t); +} + +static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } + +static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } + +static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } + +static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } + +// GET +static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { + if (v) *v = ((uint8_t*)p)[0]; + return sizeof(uint8_t); +} + +static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { + if (v) *v = ((int8_t*)p)[0]; + return sizeof(int8_t); +} + +static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) { + if (v) *v = ((uint16_t*)p)[0]; + return sizeof(uint16_t); +} + +static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) { + if (v) *v = ((int16_t*)p)[0]; + return sizeof(int16_t); +} + +static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) { + if (v) *v = ((uint32_t*)p)[0]; + return sizeof(uint32_t); +} + +static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) { + if (v) *v = ((int32_t*)p)[0]; + return sizeof(int32_t); +} + +static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) { + if (v) *v = ((uint64_t*)p)[0]; + return sizeof(uint64_t); +} + +static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { + if (v) *v = ((int64_t*)p)[0]; + return sizeof(int64_t); +} + +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { + int32_t n; + uint16_t tv; + + n = tGetU16v(p, &tv); + if (v) *v = ZIGZAGD(int16_t, tv); + + return n; +} + +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { + int32_t n; + uint32_t tv; + + n = tGetU32v(p, &tv); + if (v) *v = ZIGZAGD(int32_t, tv); + + return n; +} + +static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); } + +static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { + int32_t n; + uint64_t tv; + + n = tGetU64v(p, &tv); + if (v) *v = ZIGZAGD(int64_t, tv); + + return n; +} + +// ===================== +static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { + int n = 0; + + n += tPutU32v(p ? p + n : p, nData); + if (p) memcpy(p + n, pData, nData); n += nData; return n; } -static FORCE_INLINE int32_t tGetBinary(const uint8_t* p, const uint8_t** ppData, uint32_t* nData) { +static FORCE_INLINE int32_t tGetBinary(uint8_t* p, const uint8_t** ppData, uint32_t* nData) { int32_t n = 0; - uint32_t tv = 0; - uint32_t t; + uint32_t nt; - for (;;) { - if (p[n] <= 0x7f) { - t = p[n]; - tv |= (t << (7 * n)); - n++; - break; - } - - t = p[n] & 0x7f; - tv |= (t << (7 * n)); - n++; - } - - if (nData) *nData = n; + n += tGetU32v(p, &nt); + if (nData) *nData = nt; if (ppData) *ppData = p + n; + n += nt; - n += tv; return n; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d421c452ca..a37d8931ad 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -127,7 +127,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal COL_VAL_SET_NULL(pColVal); } else { p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset; - tGetBinary(p, &p, &n); + // tGetBinary(p, &p, &n); (todo) COL_VAL_SET_VAL(pColVal, p, n); } } else { @@ -181,7 +181,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal // get real value p = p + pTColumn->offset; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); + // tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); (todo) } else { n = pTColumn->bytes; } @@ -283,7 +283,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) { pBuilder->row.flags = 0; } -int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { +int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData) { STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol]; uint8_t *p; int32_t iCol; From 4568c672d643e5ff72fed338d91c9d9d5bac3456 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 17 May 2022 06:40:37 +0000 Subject: [PATCH 011/103] feat: tdata --- include/common/tdataformat.h | 43 ++++++-------------- include/util/tencode.h | 2 +- source/common/src/tdataformat.c | 69 ++++++++++++++++----------------- 3 files changed, 46 insertions(+), 68 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 1097f62798..bf2ddc0920 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -38,30 +38,11 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // SColVal -#define COL_VAL_SET_NONE(CV) \ - do { \ - (CV)->type = COL_NONE; \ - (CV)->nData = 0; \ - (CV)->pData = NULL; \ - } while (0) - -#define COL_VAL_SET_NULL(CV) \ - do { \ - (CV)->type = COL_NULL; \ - (CV)->nData = 0; \ - (CV)->pData = NULL; \ - } while (0) - -#define COL_VAL_SET_VAL(CV, PDATA, NDATA) \ - do { \ - (CV)->type = COL_VAL; \ - (CV)->nData = (NDATA); \ - (CV)->pData = (PDATA); \ - } while (0) +#define ColValNONE ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL}) +#define ColValNULL ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL}) +#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)}) // STSRow2 -int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); -int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); @@ -96,11 +77,11 @@ struct STSchema { #define TSROW_HAS_VAL ((uint8_t)0x4U) #define TSROW_KV_ROW ((uint8_t)0x10U) struct STSRow2 { - TSKEY ts; - uint8_t flags; - int32_t sver; - uint32_t nData; - const uint8_t *pData; + TSKEY ts; + uint8_t flags; + int32_t sver; + uint32_t nData; + uint8_t *pData; }; struct STSRowBuilder { @@ -117,11 +98,11 @@ struct STSRowBuilder { STSRow2 row; }; -typedef enum { COL_VAL = 0, COL_NONE = 1, COL_NULL = 2 } EColValT; +typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_Data = 2 } EColValT; struct SColVal { - EColValT type; - uint32_t nData; - const uint8_t *pData; + EColValT type; + uint32_t nData; + uint8_t *pData; }; #if 1 //==================================== diff --git a/include/util/tencode.h b/include/util/tencode.h index 484c9443f7..938e3018a8 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -630,7 +630,7 @@ static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nDat return n; } -static FORCE_INLINE int32_t tGetBinary(uint8_t* p, const uint8_t** ppData, uint32_t* nData) { +static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) { int32_t n = 0; uint32_t nt; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a37d8931ad..fbc2c2306d 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -38,49 +38,46 @@ typedef struct { #define GET_BIT2(p, i) (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3)) // STSRow2 -int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { - if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; - if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1; - if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; +int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { + int32_t n = 0; - ASSERT(pRow->flags & 0xf != 0); + n += tPutI64(p ? p + n : p, pRow->ts); + n += tPutI8(p ? p + n : p, pRow->flags); + n += tPutI32v(p ? p + n : p, pRow->sver); + + ASSERT(pRow->flags & 0xf); switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: - return 0; - case TSROW_HAS_VAL: - ASSERT(!TSROW_IS_KV_ROW(pRow)); + break; default: - ASSERT(pRow->nData && pRow->pData); - if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1; + n += tPutBinary(p ? p + n : p, pRow->pData, pRow->nData); break; } - return 0; + return n; } -int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { - if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1; - if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; +int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) { + int32_t n = 0; + uint8_t flags; - ASSERT(pRow->flags & 0xf != 0); + n += tGetI64(p + n, pRow ? &pRow->ts : NULL); + n += tGetI8(p + n, pRow ? &pRow->flags : &flags); + n += tGetI32v(p + n, pRow ? &pRow->sver : NULL); - switch (pRow->flags & 0xf) { + if (pRow) flags = pRow->flags; + switch (flags & 0xf) { case TSROW_HAS_NONE: case TSROW_HAS_NULL: - pRow->nData = 0; - pRow->pData = NULL; - return 0; - case TSROW_HAS_VAL: - ASSERT(!TSROW_IS_KV_ROW(pRow)); + break; default: - if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1; + n += tGetBinary(p + n, pRow ? &pRow->pData : NULL, pRow ? &pRow->nData : NULL); break; } - return 0; + return n; } static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) { @@ -109,10 +106,10 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal ASSERT(pRow->flags & 0xf != 0); switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: - COL_VAL_SET_NONE(pColVal); + // COL_VAL_SET_NONE(pColVal); return 0; case TSROW_HAS_NULL: - COL_VAL_SET_NULL(pColVal); + // COL_VAL_SET_NULL(pColVal); return 0; } @@ -122,13 +119,13 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal pTSKVRow = (STSKVRow *)pRow->pData; pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn); if (pKVIdx == NULL) { - COL_VAL_SET_NONE(pColVal); + // COL_VAL_SET_NONE(pColVal); } else if (pKVIdx->offset < 0) { - COL_VAL_SET_NULL(pColVal); + // COL_VAL_SET_NULL(pColVal); } else { p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset; // tGetBinary(p, &p, &n); (todo) - COL_VAL_SET_VAL(pColVal, p, n); + // COL_VAL_SET_VAL(pColVal, p, n); } } else { // get bitmap @@ -137,9 +134,9 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal case TSROW_HAS_NULL | TSROW_HAS_NONE: v = GET_BIT1(p, bidx); if (v == 0) { - COL_VAL_SET_NONE(pColVal); + // COL_VAL_SET_NONE(pColVal); } else { - COL_VAL_SET_NULL(pColVal); + // COL_VAL_SET_NULL(pColVal); } return 0; case TSROW_HAS_VAL | TSROW_HAS_NONE: @@ -148,7 +145,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal p = p + (pTSchema->numOfCols - 2) / 8 + 1; break; } else { - COL_VAL_SET_NONE(pColVal); + // COL_VAL_SET_NONE(pColVal); return 0; } case TSROW_HAS_VAL | TSROW_HAS_NULL: @@ -157,16 +154,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal p = p + (pTSchema->numOfCols - 2) / 8 + 1; break; } else { - COL_VAL_SET_NULL(pColVal); + // COL_VAL_SET_NULL(pColVal); return 0; } case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: v = GET_BIT2(p, bidx); if (v == 0) { - COL_VAL_SET_NONE(pColVal); + // COL_VAL_SET_NONE(pColVal); return 0; } else if (v == 1) { - COL_VAL_SET_NULL(pColVal); + // COL_VAL_SET_NULL(pColVal); return 0; } else if (v == 2) { p = p + (pTSchema->numOfCols - 2) / 4 + 1; @@ -185,7 +182,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal } else { n = pTColumn->bytes; } - COL_VAL_SET_VAL(pColVal, p, n); + // COL_VAL_SET_VAL(pColVal, p, n); } return 0; From af8ac6e1b08704e77c0d558b02bf4a85529951b5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 17 May 2022 07:54:38 +0000 Subject: [PATCH 012/103] more tdata --- include/common/tdataformat.h | 4 +- source/common/src/tdataformat.c | 65 ++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index bf2ddc0920..af035d0d1f 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -45,10 +45,12 @@ void tTSchemaDestroy(STSchema *pTSchema); // STSRow2 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); +int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow); +void tTSRowFree(STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); // STSRowBuilder -int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols); +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema); void tTSRowBuilderClear(STSRowBuilder *pBuilder); void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fbc2c2306d..3f179ac58a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -32,6 +32,8 @@ typedef struct { #pragma pack(pop) #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) +#define BIT1_SIZE(n) (((n)-1) / 8 + 1) +#define BIT2_SIZE(n) (((n)-1) / 4 + 1) #define SET_BIT1(p, i, v) ((p)[(i) / 8] = (p)[(i) / 8] & (~(((uint8_t)1) << ((i) % 8))) | ((v) << ((i) % 8))) #define SET_BIT2(p, i, v) ((p)[(i) / 4] = (p)[(i) / 4] & (~(((uint8_t)3) << ((i) % 4))) | ((v) << ((i) % 4))) #define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) @@ -80,6 +82,31 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) { return n; } +int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow) { + (*ppRow) = taosMemoryMalloc(sizeof(*pRow) + pRow->nData); + if (*ppRow == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + (*ppRow)->ts = pRow->ts; + (*ppRow)->flags = pRow->flags; + (*ppRow)->sver = pRow->sver; + (*ppRow)->nData = pRow->nData; + if (pRow->nData) { + (*ppRow)->pData = (uint8_t *)(&(*ppRow)[1]); + memcpy((*ppRow)->pData, pRow->pData, pRow->nData); + } else { + (*ppRow)->pData = NULL; + } + + return 0; +} + +void tTSRowFree(STSRow2 *pRow) { + if (pRow) taosMemoryFree(pRow); +} + static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) { col_id_t cid = *(col_id_t *)p1; SKVIdx *pKVIdx = (SKVIdx *)p2; @@ -229,11 +256,11 @@ void tTSchemaDestroy(STSchema *pTSchema) { } // STSRowBuilder -int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { +int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema) { if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; - pBuilder->szBitMap1 = (nCols - 2) / 8 + 1; - pBuilder->szBitMap2 = (nCols - 2) / 4 + 1; + pBuilder->szBitMap1 = BIT1_SIZE(nCols - 1); + pBuilder->szBitMap2 = BIT2_SIZE(nCols - 1); pBuilder->szKVBuf = sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; @@ -286,16 +313,16 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, u int32_t iCol; STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; - // use interp search (todo) - if (pTColumn->colId > cid) { + // use interp search + if (pTColumn->colId < cid) { // right search for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pTColumn->colId == cid) break; + if (pTColumn->colId >= cid) break; } - } else if (pTColumn->colId < cid) { + } else if (pTColumn->colId > cid) { // left search for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) { pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pTColumn->colId == cid) break; + if (pTColumn->colId <= cid) break; } } @@ -391,7 +418,7 @@ static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) { case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: if (pTColumn->flags & COL_SET_NULL) { SET_BIT2(p, bidx, (uint8_t)1); - } else if (pTColumn->flags & COL_SET_NULL) { + } else if (pTColumn->flags & COL_SET_VAL) { SET_BIT2(p, bidx, (uint8_t)2); } else { SET_BIT2(p, bidx, (uint8_t)0); @@ -412,14 +439,15 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { int32_t nDataTP, nDataKV; uint32_t flags; STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; + int32_t nCols = pBuilder->pTSchema->numOfCols; // error not set ts if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) { return -1; } - ASSERT(pTSKVRow->nCols < pBuilder->pTSchema->numOfCols); - if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) { + ASSERT(pTSKVRow->nCols < nCols); + if (pTSKVRow->nCols < nCols - 1) { pBuilder->row.flags |= TSROW_HAS_NONE; } @@ -453,13 +481,15 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { if (nDataKV < nDataTP) { // generate KV row + ASSERT(pBuilder->row.flags & 0xf != TSROW_HAS_VAL); + pBuilder->row.flags |= TSROW_KV_ROW; pBuilder->row.nData = nDataKV; pBuilder->row.pData = pBuilder->pKVBuf; qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); - if (pTSKVRow->nCols < pBuilder->pTSchema->numOfCols - 1) { - memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[pBuilder->pTSchema->numOfCols - 1], pBuilder->vlenKV); + if (pTSKVRow->nCols < nCols - 1) { + memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[nCols - 1], pBuilder->vlenKV); } } else { // generate TUPLE row @@ -472,14 +502,13 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { if (flags == TSROW_HAS_VAL) { pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2; } else { - if (flags == TSROW_HAS_VAL) { - p = pBuilder->pTPBuf; + if (flags == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) { + pBuilder->row.pData = pBuilder->pTPBuf; } else { - p = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; + pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; } - setBitMap(p, pBuilder->pTSchema, flags); - pBuilder->row.pData = p; + setBitMap(pBuilder->row.pData, pBuilder->pTSchema, flags); } } From aaf78935b9a1468b6e093f126c0aed4c07a5b781 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 17 May 2022 08:13:58 +0000 Subject: [PATCH 013/103] more tdata --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 55 ++++++++++++++------------------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index af035d0d1f..7e411cb301 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -100,7 +100,7 @@ struct STSRowBuilder { STSRow2 row; }; -typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_Data = 2 } EColValT; +typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT; struct SColVal { EColValT type; uint32_t nData; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 3f179ac58a..a5447c016f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -39,6 +39,8 @@ typedef struct { #define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) #define GET_BIT2(p, i) (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3)) +static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); + // STSRow2 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { int32_t n = 0; @@ -107,18 +109,6 @@ void tTSRowFree(STSRow2 *pRow) { if (pRow) taosMemoryFree(pRow); } -static FORCE_INLINE int kvRowCmprFn(const void *p1, const void *p2) { - col_id_t cid = *(col_id_t *)p1; - SKVIdx *pKVIdx = (SKVIdx *)p2; - - if (cid < pKVIdx->cid) { - return -1; - } else if (cid > pKVIdx->cid) { - return 1; - } - return 0; -} - int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { uint32_t n; const uint8_t *p; @@ -128,15 +118,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal STSKVRow *pTSKVRow; SKVIdx *pKVIdx; + ASSERT(iCol != 0); ASSERT(pTColumn->colId != 0); ASSERT(pRow->flags & 0xf != 0); switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: - // COL_VAL_SET_NONE(pColVal); + *pColVal = ColValNONE; return 0; case TSROW_HAS_NULL: - // COL_VAL_SET_NULL(pColVal); + *pColVal = ColValNULL; return 0; } @@ -144,15 +135,16 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal ASSERT((pRow->flags & 0xf) != TSROW_HAS_VAL); pTSKVRow = (STSKVRow *)pRow->pData; - pKVIdx = bsearch(&pTColumn->colId, pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), kvRowCmprFn); + pKVIdx = + bsearch(&((SKVIdx){.cid = pTColumn->colId}), pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); if (pKVIdx == NULL) { - // COL_VAL_SET_NONE(pColVal); + *pColVal = ColValNONE; } else if (pKVIdx->offset < 0) { - // COL_VAL_SET_NULL(pColVal); + *pColVal = ColValNULL; } else { p = pRow->pData + sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pKVIdx->offset; - // tGetBinary(p, &p, &n); (todo) - // COL_VAL_SET_VAL(pColVal, p, n); + pColVal->type = COL_VAL_DATA; + tGetBinary(p, &pColVal->pData, &pColVal->nData); } } else { // get bitmap @@ -161,39 +153,39 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal case TSROW_HAS_NULL | TSROW_HAS_NONE: v = GET_BIT1(p, bidx); if (v == 0) { - // COL_VAL_SET_NONE(pColVal); + *pColVal = ColValNONE; } else { - // COL_VAL_SET_NULL(pColVal); + *pColVal = ColValNULL; } return 0; case TSROW_HAS_VAL | TSROW_HAS_NONE: v = GET_BIT1(p, bidx); if (v == 1) { - p = p + (pTSchema->numOfCols - 2) / 8 + 1; + p = p + BIT1_SIZE(pTSchema->numOfCols - 1); break; } else { - // COL_VAL_SET_NONE(pColVal); + *pColVal = ColValNONE; return 0; } case TSROW_HAS_VAL | TSROW_HAS_NULL: v = GET_BIT1(p, bidx); if (v == 1) { - p = p + (pTSchema->numOfCols - 2) / 8 + 1; + p = p + BIT1_SIZE(pTSchema->numOfCols - 1); break; } else { - // COL_VAL_SET_NULL(pColVal); + *pColVal = ColValNULL; return 0; } case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: v = GET_BIT2(p, bidx); if (v == 0) { - // COL_VAL_SET_NONE(pColVal); + *pColVal = ColValNONE; return 0; } else if (v == 1) { - // COL_VAL_SET_NULL(pColVal); + *pColVal = ColValNULL; return 0; } else if (v == 2) { - p = p + (pTSchema->numOfCols - 2) / 4 + 1; + p = p + BIT2_SIZE(pTSchema->numOfCols - 1); break; } else { ASSERT(0); @@ -204,12 +196,13 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal // get real value p = p + pTColumn->offset; + pColVal->type = COL_VAL_DATA; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - // tGetBinary(p + pTSchema->flen + *(int32_t *)p, &p, &n); (todo) + tGetBinary(p + pTSchema->flen + *(int32_t *)p, &pColVal->pData, &pColVal->nData); } else { - n = pTColumn->bytes; + pColVal->pData = p; + pColVal->nData = pTColumn->bytes; } - // COL_VAL_SET_VAL(pColVal, p, n); } return 0; From d6521a0a52a47c3a4257dd7957e483316877bce8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 17 May 2022 08:16:23 +0000 Subject: [PATCH 014/103] more tdata --- source/common/src/tdataformat.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a5447c016f..4b9531aa6b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -110,13 +110,13 @@ void tTSRowFree(STSRow2 *pRow) { } int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { - uint32_t n; - const uint8_t *p; - uint8_t v; - int32_t bidx = iCol - 1; - STColumn *pTColumn = &pTSchema->columns[iCol]; - STSKVRow *pTSKVRow; - SKVIdx *pKVIdx; + uint32_t n; + uint8_t *p; + uint8_t v; + int32_t bidx = iCol - 1; + STColumn *pTColumn = &pTSchema->columns[iCol]; + STSKVRow *pTSKVRow; + SKVIdx *pKVIdx; ASSERT(iCol != 0); ASSERT(pTColumn->colId != 0); From aefa9bd89186464dc02a7a810c4e2f6afed86dcb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 17 May 2022 16:53:55 +0800 Subject: [PATCH 015/103] refactor: do some internal refactor. --- include/libs/function/function.h | 38 +- source/libs/executor/src/executorimpl.c | 16 +- source/libs/function/src/builtinsimpl.c | 23 +- source/libs/function/src/taggfunction.c | 652 +----------------------- 4 files changed, 36 insertions(+), 693 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 616aec8c02..7d3e969c41 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -170,32 +170,18 @@ typedef struct SInputColumnInfoData { // sql function runtime context typedef struct SqlFunctionCtx { - SInputColumnInfoData input; - SResultDataInfo resDataInfo; - uint32_t order; // data block scanner order: asc|desc - uint8_t scanFlag; // record current running step, default: 0 - //////////////////////////////////////////////////////////////// - int32_t startRow; // start row index - int32_t size; // handled processed row number - SColumnInfoData* pInput; - SColumnDataAgg agg; - int16_t inputType; // TODO remove it - int16_t inputBytes; // TODO remove it - bool hasNull; // null value exist in current block, TODO remove it - bool requireNull; // require null in some function, TODO remove it - int32_t columnIndex; // TODO remove it - bool isAggSet; - int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it - bool stableQuery; - ///////////////////////////////////////////////////////////////// - int16_t functionId; // function id - char * pOutput; // final result output buffer, point to sdata->data - int32_t numOfParams; - SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - int64_t *ptsList; // corresponding timestamp array list - SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - int32_t offset; - SVariant tag; + SInputColumnInfoData input; + SResultDataInfo resDataInfo; + uint32_t order; // data block scanner order: asc|desc + uint8_t scanFlag; // record current running step, default: 0 + int16_t functionId; // function id + char *pOutput; // final result output buffer, point to sdata->data + int32_t numOfParams; + SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param + int64_t *ptsList; // corresponding timestamp array list + SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + int32_t offset; + SVariant tag; struct SResultRowEntryInfo *resultInfo; SSubsidiaryResInfo subsidiaries; SPoint1 start; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c261271728..1ba494a041 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -602,8 +602,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].startTs = pWin->skey; - // keep it temporarily bool hasAgg = pCtx[k].input.colDataAggIsSet; int32_t numOfRows = pCtx[k].input.numOfRows; @@ -619,8 +617,8 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow // not a whole block involved in query processing, statistics data can not be used // NOTE: the original value of isSet have been changed here - if (pCtx[k].isAggSet && forwardStep < numOfTotal) { - pCtx[k].isAggSet = false; + if (pCtx[k].input.colDataAggIsSet && forwardStep < numOfTotal) { + pCtx[k].input.colDataAggIsSet = false; } if (fmIsWindowPseudoColumnFunc(pCtx[k].functionId)) { @@ -680,7 +678,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC int32_t order) { for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; + pCtx[i].input.numOfRows = pBlock->info.rows; setBlockStatisInfo(&pCtx[i], &pOperator->pExpr[i], pBlock); } } @@ -742,7 +740,8 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; + pCtx[i].input.numOfRows = pBlock->info.rows; + pCtx[i].pSrcBlock = pBlock; pCtx[i].scanFlag = scanFlag; @@ -827,7 +826,6 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt static int32_t doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx) { for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { if (functionNeedToExecute(&pCtx[k])) { - pCtx[k].startTs = startTs; // todo add a dummy funtion to avoid process check if (pCtx[k].fpSet.process != NULL) { int32_t code = pCtx[k].fpSet.process(&pCtx[k]); @@ -3330,7 +3328,7 @@ static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int3 static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExpr, int32_t rowIndex) { for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index - pCtx[j].startRow = rowIndex; +// pCtx[j].startRow = rowIndex; } for (int32_t j = 0; j < numOfExpr; ++j) { @@ -3381,7 +3379,7 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - pCtx[i].size = 1; +// pCtx[i].size = 1; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index af86eb4e90..4571631bc0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -287,6 +287,7 @@ static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) { } return numOfElem; } + /* * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer @@ -781,16 +782,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { pBuf->v = val; - // for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { - // SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; - // if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor - // __ctx->tag.i = key; - // __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - // } - // - // __ctx->fpSet.process(__ctx); - // } - if (pCtx->subsidiaries.num > 0) { saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } @@ -803,15 +794,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { pBuf->v = val; - // for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { - // SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; - // if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor - // __ctx->tag.i = key; - // __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; - // } - // - // __ctx->fpSet.process(__ctx); - // } if (pCtx->subsidiaries.num > 0) { saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } @@ -823,7 +805,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { double val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { pBuf->v = val; - if (pCtx->subsidiaries.num > 0) { saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } @@ -1703,7 +1684,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); + GET_TYPED_DATA(v, double, type, data); if (v < GET_DOUBLE_VAL(&pInfo->minval)) { SET_DOUBLE_VAL(&pInfo->minval, v); } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index b6d5d38c9e..950655e480 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -480,7 +480,7 @@ static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf initResultRowEntry(pResultInfo, pCtx->resDataInfo.interBufSize); return true; } - +#if 0 /** * in handling the stable query, function_finalizer is called after the secondary * merge being completed, during the first merge procedure, which is executed at the @@ -497,53 +497,6 @@ static void function_finalizer(SqlFunctionCtx *pCtx) { doFinalizer(pCtx); } -/* - * count function does need the finalize, if data is missing, the default value, which is 0, is used - * count function does not use the pCtx->interResBuf to keep the intermediate buffer - */ -static void count_function(SqlFunctionCtx *pCtx) { - int32_t numOfElem = 0; - - /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->isAggSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->isAggSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->isAggSet == false; - */ - if (pCtx->isAggSet) { - numOfElem = pCtx->size - pCtx->agg.numOfNull; - } else { - if (pCtx->hasNull) { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_DATA(pCtx, i); - if (isNull(val, pCtx->inputType)) { - continue; - } - - numOfElem += 1; - } - } else { - //when counting on the primary time stamp column and no statistics data is presented, use the size value directly. - numOfElem = pCtx->size; - } - } - - if (numOfElem > 0) { -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; - } - - *((int64_t *)pCtx->pOutput) += numOfElem; - SET_VAL(pCtx, numOfElem, 1); -} - -static void count_func_merge(SqlFunctionCtx *pCtx) { - int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); - for (int32_t i = 0; i < pCtx->size; ++i) { - *((int64_t *)pCtx->pOutput) += pData[i]; - } - - SET_VAL(pCtx, pCtx->size, 1); -} - /** * 1. If the column value for filter exists, we need to load the SFields, which serves * as the pre-filter to decide if the actual data block is required or not. @@ -633,113 +586,6 @@ int32_t noDataRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ } while (0) -static void do_sum(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // Only the pre-computing information loaded and actual data does not loaded - if (pCtx->isAggSet) { - notNullElems = pCtx->size - pCtx->agg.numOfNull; - assert(pCtx->size >= pCtx->agg.numOfNull); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->pOutput; - *retVal += pCtx->agg.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->pOutput; - *retVal += (uint64_t)pCtx->agg.sum; - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - double *retVal = (double*) pCtx->pOutput; - SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->agg.sum))); - } - } else { // computing based on the true data block - void *pData = GET_INPUT_DATA_LIST(pCtx); - notNullElems = 0; - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->pOutput; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*retVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*retVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->pOutput; - - if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*retVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double *)pCtx->pOutput; - LIST_ADD_N_DOUBLE(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double *)pCtx->pOutput; - LIST_ADD_N_DOUBLE_FLOAT(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } - } - - // data in the check operation are all null, not output - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; - } -} - -static void sum_function(SqlFunctionCtx *pCtx) { - do_sum(pCtx); - - // keep the result data in output buffer, not in the intermediate buffer - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - // set the flag for super table query - SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; - pSum->hasResult = DATA_SET_FLAG; -// } -} - -static void sum_func_merge(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char * input = GET_INPUT_DATA(pCtx, i); - SSumInfo *pInput = (SSumInfo *)input; - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } - - notNullElems++; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - *(int64_t *)pCtx->pOutput += pInput->isum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - *(uint64_t *) pCtx->pOutput += pInput->usum; - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput, *(double *)pCtx->pOutput + pInput->dsum); - } - } - - SET_VAL(pCtx, notNullElems, 1); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - static int32_t statisRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_SMA_LOAD; } @@ -822,17 +668,17 @@ static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_ */ static void avg_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; - + // NOTE: keep the intermediate result into the interResultBuf SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); double *pVal = &pAvgInfo->sum; - + if (pCtx->isAggSet) { // Pre-aggregation notNullElems = pCtx->size - pCtx->agg.numOfNull; assert(notNullElems >= 0); - + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { *pVal += pCtx->agg.sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { @@ -842,7 +688,7 @@ static void avg_function(SqlFunctionCtx *pCtx) { } } else { void *pData = GET_INPUT_DATA_LIST(pCtx); - + if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { @@ -865,18 +711,18 @@ static void avg_function(SqlFunctionCtx *pCtx) { LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); } } - + if (!pCtx->hasNull) { assert(notNullElems == pCtx->size); } - + SET_VAL(pCtx, notNullElems, 1); pAvgInfo->num += notNullElems; - + if (notNullElems > 0) { //pResInfo->hasResult = DATA_SET_FLAG; } - + // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); @@ -885,18 +731,18 @@ static void avg_function(SqlFunctionCtx *pCtx) { static void avg_func_merge(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - + double *sum = (double*) pCtx->pOutput; char *input = GET_INPUT_DATA_LIST(pCtx); - + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; if (pInput->num == 0) { // current input is null continue; } - + SET_DOUBLE_VAL(sum, *sum + pInput->sum); - + // keep the number of data into the temp buffer *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; } @@ -2735,18 +2581,6 @@ static void deriv_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += notNullElems; } -#define DIFF_IMPL(ctx, d, type) \ - do { \ - if ((ctx)->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED) { \ - (ctx)->param[1].param.nType = (ctx)->inputType; \ - *(type *)&(ctx)->param[1].param.i = *(type *)(d); \ - } else { \ - *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].param.i)); \ - *(type *)(&(ctx)->param[1].param.i) = *(type *)(d); \ - *(int64_t *)(ctx)->pTsOutput = GET_TS_DATA(ctx, index); \ - } \ - } while (0); - // TODO difference in date column static void diff_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); @@ -4072,460 +3906,4 @@ int32_t functionCompatList[] = { // tid_tag, derivative, blk_info 6, 8, 7, }; - -SAggFunctionInfo aggFunc[35] = {{ - // 0, count function does not invoke the finalize function - "count", - FUNCTION_TYPE_AGG, - FUNCTION_COUNT, - FUNCTION_COUNT, - BASIC_FUNC_SO, - function_setup, - count_function, - doFinalizer, - count_func_merge, - countRequired, - }, - { - // 1 - "sum", - FUNCTION_TYPE_AGG, - FUNCTION_SUM, - FUNCTION_SUM, - BASIC_FUNC_SO, - function_setup, - sum_function, - function_finalizer, - sum_func_merge, - statisRequired, - }, - { - // 2 - "avg", - FUNCTION_TYPE_AGG, - FUNCTION_AVG, - FUNCTION_AVG, - BASIC_FUNC_SO, - function_setup, - avg_function, - avg_finalizer, - avg_func_merge, - statisRequired, - }, - { - // 3 - "min", - FUNCTION_TYPE_AGG, - FUNCTION_MIN, - FUNCTION_MIN, - BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, - min_func_setup, - NULL, - function_finalizer, - min_func_merge, - statisRequired, - }, - { - // 4 - "max", - FUNCTION_TYPE_AGG, - FUNCTION_MAX, - FUNCTION_MAX, - BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, - max_func_setup, - NULL, - function_finalizer, - max_func_merge, - statisRequired, - }, - { - // 5 - "stddev", - FUNCTION_TYPE_AGG, - FUNCTION_STDDEV, - FUNCTION_STDDEV_DST, - FUNCSTATE_SO | FUNCSTATE_STREAM, - function_setup, - stddev_function, - stddev_finalizer, - noop1, - dataBlockRequired, - }, - { - // 6 - "percentile", - FUNCTION_TYPE_AGG, - FUNCTION_PERCT, - FUNCTION_INVALID_ID, - FUNCSTATE_SO | FUNCSTATE_STREAM, - percentile_function_setup, - percentile_function, - percentile_finalizer, - noop1, - dataBlockRequired, - }, - { - // 7 - "apercentile", - FUNCTION_TYPE_AGG, - FUNCTION_APERCT, - FUNCTION_APERCT, - FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE, - apercentile_function_setup, - apercentile_function, - apercentile_finalizer, - apercentile_func_merge, - dataBlockRequired, - }, - { - // 8 - "first", - FUNCTION_TYPE_AGG, - FUNCTION_FIRST, - FUNCTION_FIRST_DST, - BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, - function_setup, - first_function, - function_finalizer, - noop1, - firstFuncRequired, - }, - { - // 9 - "last", - FUNCTION_TYPE_AGG, - FUNCTION_LAST, - FUNCTION_LAST_DST, - BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, - function_setup, - last_function, - function_finalizer, - noop1, - lastFuncRequired, - }, - { - // 10 - "last_row", - FUNCTION_TYPE_AGG, - FUNCTION_LAST_ROW, - FUNCTION_LAST_ROW, - FUNCSTATE_SO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_row_function, - last_row_finalizer, - last_dist_func_merge, - dataBlockRequired, - }, - { - // 11 - "top", - FUNCTION_TYPE_AGG, - FUNCTION_TOP, - FUNCTION_TOP, - FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - top_function, - top_bottom_func_finalizer, - top_func_merge, - dataBlockRequired, - }, - { - // 12 - "bottom", - FUNCTION_TYPE_AGG, - FUNCTION_BOTTOM, - FUNCTION_BOTTOM, - FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, - top_bottom_function_setup, - bottom_function, - top_bottom_func_finalizer, - bottom_func_merge, - dataBlockRequired, - }, - { - // 13 - "spread", - FUNCTION_TYPE_AGG, - FUNCTION_SPREAD, - FUNCTION_SPREAD, - BASIC_FUNC_SO, - spread_function_setup, - spread_function, - spread_function_finalizer, - spread_func_merge, - countRequired, - }, - { - // 14 - "twa", - FUNCTION_TYPE_AGG, - FUNCTION_TWA, - FUNCTION_TWA, - BASIC_FUNC_SO | FUNCSTATE_NEED_TS, - twa_function_setup, - twa_function, - twa_function_finalizer, - twa_function_copy, - dataBlockRequired, - }, - { - // 15 - "leastsquares", - FUNCTION_TYPE_AGG, - FUNCTION_LEASTSQR, - FUNCTION_INVALID_ID, - FUNCSTATE_SO | FUNCSTATE_STREAM, - leastsquares_function_setup, - leastsquares_function, - leastsquares_finalizer, - noop1, - dataBlockRequired, - }, - { - // 16 - "dummy", - FUNCTION_TYPE_AGG, - FUNCTION_TS, - FUNCTION_TS, - BASIC_FUNC_SO | FUNCSTATE_NEED_TS, - function_setup, - date_col_output_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 17 - "ts", - FUNCTION_TYPE_AGG, - FUNCTION_TS_DUMMY, - FUNCTION_TS_DUMMY, - BASIC_FUNC_SO | FUNCSTATE_NEED_TS, - function_setup, - noop1, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 18 - "tag_dummy", - FUNCTION_TYPE_AGG, - FUNCTION_TAG_DUMMY, - FUNCTION_TAG_DUMMY, - BASIC_FUNC_SO, - function_setup, - tag_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 19 - "ts", - FUNCTION_TYPE_AGG, - FUNCTION_TS_COMP, - FUNCTION_TS_COMP, - FUNCSTATE_MO | FUNCSTATE_NEED_TS, - ts_comp_function_setup, - ts_comp_function, - ts_comp_finalize, - copy_function, - dataBlockRequired, - }, - { - // 20 - "tag", - FUNCTION_TYPE_AGG, - FUNCTION_TAG, - FUNCTION_TAG, - BASIC_FUNC_SO, - function_setup, - tag_function, - doFinalizer, - copy_function, - noDataRequired, - }, - {//TODO this is a scala function - // 21, column project sql function - "colprj", - FUNCTION_TYPE_AGG, - FUNCTION_PRJ, - FUNCTION_PRJ, - BASIC_FUNC_MO | FUNCSTATE_NEED_TS, - function_setup, - col_project_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 22, multi-output, tag function has only one result - "tagprj", - FUNCTION_TYPE_AGG, - FUNCTION_TAGPRJ, - FUNCTION_TAGPRJ, - BASIC_FUNC_MO, - function_setup, - tag_project_function, - doFinalizer, - copy_function, - noDataRequired, - }, - { - // 23 - "arithmetic", - FUNCTION_TYPE_AGG, - FUNCTION_ARITHM, - FUNCTION_ARITHM, - FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS, - function_setup, - arithmetic_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 24 - "diff", - FUNCTION_TYPE_AGG, - FUNCTION_DIFF, - FUNCTION_INVALID_ID, - FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, - diff_function_setup, - diff_function, - doFinalizer, - noop1, - dataBlockRequired, - }, - // distributed version used in two-stage aggregation processes - { - // 25 - "first_dist", - FUNCTION_TYPE_AGG, - FUNCTION_FIRST_DST, - FUNCTION_FIRST_DST, - BASIC_FUNC_SO | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, - first_last_function_setup, - first_dist_function, - function_finalizer, - first_dist_func_merge, - firstDistFuncRequired, - }, - { - // 26 - "last_dist", - FUNCTION_TYPE_AGG, - FUNCTION_LAST_DST, - FUNCTION_LAST_DST, - BASIC_FUNC_SO | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, - first_last_function_setup, - last_dist_function, - function_finalizer, - last_dist_func_merge, - lastDistFuncRequired, - }, - { - // 27 - "stddev", // return table id and the corresponding tags for join match and subscribe - FUNCTION_TYPE_AGG, - FUNCTION_STDDEV_DST, - FUNCTION_AVG, - FUNCSTATE_SO | FUNCSTATE_STABLE, - function_setup, - NULL, - NULL, - NULL, - dataBlockRequired, - }, - { - // 28 - "interp", - FUNCTION_TYPE_AGG, - FUNCTION_INTERP, - FUNCTION_INTERP, - FUNCSTATE_SO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS , - function_setup, - interp_function, - doFinalizer, - copy_function, - dataBlockRequired, - }, - { - // 29 - "rate", - FUNCTION_TYPE_AGG, - FUNCTION_RATE, - FUNCTION_RATE, - BASIC_FUNC_SO | FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_finalizer, - rate_func_copy, - dataBlockRequired, - }, - { - // 30 - "irate", - FUNCTION_TYPE_AGG, - FUNCTION_IRATE, - FUNCTION_IRATE, - BASIC_FUNC_SO | FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - rate_finalizer, - rate_func_copy, - dataBlockRequired, - }, - { - // 31 - "tbid", // return table id and the corresponding tags for join match and subscribe - FUNCTION_TYPE_AGG, - FUNCTION_TID_TAG, - FUNCTION_TID_TAG, - FUNCSTATE_MO | FUNCSTATE_STABLE, - function_setup, - noop1, - noop1, - noop1, - dataBlockRequired, - }, - { //32 - "derivative", // return table id and the corresponding tags for join match and subscribe - FUNCTION_TYPE_AGG, - FUNCTION_DERIVATIVE, - FUNCTION_INVALID_ID, - FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, - deriv_function_setup, - deriv_function, - doFinalizer, - noop1, - dataBlockRequired, - }, - { - // 33 - "block_dist", // return table id and the corresponding tags for join match and subscribe - FUNCTION_TYPE_AGG, - FUNCTION_BLKINFO, - FUNCTION_BLKINFO, - FUNCSTATE_SO | FUNCSTATE_STABLE, - function_setup, - blockInfo_func, - blockinfo_func_finalizer, - block_func_merge, - dataBlockRequired, - }, - { - // 34 - "cov", // return table id and the corresponding tags for join match and subscribe - FUNCTION_TYPE_AGG, - FUNCTION_COV, - FUNCTION_COV, - FUNCSTATE_SO | FUNCSTATE_STABLE, - function_setup, - sum_function, - function_finalizer, - sum_func_merge, - statisRequired, - } - }; +#endif From 3a851ca9f29b09cc1d2e2b37e045f8b7cad61558 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 17 May 2022 09:50:21 +0000 Subject: [PATCH 016/103] more tdata --- include/common/tdataformat.h | 18 +++++- source/common/src/tdataformat.c | 104 +++++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 7e411cb301..e13705d403 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -32,6 +32,8 @@ typedef struct STSchema STSchema; typedef struct SColVal SColVal; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; +typedef struct STagVal STagVal; +typedef struct STag STag; // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); @@ -56,6 +58,13 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); +// STag +int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag); +void tTagFree(STag *pTag); +void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData); +int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag); +int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag); + // STRUCT ================= struct STColumn { col_id_t colId; @@ -107,7 +116,14 @@ struct SColVal { uint8_t *pData; }; -#if 1 //==================================== +struct STagVal { + int16_t cid; + int8_t type; + uint32_t nData; + uint8_t *pData; +}; + +#if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP #define TD_SUPPORT_READ2 diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 4b9531aa6b..8aa8ed2f14 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -31,6 +31,19 @@ typedef struct { } STSKVRow; #pragma pack(pop) +typedef struct STagIdx { + int16_t cid; + uint16_t offset; +} STagIdx; + +#pragma pack(push, 1) +struct STag { + uint16_t len; + uint16_t nTag; + STagIdx idx[]; +}; +#pragma pack(pop) + #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) #define BIT1_SIZE(n) (((n)-1) / 8 + 1) #define BIT2_SIZE(n) (((n)-1) / 4 + 1) @@ -508,7 +521,96 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { return 0; } -#if 1 // ==================== +static FORCE_INLINE int tTagIdxCmprFn(const void *p1, const void *p2) { + STagIdx *pTagIdx1 = (STagIdx *)p1; + STagIdx *pTagIdx2 = (STagIdx *)p2; + if (pTagIdx1->cid < pTagIdx1->cid) { + return -1; + } else if (pTagIdx1->cid > pTagIdx1->cid) { + return 1; + } + return 0; +} +int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag) { + STagVal *pTagVal; + uint8_t *p; + int32_t n; + uint16_t tsize = sizeof(STag) + sizeof(STagIdx) * nTag; + + for (int16_t iTag = 0; iTag < nTag; iTag++) { + pTagVal = &pTagVals[iTag]; + + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + tsize += tPutBinary(NULL, pTagVal->pData, pTagVal->nData); + } else { + ASSERT(pTagVal->nData == TYPE_BYTES[pTagVal->type]); + tsize += pTagVal->nData; + } + } + + (*ppTag) = (STag *)taosMemoryMalloc(tsize); + if (*ppTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + p = (uint8_t *)&((*ppTag)->idx[nTag]); + n = 0; + + (*ppTag)->len = tsize; + (*ppTag)->nTag = nTag; + for (int16_t iTag = 0; iTag < nTag; iTag++) { + pTagVal = &pTagVals[iTag]; + + (*ppTag)->idx[iTag].cid = pTagVal->cid; + (*ppTag)->idx[iTag].offset = n; + + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + n += tPutBinary(p + n, pTagVal->pData, pTagVal->nData); + } else { + memcpy(p + n, pTagVal->pData, pTagVal->nData); + n += pTagVal->nData; + } + } + + qsort((*ppTag)->idx, (*ppTag)->nTag, sizeof(STagIdx), tTagIdxCmprFn); + return 0; +} + +void tTagFree(STag *pTag) { + if (pTag) taosMemoryFree(pTag); +} + +void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData) { + STagIdx *pTagIdx = bsearch(&((STagIdx){.cid = cid}), pTag->idx, pTag->nTag, sizeof(STagIdx), tTagIdxCmprFn); + if (pTagIdx == NULL) { + *ppData = NULL; + *nData = 0; + } else { + uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag] + pTagIdx->offset; + if (IS_VAR_DATA_TYPE(type)) { + tGetBinary(p, ppData, nData); + } else { + *ppData = p; + *nData = TYPE_BYTES[type]; + } + } +} + +int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag) { + // return tEncodeBinary(pEncoder, (uint8_t *)pTag, pTag->len); + ASSERT(0); + return 0; +} + +int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag) { + // uint32_t n; + // return tDecodeBinary(pDecoder, (const uint8_t **)ppTag, &n); + ASSERT(0); + return 0; +} + +#if 1 // =================================================================================================================== static void dataColSetNEleNull(SDataCol *pCol, int nEle); int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; From 82a8f63decd088b53affb45c7aa137eb987b3158 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 May 2022 21:20:43 +0800 Subject: [PATCH 017/103] fix: bad code merge --- source/dnode/mnode/impl/src/mndTopic.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 8847bce890..e99046a32d 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -502,6 +502,17 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name); + if (pTopic == NULL) { + if (dropReq.igNotExists) { + mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); + return 0; + } else { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + return -1; + } + } + if (pTopic->refConsumerCnt != 0) { mndReleaseTopic(pMnode, pTopic); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; From 7563605418dd035aa07b9875a23949e24851f69c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 May 2022 21:24:32 +0800 Subject: [PATCH 018/103] fix: wrong db status --- source/dnode/mnode/impl/src/mndDb.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index bb7e593fea..c666c50864 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1447,8 +1447,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } char *status = "ready"; - char b[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(b, status, strlen(status)); + char statusB[24] = {0}; + STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status)); if (sysDb) { for (int32_t i = 0; i < pShow->numOfColumns; ++i) { @@ -1458,7 +1458,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } else if (i == 3) { colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); } else if (i == 20) { - colDataAppend(pColInfo, rows, b, false); + colDataAppend(pColInfo, rows, statusB, false); } else { colDataAppendNULL(pColInfo, rows); } @@ -1481,9 +1481,10 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); const char *src = pDb->cfg.strict ? "strict" : "nostrict"; - STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src)); + char strict[24] = {0}; + STR_WITH_SIZE_TO_VARSTR(strict, src, strlen(src)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)b, false); + colDataAppend(pColInfo, rows, (const char *)strict, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.daysPerFile, false); @@ -1554,7 +1555,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - colDataAppend(pColInfo, rows, (const char *)b, false); + colDataAppend(pColInfo, rows, (const char *)statusB, false); } } From b858688fab96a667c2d8fd486f0ec470d9afa0f0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 17 May 2022 21:31:59 +0800 Subject: [PATCH 019/103] fix: unreasonable error message when creating db with insufficient dnodes --- source/dnode/mnode/impl/src/mndDb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c666c50864..c3e5195ad2 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -398,11 +398,14 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; - if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; + if (pCfg->replications > mndGetDnodeSize(pMnode)) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } terrno = 0; return TSDB_CODE_SUCCESS; From 37e10aca292dcd483c9dbf78a6271a6cae72f6a7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 17 May 2022 21:46:46 +0800 Subject: [PATCH 020/103] fix:precision problems in time conversion --- source/client/src/clientSml.c | 10 ++- source/client/test/smlTest.cpp | 33 +++++++++ source/common/src/ttime.c | 122 +++++++++++++++++++++++++++++---- 3 files changed, 149 insertions(+), 16 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 884756ced9..f6f8db8590 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -714,25 +714,31 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { char *endPtr = NULL; - double ts = (double)strtoll(value, &endPtr, 10); + int64_t tsInt64 = strtoll(value, &endPtr, 10); if(value + len != endPtr){ return -1; } + double ts = tsInt64; switch (type) { case TSDB_TIME_PRECISION_HOURS: ts *= (3600 * 1e9); + tsInt64 *= (3600 * 1e9); break; case TSDB_TIME_PRECISION_MINUTES: ts *= (60 * 1e9); + tsInt64 *= (60 * 1e9); break; case TSDB_TIME_PRECISION_SECONDS: ts *= (1e9); + tsInt64 *= (1e9); break; case TSDB_TIME_PRECISION_MILLI: ts *= (1e6); + tsInt64 *= (1e6); break; case TSDB_TIME_PRECISION_MICRO: ts *= (1e3); + tsInt64 *= (1e3); break; case TSDB_TIME_PRECISION_NANO: break; @@ -743,7 +749,7 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { return -1; } - return (int64_t)ts; + return tsInt64; } static int64_t smlGetTimeNow(int8_t precision) { diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index eeab844712..d9a81ad3e6 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1188,3 +1188,36 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { destroyRequest(request); smlDestroyInfo(info); } + +TEST(testCase, sml_TD15662_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_15662 precision 'ns'"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_15662"); + taos_free_result(pRes); + + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, nullptr); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT_NE(info, nullptr); + + const char *sql[] = { + "iyyyje,id=iyyyje_41943_1303,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + }; + int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0])); + ASSERT_EQ(ret, 0); + + // case 1 + TAOS_RES *res = taos_query(taos, "select * from t_a5615048edae55218a22a149edebdc82"); + ASSERT_NE(res, nullptr); + + TAOS_ROW row = taos_fetch_row(res); + int64_t ts = *(int64_t*)row[0]; + ASSERT_EQ(ts, 1626006833639000000); + + taos_free_result(res); +} \ No newline at end of file diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index c2892c070f..c121b2d83d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -374,39 +374,133 @@ char getPrecisionUnit(int32_t precision) { } int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { - assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || + assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || + fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); - assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || + assert(toPrecision == TSDB_TIME_PRECISION_MILLI || + toPrecision == TSDB_TIME_PRECISION_MICRO || toPrecision == TSDB_TIME_PRECISION_NANO); - static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}}; - return (int64_t)((double)time * factors[fromPrecision][toPrecision]); + double tempResult = (double)time; + switch(fromPrecision) { + case TSDB_TIME_PRECISION_MILLI: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time; + case TSDB_TIME_PRECISION_MICRO: + tempResult *= 1000; + time *= 1000; + goto end_; + case TSDB_TIME_PRECISION_NANO: + tempResult *= 1000000; + time *= 1000000; + goto end_; + } + } // end from milli + case TSDB_TIME_PRECISION_MICRO: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time / 1000; + case TSDB_TIME_PRECISION_MICRO: + return time; + case TSDB_TIME_PRECISION_NANO: + tempResult *= 1000; + time *= 1000; + goto end_; + } + } //end from micro + case TSDB_TIME_PRECISION_NANO: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time / 1000000; + case TSDB_TIME_PRECISION_MICRO: + return time / 1000; + case TSDB_TIME_PRECISION_NANO: + return time; + } + } //end from nano + default: { + assert(0); + return time; // only to pass windows compilation + } + } //end switch fromPrecision +end_: + if (tempResult >= (double)INT64_MAX) return INT64_MAX; + if (tempResult <= (double)INT64_MIN) return INT64_MIN; // INT64_MIN means NULL + return time; } +// !!!!notice:there are precision problems, double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = 1626006833631000064 +//int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { +// assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || +// fromPrecision == TSDB_TIME_PRECISION_NANO); +// assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || +// toPrecision == TSDB_TIME_PRECISION_NANO); +// static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}}; +// ((double)time * factors[fromPrecision][toPrecision]); +//} + int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); - static double factors[3] = {1000000., 1000., 1.}; + int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; + double tmp = time; switch (toUnit) { - case 's': - return time * factors[fromPrecision] / NANOSECOND_PER_SEC; + case 's':{ + tmp *= (NANOSECOND_PER_SEC/factors[fromPrecision]); // the result of division is an integer + time *= (NANOSECOND_PER_SEC/factors[fromPrecision]); + break; + } case 'm': - return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE; + tmp *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); // the result of division is an integer + time *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); + break; case 'h': - return time * factors[fromPrecision] / NANOSECOND_PER_HOUR; + tmp *= (NANOSECOND_PER_HOUR/factors[fromPrecision]); // the result of division is an integer + time *= (NANOSECOND_PER_HOUR/factors[fromPrecision]); + break; case 'd': - return time * factors[fromPrecision] / NANOSECOND_PER_DAY; + tmp *= (NANOSECOND_PER_DAY/factors[fromPrecision]); // the result of division is an integer + time *= (NANOSECOND_PER_DAY/factors[fromPrecision]); + break; case 'w': - return time * factors[fromPrecision] / NANOSECOND_PER_WEEK; + tmp *= (NANOSECOND_PER_WEEK/factors[fromPrecision]); // the result of division is an integer + time *= (NANOSECOND_PER_WEEK/factors[fromPrecision]); + break; case 'a': - return time * factors[fromPrecision] / NANOSECOND_PER_MSEC; + tmp *= (NANOSECOND_PER_MSEC/factors[fromPrecision]); // the result of division is an integer + time *= (NANOSECOND_PER_MSEC/factors[fromPrecision]); + break; case 'u': - return time * factors[fromPrecision] / NANOSECOND_PER_USEC; + // the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double + switch (fromPrecision) { + case TSDB_TIME_PRECISION_MILLI:{ + tmp /= 1000; + time /= 1000; + break; + } + case TSDB_TIME_PRECISION_MICRO:{ + tmp /= 1; + time /= 1; + break; + } + case TSDB_TIME_PRECISION_NANO:{ + tmp *= 1000; + time *= 1000; + break; + } + } + break; case 'b': - return time * factors[fromPrecision]; + tmp *= factors[fromPrecision]; + time *= factors[fromPrecision]; + break; default: { return -1; } } + if (tmp >= (double)INT64_MAX) return INT64_MAX; + if (tmp <= (double)INT64_MIN) return INT64_MIN; + return time; } int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) { From a3b0136e3ce6d61951c946045ce1e6b4c9f60cf5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 17 May 2022 22:03:50 +0800 Subject: [PATCH 021/103] stmt query --- source/client/src/clientStmt.c | 8 ++++++ tests/script/api/batchprepare.c | 47 ++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index fa8df28523..f2c5fdec0f 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -218,6 +218,13 @@ int32_t stmtParseSql(STscStmt* pStmt) { pStmt->bInfo.needParse = false; + if (pStmt->sql.pQuery->pRoot && 0 == pStmt->sql.type) { + pStmt->sql.type = STMT_TYPE_INSERT; + } else if (pStmt->sql.pQuery->pPrepareRoot) { + pStmt->sql.type = STMT_TYPE_QUERY; + } + +/* switch (nodeType(pStmt->sql.pQuery->pRoot)) { case QUERY_NODE_VNODE_MODIF_STMT: if (0 == pStmt->sql.type) { @@ -231,6 +238,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { tscError("not supported stmt type %d", nodeType(pStmt->sql.pQuery->pRoot)); STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR); } +*/ return TSDB_CODE_SUCCESS; } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index cb914f8d6d..546075a1d9 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -170,11 +170,11 @@ CaseCfg gCase[] = { // 22 {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, true, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, -// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, -// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, - {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, - {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, }; @@ -209,7 +209,7 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 0 +#if 1 CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, .printCreateTblSql = false, @@ -267,7 +267,7 @@ CaseCtrl gCaseCtrl = { }; #endif -#if 1 +#if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 1, .printCreateTblSql = false, @@ -292,7 +292,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper - .bindNullNum = 0, + .bindNullNum = 1, .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, @@ -309,10 +309,10 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .printRes = true, .runTimes = 0, .caseRunIdx = -1, - .optrIdxListNum = tListLen(optrIdxList), - .optrIdxList = optrIdxList, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, + //.optrIdxListNum = tListLen(optrIdxList), + //.optrIdxList = optrIdxList, + //.bindColTypeNum = tListLen(bindColTypeList), + //.bindColTypeList = bindColTypeList, .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, @@ -665,15 +665,16 @@ void bpGenerateConstInFuncSQL(BindData *data, int32_t tblIdx) { void generateQueryMiscSQL(BindData *data, int32_t tblIdx) { - switch(tblIdx) { - case 0: - //TODO FILL TEST - default: - bpGenerateConstInOpSQL(data, tblIdx); - break; - case FUNCTION_TEST_IDX: - bpGenerateConstInFuncSQL(data, tblIdx); - break; + if (tblIdx == FUNCTION_TEST_IDX && gCurCase->bindNullNum <= 0) { + bpGenerateConstInFuncSQL(data, tblIdx); + } else { + switch(tblIdx) { + case 0: + //TODO FILL TEST + default: + bpGenerateConstInOpSQL(data, tblIdx); + break; + } } if (gCaseCtrl.printStmtSql) { @@ -1064,6 +1065,8 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { if (tblIdx == FUNCTION_TEST_IDX) { gCaseCtrl.numericParam = true; + } else { + gCaseCtrl.numericParam = false; } for (int b = 0; b < bindNum; b++) { @@ -1072,8 +1075,10 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { } } + gCaseCtrl.numericParam = false; + generateQueryMiscSQL(data, tblIdx); - + return 0; } From 485a1aedd5a46d69e30bcd0bb736c06586a78edb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 17 May 2022 22:27:46 +0800 Subject: [PATCH 022/103] fix(query): fix elapsed parameter number --- source/libs/function/src/builtinsimpl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index b98fd0f243..4ab1a52897 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2516,7 +2516,7 @@ bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo pInfo->min = MAX_TS_KEY; pInfo->max = 0; - if (pCtx->numOfParams == 3) { + if (pCtx->numOfParams == 2) { pInfo->timeUnit = pCtx->param[1].param.i; } else { pInfo->timeUnit = 1; @@ -2540,7 +2540,6 @@ int32_t elapsedFunction(SqlFunctionCtx *pCtx) { } if (pInput->colDataAggIsSet) { - if (pInfo->min == MAX_TS_KEY) { pInfo->min = GET_INT64_VAL(&pAgg->min); pInfo->max = GET_INT64_VAL(&pAgg->max); From 1cc0708d856afa2d670b046dcb27fc79c2139112 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 17 May 2022 22:42:11 +0800 Subject: [PATCH 023/103] fix(query): expand the capacity for aggregate operator. --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1ba494a041..ae9f2822d4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4246,7 +4246,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - int32_t numOfRows = 10; + int32_t numOfRows = 1024; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, numOfRows); From 52f6a66ba550a9abd98758de3d4f00f2e4a2310e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 17 May 2022 22:58:17 +0800 Subject: [PATCH 024/103] fix(query): fix some syntax error. --- source/libs/function/src/builtinsimpl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index cf67bcc7ec..de96526a25 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2533,7 +2533,7 @@ int32_t elapsedFunction(SqlFunctionCtx *pCtx) { } } } else { // computing based on the true data block - if (0 == pCtx->size) { + if (0 == pInput->numOfRows) { if (pCtx->order == TSDB_ORDER_DESC) { if (pCtx->end.key != INT64_MIN) { pInfo->min = pCtx->end.key; @@ -2552,7 +2552,7 @@ int32_t elapsedFunction(SqlFunctionCtx *pCtx) { TSKEY* ptsList = (int64_t*)colDataGetData(pCol, start); if (pCtx->order == TSDB_ORDER_DESC) { if (pCtx->start.key == INT64_MIN) { - pInfo->max = (pInfo->max < ptsList[pCtx->size - 1]) ? ptsList[pCtx->size - 1] : pInfo->max; + pInfo->max = (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? ptsList[start + pInput->numOfRows - 1] : pInfo->max; } else { pInfo->max = pCtx->start.key + 1; } @@ -2572,7 +2572,7 @@ int32_t elapsedFunction(SqlFunctionCtx *pCtx) { if (pCtx->end.key != INT64_MIN) { pInfo->max = pCtx->end.key + 1; } else { - pInfo->max = ptsList[pCtx->size - 1]; + pInfo->max = ptsList[start + pInput->numOfRows - 1]; } } } From b1f710fb8537191f7633e919432ede715b3e80af Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 00:23:27 +0800 Subject: [PATCH 025/103] enh: trace optimization --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 254d452cb3..72c198f48c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -372,13 +372,13 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, } if (level == TSDB_RETENTION_L0) { - tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L0); + tsdbDebug("vgId:%d readh:%p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L0); return VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { - tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L1); + tsdbDebug("vgId:%d readh:%p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L1); return VND_RSMA1(pVnode); } else { - tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L2); + tsdbDebug("vgId:%d readh:%p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L2); return VND_RSMA2(pVnode); } } From d588d47bdb054dc78ebcd392e6a7d2aed9e44e6d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 17 May 2022 20:35:19 +0800 Subject: [PATCH 026/103] ci(stream): add test --- tests/script/tsim/tstream/basic1.sim | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim index 37f9cb94c9..cb084ad537 100644 --- a/tests/script/tsim/tstream/basic1.sim +++ b/tests/script/tsim/tstream/basic1.sim @@ -372,4 +372,92 @@ if $data25 != 3 then return -1 endi +sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); +sleep 100 +sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 0 +if $data01 != 4 then + print ======$data01 + return -1 +endi + +if $data02 != 4 then + print ======$data02 + return -1 +endi + +if $data03 != 14 then + print ======$data03 + return -1 +endi + +if $data04 != 4 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); +sleep 100 +sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 4 then + print ======$data11 + # return -1 +endi + +if $data12 != 4 then + print ======$data12 + # return -1 +endi + +if $data13 != 10 then + print ======$data13 + # return -1 +endi + +if $data14 != 3 then + print ======$data14 + # return -1 +endi + +if $data15 != 1 then + print ======$data15 + # return -1 +endi + +# row 2 +if $data21 != 4 then + print ======$data21 + # return -1 +endi + +if $data22 != 4 then + print ======$data22 + # return -1 +endi + +if $data23 != 15 then + print ======$data23 + # return -1 +endi + +if $data24 != 4 then + print ======$data24 + # return -1 +endi + +if $data25 != 3 then + print ======$data25 + # return -1 +endi + + + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 67b0fd9261e13783fc9388a3820737531bbc9e5f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 09:25:00 +0800 Subject: [PATCH 027/103] fix: wrong db status --- source/dnode/mnode/impl/src/mndTopic.c | 20 +++++++++---------- .../script/tsim/tmq/prepareBasicEnv-1vgrp.sim | 2 +- .../script/tsim/tmq/prepareBasicEnv-4vgrp.sim | 2 +- tests/script/tsim/tmq/topic.sim | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e99046a32d..3f559cb6c0 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -502,16 +502,16 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name); - if (pTopic == NULL) { - if (dropReq.igNotExists) { - mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); - return 0; - } else { - terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); - return -1; - } - } + // if (pTopic == NULL) { + // if (dropReq.igNotExists) { + // mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); + // return 0; + // } else { + // terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + // mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + // return -1; + // } + // } if (pTopic->refConsumerCnt != 0) { mndReleaseTopic(pMnode, pTopic); diff --git a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim index fbdea96f93..43b93b7372 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-1vgrp.sim @@ -39,7 +39,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != nostrict then +if $data(db)[19] != ready then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready diff --git a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim index 8b5573486d..5e81137ffa 100644 --- a/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim +++ b/tests/script/tsim/tmq/prepareBasicEnv-4vgrp.sim @@ -39,7 +39,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != nostrict then +if $data(db)[19] != ready then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index f1dbf98bb0..9add349a18 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -31,7 +31,7 @@ sql show databases print ==> rows: $rows print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] -if $data(db)[19] != nostrict then +if $data(db)[19] != ready then sleep 100 $loop_cnt = $loop_cnt + 1 goto check_db_ready From 5c2c2b378b3040b7ba808a6ef2873dd2d62a9d4d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 18 May 2022 10:15:29 +0800 Subject: [PATCH 028/103] fix(query): fix the invalid column length value during spill out the group data into disk. --- source/common/src/tdatablock.c | 29 ++++++++++++++++++++---- source/libs/executor/src/groupoperator.c | 28 ++++++++++++++++------- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 43dcf2dfa9..4d77f4eb71 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -600,10 +600,11 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { } int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) { - pBlock->info.rows = *(int32_t*)buf; + pBlock->info.rows = *(int32_t*) buf; + pBlock->info.groupId = *(uint64_t*) (buf + sizeof(int32_t)); int32_t numOfCols = pBlock->info.numOfCols; - const char* pStart = buf + sizeof(uint32_t); + const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); @@ -669,7 +670,7 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t); } -double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { +double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); double rowSize = 0; @@ -1224,7 +1225,27 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { } size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - return (int32_t)((pageSize - blockDataGetSerialMetaSize(pBlock)) / blockDataGetSerialRowSize(pBlock)); + int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock); + + int32_t rowSize = pBlock->info.rowSize; + + int32_t nRows = payloadSize / rowSize; + + // the true value must be less than the value of nRows + int32_t additional = 0; + for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + additional += nRows * sizeof(int32_t); + } else { + additional += BitmapLen(nRows); + } + } + + int32_t newRows = (payloadSize - additional) / rowSize; + ASSERT(newRows <= nRows && newRows > 1); + + return newRows; } void colDataDestroy(SColumnInfoData* pColData) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 76a773a095..b1e19213dd 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -396,8 +396,11 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { pGInfo->groupId = calcGroupId(pInfo->keyBuf, len); } + // number of rows int32_t* rows = (int32_t*) pPage; + // group id + size_t numOfCols = pOperator->numOfExprs; for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = &pOperator->pExpr[i]; @@ -408,13 +411,13 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t bytes = pColInfoData->info.bytes; int32_t startOffset = pInfo->columnOffset[i]; - char* columnLen = NULL; - int32_t contentLen = 0; + int32_t* columnLen = NULL; + int32_t contentLen = 0; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { int32_t* offset = (int32_t*)((char*)pPage + startOffset); - columnLen = (char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; - char* data = (char*)(columnLen + sizeof(int32_t)); + columnLen = (int32_t*) ((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); + char* data = (char*)((char*) columnLen + sizeof(int32_t)); if (colDataIsNull_s(pColInfoData, j)) { offset[(*rows)] = -1; @@ -423,11 +426,15 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { offset[*rows] = (*columnLen); char* src = colDataGetData(pColInfoData, j); memcpy(data + (*columnLen), src, varDataTLen(src)); + int32_t v = (data + (*columnLen) + varDataTLen(src) - (char*)pPage); + ASSERT(v > 0); + printf("len:%d\n", v); + contentLen = varDataTLen(src); } } else { char* bitmap = (char*)pPage + startOffset; - columnLen = (char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity); + columnLen = (int32_t*) ((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); char* data = (char*) columnLen + sizeof(int32_t); bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); @@ -440,6 +447,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } (*columnLen) += contentLen; + ASSERT(*columnLen >= 0); } (*rows) += 1; @@ -476,7 +484,12 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf pPage = getNewBufPage(pInfo->pBuf, 0, &pageId); taosArrayPush(p->pPageList, &pageId); - *(int32_t*) pPage = 0; +// // number of rows +// *(int32_t*) pPage = 0; +// +// uint64_t* groupId = (pPage + sizeof(int32_t)); +// *groupId = 0; + memset(pPage, 0, getBufPageSize(pInfo->pBuf)); } } @@ -500,7 +513,7 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { size_t numOfCols = pBlock->info.numOfCols; int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t)); - offset[0] = sizeof(int32_t); // the number of rows in current page, ref to SSDataBlock paged serialization format + offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format for(int32_t i = 0; i < numOfCols - 1; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); @@ -571,7 +584,6 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { break; } - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfExprs); doHashPartition(pOperator, pBlock); } From d5c30f5ea5b2eb487a20aa89bfa3641f6efa3b7c Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 May 2022 10:29:32 +0800 Subject: [PATCH 029/103] fix: release udf func handle when udf agg failure --- source/libs/function/src/tudf.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 4322705c13..5c6209dc4d 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1386,7 +1386,7 @@ int32_t cleanUpUdfs() { if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { taosArrayPush(udfStubs, stub); } else { - fnInfo("invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", + fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", stub->udfName, stub->refCount, stub->lastRefTime); } } @@ -1602,6 +1602,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); + releaseUdfFuncHandle(pCtx->udfName); return false; } udfRes->interResNum = buf.numOfResult; @@ -1609,6 +1610,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); } else { fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); + releaseUdfFuncHandle(pCtx->udfName); return false; } freeUdfInterBuf(&buf); @@ -1674,6 +1676,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { blockDataDestroy(inputBlock); taosArrayDestroy(tempBlock.pDataBlock); + if (udfCode != 0) { + releaseUdfFuncHandle(pCtx->udfName); + } freeUdfInterBuf(&newState); return udfCode; } From e20590814266d8833cfc48ea360482f227d2f1e3 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 18 May 2022 10:57:54 +0800 Subject: [PATCH 030/103] [test: add case of tmq] --- tests/system-test/7-tmq/subscribeDb1.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index eacf231c9a..7319fadc80 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -468,14 +468,10 @@ class TDTestCase: cfgPath = buildPath + "/../sim/psim/cfg" tdLog.info("cfgPath: %s" % cfgPath) - #self.tmqCase8(cfgPath, buildPath) - #self.tmqCase9(cfgPath, buildPath) - #self.tmqCase10(cfgPath, buildPath) + self.tmqCase8(cfgPath, buildPath) + self.tmqCase9(cfgPath, buildPath) + self.tmqCase10(cfgPath, buildPath) self.tmqCase11(cfgPath, buildPath) - # self.tmqCase12(cfgPath, buildPath) - # self.tmqCase13(cfgPath, buildPath) - # self.tmqCase14(cfgPath, buildPath) - def stop(self): tdSql.close() From 325959bd27bfda3a8b1e7679bcfc8f14b2ca0404 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 18 May 2022 11:01:39 +0800 Subject: [PATCH 031/103] fix(query): add parameters check for top/bottom functoin. --- source/libs/function/src/builtins.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 24d6c1ade9..bcb4c5c585 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -14,6 +14,7 @@ */ #include "builtins.h" +#include "querynodes.h" #include "builtinsimpl.h" #include "scalar.h" #include "taoserror.h" @@ -201,15 +202,32 @@ static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_ } static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t paraNum = LIST_LENGTH(pFunc->pParameterList); + if (2 != paraNum) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, 1); + if (nodeType(pParamNode) != QUERY_NODE_VALUE) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*) pParamNode; + if (pValue->node.resType.type != TSDB_DATA_TYPE_BIGINT) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (pValue->datum.i < 1 || pValue->datum.i > 100) { + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } + SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type}; return TSDB_CODE_SUCCESS; } static int32_t translateBottom(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; - pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type}; - return TSDB_CODE_SUCCESS; + return translateTop(pFunc, pErrBuf, len); } static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { From 00c11b01fe6e8092e44bb4ea57eaa9e01b37f11a Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 May 2022 11:43:03 +0800 Subject: [PATCH 032/103] fix: udfc checks output type and output bytes when executing scalar function --- source/libs/function/src/tudf.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5c6209dc4d..068998e469 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1528,7 +1528,15 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, if (code != 0) { return code; } + SUdfcUvSession *session = handle; code = doCallUdfScalarFunc(handle, input, numOfCols, output); + if (session->outputType != output->columnData->info.type + || session->outputLen != output->columnData->info.bytes) { + fnError("udfc scalar function calculate error, session type: %d(%d), output type: %d(%d)", + session->outputType, session->outputLen, + output->columnData->info.type, output->columnData->info.bytes); + code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } releaseUdfFuncHandle(udfName); return code; } From 715aa50f4d4aaecf9c210ab5ac6d11f6952a1328 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 18 May 2022 12:03:08 +0800 Subject: [PATCH 033/103] fix:precision problems in time conversion --- source/common/src/ttime.c | 26 +++++++++++---------- tests/script/tsim/query/interval-offset.sim | 24 +------------------ 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index c121b2d83d..d6332fbbc5 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -439,6 +439,8 @@ end_: // ((double)time * factors[fromPrecision][toPrecision]); //} + +// !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = 1626006833631000064 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); @@ -446,29 +448,29 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char double tmp = time; switch (toUnit) { case 's':{ - tmp *= (NANOSECOND_PER_SEC/factors[fromPrecision]); // the result of division is an integer - time *= (NANOSECOND_PER_SEC/factors[fromPrecision]); + tmp /= (NANOSECOND_PER_SEC/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_SEC/factors[fromPrecision]); break; } case 'm': - tmp *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); // the result of division is an integer - time *= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); + tmp /= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_MINUTE/factors[fromPrecision]); break; case 'h': - tmp *= (NANOSECOND_PER_HOUR/factors[fromPrecision]); // the result of division is an integer - time *= (NANOSECOND_PER_HOUR/factors[fromPrecision]); + tmp /= (NANOSECOND_PER_HOUR/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_HOUR/factors[fromPrecision]); break; case 'd': - tmp *= (NANOSECOND_PER_DAY/factors[fromPrecision]); // the result of division is an integer - time *= (NANOSECOND_PER_DAY/factors[fromPrecision]); + tmp /= (NANOSECOND_PER_DAY/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_DAY/factors[fromPrecision]); break; case 'w': - tmp *= (NANOSECOND_PER_WEEK/factors[fromPrecision]); // the result of division is an integer - time *= (NANOSECOND_PER_WEEK/factors[fromPrecision]); + tmp /= (NANOSECOND_PER_WEEK/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_WEEK/factors[fromPrecision]); break; case 'a': - tmp *= (NANOSECOND_PER_MSEC/factors[fromPrecision]); // the result of division is an integer - time *= (NANOSECOND_PER_MSEC/factors[fromPrecision]); + tmp /= (NANOSECOND_PER_MSEC/factors[fromPrecision]); // the result of division is an integer + time /= (NANOSECOND_PER_MSEC/factors[fromPrecision]); break; case 'u': // the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 68860dc2cb..5ce3d9382f 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -177,29 +177,7 @@ if $data70 != 1 then return -1 endi -sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) -print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) -print ===> rows: $rows -print ===> rows0: $data00 $data01 $data02 $data03 $data04 -print ===> rows1: $data10 $data11 $data12 $data13 $data14 -print ===> rows2: $data20 $data21 $data22 $data23 $data24 -print ===> rows3: $data30 $data31 $data32 $data33 $data34 -if $rows != 4 then - return -1 -endi -if $data00 != @21-12-08 00:00:00.000@ then - return -1 -endi -if $data31 != 1 then - return -1 -endi -if $data34 != $data31 then - return -1 -endi -if $data02 != 2678400000 then - return -1 -endi - +sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(2w) sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(4w) From 5656c59261f1597ff30ab78e483db371b2252b8e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 18 May 2022 12:06:13 +0800 Subject: [PATCH 034/103] fix:precision problems in time conversion --- source/common/src/ttime.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d6332fbbc5..62a7179626 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -476,8 +476,8 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char // the result of (NANOSECOND_PER_USEC/(double)factors[fromPrecision]) maybe a double switch (fromPrecision) { case TSDB_TIME_PRECISION_MILLI:{ - tmp /= 1000; - time /= 1000; + tmp *= 1000; + time *= 1000; break; } case TSDB_TIME_PRECISION_MICRO:{ @@ -486,8 +486,8 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char break; } case TSDB_TIME_PRECISION_NANO:{ - tmp *= 1000; - time *= 1000; + tmp /= 1000; + time /= 1000; break; } } From 950835e1fcc6a2eaa79dbb0d3e9b1a5d9436e0be Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 May 2022 12:16:28 +0800 Subject: [PATCH 035/103] fix: set correct udf output type when retrieve from mnode --- source/libs/function/src/udfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index cbb2e7f362..006914bf65 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -139,7 +139,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdf* udf = msgInfo->param; udf->funcType = pFuncInfo->funcType; udf->scriptType = pFuncInfo->scriptType; - udf->outputType = pFuncInfo->funcType; + udf->outputType = pFuncInfo->outputType; udf->outputLen = pFuncInfo->outputLen; udf->bufSize = pFuncInfo->bufSize; From 0b8a198074868b5db6bd2018d8c79c2cbe563c29 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 18 May 2022 12:50:53 +0800 Subject: [PATCH 036/103] fix:precision problems in time conversion --- tests/script/tsim/query/interval-offset.sim | 24 ++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 5ce3d9382f..68860dc2cb 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -177,7 +177,29 @@ if $data70 != 1 then return -1 endi -sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) +sql select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 +if $rows != 4 then + return -1 +endi +if $data00 != @21-12-08 00:00:00.000@ then + return -1 +endi +if $data31 != 1 then + return -1 +endi +if $data34 != $data31 then + return -1 +endi +if $data02 != 2678400000 then + return -1 +endi + sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(2w) sql_error select _wstartts, count(tbcol), _wduration, _wstartts, count(*) from ct3 interval(1n, 1w) sliding(4w) From def7200d7caa437f6228e1e97c0e3cd4446329b4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 05:22:48 +0000 Subject: [PATCH 037/103] feat: return sver in submit rsp --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 8 ++++++++ 3 files changed, 11 insertions(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6af4325371..3ff566afe2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -258,6 +258,7 @@ typedef struct { char* tblFName; int32_t numOfRows; int32_t affectedRows; + int64_t sver; } SSubmitBlkRsp; typedef struct { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d180c2ff6e..4e97ebbe47 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4093,6 +4093,7 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl } if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; + if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1; tEndEncode(pEncoder); return 0; @@ -4111,6 +4112,7 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { } if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; + if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; tEndDecode(pDecoder); return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 037b099345..2fdbfdd206 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -309,6 +309,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo TSKEY keyMin; TSKEY keyMax; SSubmitBlk *pBlkCopy; + int64_t sverNew; // check if table exists SMetaReader mr = {0}; @@ -319,6 +320,12 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } + if (mr.me.type == TSDB_NORMAL_TABLE) { + sverNew = mr.me.ntbEntry.schema.sver; + } else { + metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid); + sverNew = mr.me.stbEntry.schema.sver; + } metaReaderClear(&mr); // create container is nedd @@ -367,6 +374,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo pRsp->numOfRows = pMsgIter->numOfRows; pRsp->affectedRows = pMsgIter->numOfRows; + pRsp->sver = sverNew; return 0; } From f8b2bdecf1d6906c1ab4580e2176e4fef439284f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 05:46:16 +0000 Subject: [PATCH 038/103] refact: TDB --- source/dnode/vnode/src/meta/metaOpen.c | 48 +++++++++++------------ source/dnode/vnode/src/meta/metaQuery.c | 14 +++---- source/dnode/vnode/src/meta/metaSma.c | 8 ++-- source/dnode/vnode/src/meta/metaTable.c | 30 +++++++------- source/dnode/vnode/src/sma/smaTDBImpl.c | 8 ++-- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 8 ++-- source/libs/tdb/inc/tdb.h | 16 ++++---- source/libs/tdb/src/db/tdbDb.c | 16 ++++---- source/libs/tdb/test/tdbTest.cpp | 42 ++++++++++---------- 9 files changed, 95 insertions(+), 95 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 3ce146904c..127eb43f2b 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -57,56 +57,56 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { } // open pTbDb - ret = tdbDbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); + ret = tdbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); if (ret < 0) { metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSkmDb - ret = tdbDbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); + ret = tdbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); if (ret < 0) { metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pUidIdx - ret = tdbDbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); + ret = tdbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); if (ret < 0) { metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pNameIdx - ret = tdbDbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); + ret = tdbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); if (ret < 0) { metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pCtbIdx - ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); + ret = tdbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTagIdx - ret = tdbDbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); + ret = tdbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTtlIdx - ret = tdbDbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); + ret = tdbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); if (ret < 0) { metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSmaIdx - ret = tdbDbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); + ret = tdbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); if (ret < 0) { metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -125,14 +125,14 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { _err: if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); + if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); @@ -142,14 +142,14 @@ _err: int metaClose(SMeta *pMeta) { if (pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbDbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbDbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb); + if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index b76258035e..f02b6402c4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -35,7 +35,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u STbDbKey tbDbKey = {.version = version, .uid = uid}; // query table.db - if (tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _err; } @@ -58,7 +58,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { int64_t version; // query uid.idx - if (tdbDbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -72,7 +72,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { tb_uid_t uid; // query name.idx - if (tdbDbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -159,7 +159,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo pKey = &skmDbKey; kLen = sizeof(skmDbKey); metaRLock(pMeta); - ret = tdbDbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); + ret = tdbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); metaULock(pMeta); if (ret < 0) { return NULL; @@ -413,7 +413,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - memcpy((void*)pTSma->expr, mr.me.smaEntry.tsma->expr, pTSma->exprLen); + memcpy((void *)pTSma->expr, mr.me.smaEntry.tsma->expr, pTSma->exprLen); } if (pTSma->tagsFilterLen > 0) { if (!(pTSma->tagsFilter = taosMemoryCalloc(1, pTSma->tagsFilterLen))) { @@ -421,14 +421,14 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { goto _err; } } - memcpy((void*)pTSma->tagsFilter, mr.me.smaEntry.tsma->tagsFilter, pTSma->tagsFilterLen); + memcpy((void *)pTSma->tagsFilter, mr.me.smaEntry.tsma->tagsFilter, pTSma->tagsFilterLen); } else { pTSma->exprLen = 0; pTSma->expr = NULL; pTSma->tagsFilterLen = 0; pTSma->tagsFilter = NULL; } - + ++smaIdx; } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index d0a3541152..1fd81bc2cb 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -130,17 +130,17 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; - return tdbDbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); + return tdbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); } static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 9064085b2b..273f7885e6 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -389,7 +389,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl int c; // search name index - ret = tdbDbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -536,7 +536,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA int nData = 0; // search name index - ret = tdbDbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -573,9 +573,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc); /* get stbEntry*/ - tdbDbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); - tdbDbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), - (void **)&stbEntry.pBuf, &nVal); + tdbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); + tdbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), + (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); tDecoderInit(&dc, stbEntry.pBuf, nVal); metaDecodeEntry(&dc, &stbEntry); @@ -632,7 +632,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaSaveToTbDb(pMeta, &ctbEntry); // save to uid.idx - tdbDbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); + tdbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); @@ -708,7 +708,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -721,11 +721,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -748,12 +748,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, @@ -801,10 +801,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SDecoder dc = {0}; // get super table - tdbDbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + tdbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = *(int64_t *)pData; - tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); tDecoderInit(&dc, pData, nData); metaDecodeEntry(&dc, &stbEntry); @@ -817,7 +817,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } - tdbDbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + tdbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); tDecoderClear(&dc); @@ -859,7 +859,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderInit(&coder, pVal, vLen); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c index cb58d9c083..6e57656802 100644 --- a/source/dnode/vnode/src/sma/smaTDBImpl.c +++ b/source/dnode/vnode/src/sma/smaTDBImpl.c @@ -59,14 +59,14 @@ static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { // Create a database compFunc = tdSmaKeyCmpr; - if(tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { + if (tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { return -1; } return 0; } -static int32_t smaCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); } +static int32_t smaCloseDBDb(TDB *pDB) { return tdbClose(pDB); } int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile @@ -99,7 +99,7 @@ int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, in int32_t ret; printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); - ret = tdbDbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { smaError("failed to upsert tsma data into db, ret = %d", ret); return -1; @@ -112,7 +112,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_ void *pVal = NULL; int ret; - ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { smaError("failed to get tsma data from db, ret = %d", ret); diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index 8a553e94fb..bed61a186f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -60,12 +60,12 @@ static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { // Create a database compFunc = tsdbSmaKeyCmpr; - ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); + ret = tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); return 0; } -static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbDbClose(pDB); } +static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbClose(pDB); } int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; @@ -110,7 +110,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32 void *pVal = NULL; int ret; - ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { tsdbError("Failed to get sma data from db, ret = %d", ret); diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 90b07fb6ae..8ba66f1d26 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -37,14 +37,14 @@ int tdbBegin(TENV *pEnv, TXN *pTxn); int tdbCommit(TENV *pEnv, TXN *pTxn); // TDB -int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); -int tdbDbClose(TDB *pDb); -int tdbDbDrop(TDB *pDb); -int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); -int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); +int tdbClose(TDB *pDb); +int tdbDrop(TDB *pDb); +int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); +int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // TDBC int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 383807cc35..fdad797333 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -24,7 +24,7 @@ struct STDBC { SBTC btc; }; -int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) { +int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) { TDB *pDb; SPager *pPager; int ret; @@ -65,7 +65,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn return 0; } -int tdbDbClose(TDB *pDb) { +int tdbClose(TDB *pDb) { if (pDb) { tdbBtreeClose(pDb->pBt); tdbOsFree(pDb); @@ -73,26 +73,26 @@ int tdbDbClose(TDB *pDb) { return 0; } -int tdbDbDrop(TDB *pDb) { +int tdbDrop(TDB *pDb) { // TODO return 0; } -int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { +int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); } -int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } +int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } -int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { +int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); } -int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { +int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } -int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { +int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index e575ac156f..3c8bcee3f7 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -131,7 +131,7 @@ TEST(tdb_test, simple_insert1) { // Create a database compFunc = tKeyCmpr; - ret = tdbDbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -152,7 +152,7 @@ TEST(tdb_test, simple_insert1) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -181,7 +181,7 @@ TEST(tdb_test, simple_insert1) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); - ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen); + ret = tdbGet(pDb, key, strlen(key), &pVal, &vLen); ASSERT(ret == 0); GTEST_ASSERT_EQ(ret, 0); @@ -224,11 +224,11 @@ TEST(tdb_test, simple_insert1) { } } - ret = tdbDbDrop(pDb); + ret = tdbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbDbClose(pDb); + tdbClose(pDb); // Close Env ret = tdbEnvClose(pEnv); @@ -251,7 +251,7 @@ TEST(tdb_test, simple_insert2) { // Create a database compFunc = tDefaultKeyCmpr; - ret = tdbDbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -271,7 +271,7 @@ TEST(tdb_test, simple_insert2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -311,11 +311,11 @@ TEST(tdb_test, simple_insert2) { tdbCommit(pEnv, &txn); tdbTxnClose(&txn); - ret = tdbDbDrop(pDb); + ret = tdbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbDbClose(pDb); + tdbClose(pDb); // Close Env ret = tdbEnvClose(pEnv); @@ -346,7 +346,7 @@ TEST(tdb_test, simple_delete1) { GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbDbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); + ret = tdbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); @@ -356,7 +356,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -365,7 +365,7 @@ TEST(tdb_test, simple_delete1) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); } @@ -374,7 +374,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); - ret = tdbDbDelete(pDb, key, strlen(key), &txn); + ret = tdbDelete(pDb, key, strlen(key), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -382,7 +382,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); - ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbGet(pDb, key, strlen(key), &pData, &nData); if (iData <= 30) { GTEST_ASSERT_EQ(ret, 0); } else { @@ -413,7 +413,7 @@ TEST(tdb_test, simple_delete1) { closePool(pPool); - tdbDbClose(pDb); + tdbClose(pDb); tdbEnvClose(pEnv); } @@ -435,7 +435,7 @@ TEST(tdb_test, simple_upsert1) { GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + ret = tdbOpen("db.db", -1, -1, NULL, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); pPool = openPool(); @@ -446,7 +446,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -454,7 +454,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } @@ -463,7 +463,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbDbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -473,11 +473,11 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } - tdbDbClose(pDb); + tdbClose(pDb); tdbEnvClose(pEnv); } \ No newline at end of file From d1078b5af23047a3e9c482f67721b8198138ef99 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 May 2022 14:06:13 +0800 Subject: [PATCH 039/103] stmt query --- source/client/src/clientStmt.c | 14 ++++++++----- source/libs/catalog/inc/catalogInt.h | 1 + source/libs/catalog/src/catalog.c | 29 +++++++++++++++++++++++++- source/libs/parser/src/parInsert.c | 1 + source/libs/parser/src/parInsertData.c | 4 ++-- source/libs/scheduler/src/scheduler.c | 13 ++++++------ tests/script/api/batchprepare.c | 8 +++---- 7 files changed, 51 insertions(+), 19 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index f2c5fdec0f..764571e71e 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -260,8 +260,8 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { - if (STMT_TYPE_QUERY != pStmt->sql.type || freeRequest) { +int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { + if (STMT_TYPE_QUERY != pStmt->sql.type || deepClean) { taos_free_result(pStmt->exec.pRequest); pStmt->exec.pRequest = NULL; } @@ -280,7 +280,11 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) { continue; } - qFreeStmtDataBlock(pBlocks); + if (STMT_TYPE_MULTI_INSERT == pStmt->sql.type) { + qFreeStmtDataBlock(pBlocks); + } else { + qDestroyStmtDataBlock(pBlocks); + } taosHashRemove(pStmt->exec.pBlockHash, key, keyLen); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -320,11 +324,11 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { taosHashCleanup(pStmt->sql.pTableCache); pStmt->sql.pTableCache = NULL; - memset(&pStmt->sql, 0, sizeof(pStmt->sql)); - STMT_ERR_RET(stmtCleanExecInfo(pStmt, false, true)); STMT_ERR_RET(stmtCleanBindInfo(pStmt)); + memset(&pStmt->sql, 0, sizeof(pStmt->sql)); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1bf21ad7d1..857c708852 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -131,6 +131,7 @@ typedef struct SCtgCacheStat { uint64_t dbNum; uint64_t tblNum; uint64_t stblNum; + uint64_t userNum; uint64_t vgHitNum; uint64_t vgMissNum; uint64_t tblHitNum; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c96ad140a1..d3045f68a1 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -122,6 +122,11 @@ void ctgFreeDbCache(SCtgDBCache *dbCache) { ctgFreeTableMetaCache(&dbCache->tbCache); } +void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { + taosHashCleanup(userCache->createdDbs); + taosHashCleanup(userCache->readDbs); + taosHashCleanup(userCache->writeDbs); +} void ctgFreeHandle(SCatalog* pCtg) { ctgFreeMetaRent(&pCtg->dbRent); @@ -145,7 +150,24 @@ void ctgFreeHandle(SCatalog* pCtg) { CTG_CACHE_STAT_SUB(dbNum, dbNum); } - + + if (pCtg->userCache) { + int32_t userNum = taosHashGetSize(pCtg->userCache); + + void *pIter = taosHashIterate(pCtg->userCache, NULL); + while (pIter) { + SCtgUserAuth *userCache = pIter; + + ctgFreeSCtgUserAuth(userCache); + + pIter = taosHashIterate(pCtg->userCache, pIter); + } + + taosHashCleanup(pCtg->userCache); + + CTG_CACHE_STAT_SUB(userNum, userNum); + } + taosMemoryFree(pCtg); } @@ -2031,10 +2053,13 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); + + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); if (!inCache) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); + continue; } @@ -2306,6 +2331,8 @@ int32_t ctgActUpdateUser(SCtgMetaAction *action) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + taosMemoryFreeClear(msg); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 01833e1776..29951ccdee 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1045,6 +1045,7 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); taosHashCleanup(pCxt->pSubTableHashObj); + taosHashCleanup(pCxt->pTableNameHashObj); destroyBlockHashmap(pCxt->pTableBlockHashObj); destroyBlockArrayList(pCxt->pVgDataBlocks); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index deb899309e..f82c792c96 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -235,12 +235,12 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); - if (!pDataBlock->cloned) { +// if (!pDataBlock->cloned) { // free the refcount for metermeta taosMemoryFreeClear(pDataBlock->pTableMeta); destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - } +// } taosMemoryFreeClear(pDataBlock); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 4fcdb35e0d..bdf674ab93 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1438,24 +1438,24 @@ int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { } int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { + SSchedulerHbRsp rsp = {0}; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; + if (code) { qError("hb rsp error:%s", tstrerror(code)); - SCH_ERR_RET(code); + SCH_ERR_JRET(code); } - SSchedulerHbRsp rsp = {0}; if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) { qError("invalid hb rsp msg, size:%d", pMsg->len); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SSchTrans trans = {0}; trans.transInst = pParam->transport; trans.transHandle = pMsg->handle; - SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans)); + SCH_ERR_JRET(schUpdateHbConnection(&rsp.epId, &trans)); int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, @@ -1483,6 +1483,7 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { _return: tFreeSSchedulerHbRsp(&rsp); + taosMemoryFree(param); SCH_RET(code); } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 546075a1d9..ba1cd00fcb 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -251,16 +251,12 @@ CaseCtrl gCaseCtrl = { .bindColNum = 0, .bindTagNum = 0, .bindRowNum = 0, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, .bindTagTypeNum = 0, .bindTagTypeList = NULL, - .optrIdxListNum = tListLen(optrIdxList), - .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = false, .runTimes = 0, - .caseIdx = 23, + .caseIdx = 1, .caseNum = 1, .caseRunIdx = -1, .caseRunNum = 1, @@ -1101,7 +1097,9 @@ void destroyData(BindData *data) { taosMemoryFree(data->binaryLen); taosMemoryFree(data->isNull); taosMemoryFree(data->pBind); + taosMemoryFree(data->pTags); taosMemoryFree(data->colTypes); + taosMemoryFree(data->sql); } void bpFetchRows(TAOS_RES *result, bool printr, int32_t *rows) { From 3cbf4ad4d2c356d22a0ebd8698ab2ebd1d11d1b2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 May 2022 14:11:23 +0800 Subject: [PATCH 040/103] fix: avoid invalid read/write --- source/dnode/mnode/sdb/src/sdbRaw.c | 7 +++--- source/libs/index/src/indexTfile.c | 13 +++++----- source/libs/index/test/jsonUT.cc | 37 ++++++++++++++++++++++++++++ source/libs/transport/src/transCli.c | 11 ++++----- 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index d09198f66f..fd2f20c242 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -213,8 +213,9 @@ int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valL terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; return -1; } - - memcpy(pVal, pRaw->pData + dataPos, valLen); + if (pVal != NULL) { + memcpy(pVal, pRaw->pData + dataPos, valLen); + } return 0; } @@ -235,4 +236,4 @@ int32_t sdbGetRawTotalSize(SSdbRaw *pRaw) { } return sizeof(SSdbRaw) + pRaw->dataLen; -} \ No newline at end of file +} diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 9533d3429e..163bb53163 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -473,17 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR int32_t sz = 0; char* ch = (char*)fstSliceData(s, &sz); - // char* tmp = taosMemoryCalloc(1, sz + 1); - // memcpy(tmp, ch, sz); + char* tmp = taosMemoryCalloc(1, sz + 1); + memcpy(tmp, ch, sz); - if (0 != strncmp(ch, p, skip)) { + if (0 != strncmp(tmp, p, skip)) { swsResultDestroy(rt); - // taosMemoryFree(tmp); + taosMemoryFree(tmp); break; } - TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); - + TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType)); if (MATCH == cond) { tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total); } else if (CONTINUE == cond) { @@ -491,7 +490,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR swsResultDestroy(rt); break; } - // taosMemoryFree(tmp); + taosMemoryFree(tmp); swsResultDestroy(rt); } streamWithStateDestroy(st); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 135ae61e83..f0c6455442 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -578,3 +578,40 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) { EXPECT_EQ(1000, taosArrayGetSize(res)); } } +TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) { + { + double val = 10.0; + std::string colName("test1"); + for (int i = 0; i < 1000; i++) { + WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); + } + } + { + double val = 2.0; + std::string colName("test1"); + for (int i = 0; i < 1000; i++) { + WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); + } + } + { + SArray* res = NULL; + std::string colName("test1"); + double val = 1.9; + Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(2000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + std::string colName("test1"); + double val = 2.1; + Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } + { + std::string colName("test1"); + SArray* res = NULL; + double val = 2.1; + Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } +} diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d2d38bd9bc..92c5e9faf7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -145,9 +145,9 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); } while (0) #define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) -#define CONN_PERSIST_TIME(para) (para * 1000 * 10) -#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_PERSIST_TIME(para) (para * 1000 * 10) +#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ @@ -227,7 +227,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); #define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) -#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) +#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) static void* cliWorkThread(void* arg); @@ -924,8 +924,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { arg->param1 = pMsg; arg->param2 = pThrd; transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - - cliDestroy((uv_handle_t*)pConn->stream); + cliDestroyConn(pConn, true); return -1; } } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { From 77575d1e70d999b8b96a2476d61e752f017a6f3e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 14:20:33 +0800 Subject: [PATCH 041/103] refactor: shm queue in multi process mode --- include/common/tmsgcb.h | 8 +- source/common/src/tmsgcb.c | 12 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 + source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 1 + source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 8 +- source/dnode/mgmt/node_mgmt/src/dmProc.c | 237 +++++++----------- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 26 +- source/dnode/mnode/impl/src/mnode.c | 2 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 3 +- source/libs/sync/src/syncIO.c | 1 + source/libs/wal/src/walMgmt.c | 1 + source/util/src/tlog.c | 1 + source/util/src/tsched.c | 1 + source/util/src/tworker.c | 2 + tests/test/c/create_table.c | 1 + tests/test/c/tmqSim.c | 1 + tests/tsim/src/simSystem.c | 1 + tools/shell/src/shellEngine.c | 1 + 18 files changed, 135 insertions(+), 174 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 679e3ba775..7ba6e5044c 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -41,8 +41,8 @@ typedef enum { typedef int32_t (*PutToQueueFp)(void* pMgmt, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); -typedef void (*SendRspFp)(const SRpcMsg* pMsg); -typedef void (*SendRedirectRspFp)(const SRpcMsg* pMsg, const SEpSet* pNewEpSet); +typedef void (*SendRspFp)(SRpcMsg* pMsg); +typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); @@ -64,8 +64,8 @@ void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg); int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); -void tmsgSendRsp(const SRpcMsg* pMsg); -void tmsgSendRedirectRsp(const SRpcMsg* pMsg, const SEpSet* pNewEpSet); +void tmsgSendRsp(SRpcMsg* pMsg); +void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index d28e2c675d..f69fb65f04 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -21,9 +21,9 @@ static SMsgCb tsDefaultMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } -int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { +int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg) { PutToQueueFp fp = pMsgCb->queueFps[qtype]; - return (*fp)(pMsgCb->mgmt, pReq); + return (*fp)(pMsgCb->mgmt, pMsg); } int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { @@ -31,17 +31,17 @@ int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { return (*fp)(pMsgCb->mgmt, vgId, qtype); } -int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pReq) { +int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { SendReqFp fp = tsDefaultMsgCb.sendReqFp; - return (*fp)(epSet, pReq); + return (*fp)(epSet, pMsg); } -void tmsgSendRsp(const SRpcMsg* pMsg) { +void tmsgSendRsp(SRpcMsg* pMsg) { SendRspFp fp = tsDefaultMsgCb.sendRspFp; return (*fp)(pMsg); } -void tmsgSendRedirectRsp(const SRpcMsg* pMsg, const SEpSet* pNewEpSet) { +void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; (*fp)(pMsg, pNewEpSet); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 7daf25bb8a..787c6c98c8 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -75,6 +75,7 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { void dmStopStatusThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->statusThread)) { taosThreadJoin(pMgmt->statusThread, NULL); + taosThreadClear(&pMgmt->statusThread); } } @@ -95,6 +96,7 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { void dmStopMonitorThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->monitorThread)) { taosThreadJoin(pMgmt->monitorThread, NULL); + taosThreadClear(&pMgmt->monitorThread); } } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index ab41ee5df3..8c3b8576a8 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -196,6 +196,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { SVnodeThread *pThread = &threads[t]; if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { taosThreadJoin(pThread->thread, NULL); + taosThreadClear(&pThread->thread); } taosMemoryFree(pThread->pCfgs); } diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 2d03267fe8..7484c1e18f 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -151,12 +151,10 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper); void dmCleanupProc(struct SMgmtWrapper *pWrapper); int32_t dmRunProc(SProc *proc); void dmStopProc(SProc *proc); -int64_t dmRemoveProcRpcHandle(SProc *proc, void *handle); +void dmRemoveProcRpcHandle(SProc *proc, void *handle); void dmCloseProcRpcHandles(SProc *proc); -int32_t dmPutToProcCQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - void *handle, int64_t handleRef, EProcFuncType ftype); -void dmPutToProcPQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - EProcFuncType ftype); +int32_t dmPutToProcCQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype); +void dmPutToProcPQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype); // dmTransport.c int32_t dmInitServer(SDnode *pDnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index f9875f7182..2e24e3fa1c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -16,10 +16,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" -static inline int32_t CEIL8(int32_t v) { - const int32_t c = ceil((float)(v) / 8) * 8; - return c < 8 ? 8 : c; -} +static inline int32_t CEIL8(int32_t v) { return ceil((float)(v) / 8) * 8; } static int32_t dmInitProcMutex(SProcQueue *queue) { TdThreadMutexAttr mattr = {0}; @@ -87,42 +84,17 @@ static SProcQueue *dmInitProcQueue(SProc *proc, char *ptr, int32_t size) { return queue; } -#if 0 -static void dmDestroyProcQueue(SProcQueue *queue) { - if (queue->mutex != NULL) { - taosThreadMutexDestroy(queue->mutex); - queue->mutex = NULL; - } -} +static void dmCleanupProcQueue(SProcQueue *queue) {} -static void dmDestroyProcSem(SProcQueue *queue) { - if (queue->sem != NULL) { - tsem_destroy(queue->sem); - queue->sem = NULL; - } -} -#endif - -static void dmCleanupProcQueue(SProcQueue *queue) { -#if 0 - if (queue != NULL) { - dmDestroyProcQueue(queue); - dmDestroyProcSem(queue); - } -#endif -} - -static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHead, int16_t rawHeadLen, - const char *pBody, int32_t rawBodyLen, int64_t handle, int64_t handleRef, - EProcFuncType ftype) { - if (rawHeadLen == 0 || pHead == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - return -1; - } - - const int32_t headLen = CEIL8(rawHeadLen); +static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg *pMsg, EProcFuncType ftype) { + const void *pHead = pMsg; + const void *pBody = pMsg->pCont; + const int16_t rawHeadLen = sizeof(SRpcMsg); + const int32_t rawBodyLen = pMsg->contLen; + const int16_t headLen = CEIL8(rawHeadLen); const int32_t bodyLen = CEIL8(rawBodyLen); const int32_t fullLen = headLen + bodyLen + 8; + const int64_t handle = (int64_t)pMsg->info.handle; taosThreadMutexLock(&queue->mutex); if (fullLen > queue->avail) { @@ -131,8 +103,8 @@ static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHe return -1; } - if (handle != 0 && ftype == DND_FUNC_REQ) { - if (taosHashPut(proc->hash, &handle, sizeof(int64_t), &handleRef, sizeof(int64_t)) != 0) { + if (ftype == DND_FUNC_REQ && IsReq(pMsg) && pMsg->code == 0 && handle != 0) { + if (taosHashPut(proc->hash, &handle, sizeof(int64_t), &pMsg->info, sizeof(SRpcConnInfo)) != 0) { taosThreadMutexUnlock(&queue->mutex); return -1; } @@ -151,31 +123,31 @@ static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHe if (queue->tail < queue->head) { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); - memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, rawBodyLen); + if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, rawBodyLen); queue->tail = queue->tail + 8 + headLen + bodyLen; } else { int32_t remain = queue->total - queue->tail; if (remain == 0) { memcpy(queue->pBuffer + 8, pHead, rawHeadLen); - memcpy(queue->pBuffer + 8 + headLen, pBody, rawBodyLen); + if (rawBodyLen > 0) memcpy(queue->pBuffer + 8 + headLen, pBody, rawBodyLen); queue->tail = 8 + headLen + bodyLen; } else if (remain == 8) { memcpy(queue->pBuffer, pHead, rawHeadLen); - memcpy(queue->pBuffer + headLen, pBody, rawBodyLen); + if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen, pBody, rawBodyLen); queue->tail = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, remain - 8); memcpy(queue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8)); - memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); + if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); queue->tail = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); - memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen); - memcpy(queue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen); + if (rawBodyLen > 0) memcpy(queue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); queue->tail = bodyLen - (remain - 8 - headLen); } else { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); - memcpy(queue->pBuffer + queue->tail + headLen + 8, pBody, rawBodyLen); + if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + headLen + 8, pBody, rawBodyLen); queue->tail = queue->tail + headLen + bodyLen + 8; } } @@ -185,13 +157,12 @@ static int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, const char *pHe taosThreadMutexUnlock(&queue->mutex); tsem_post(&queue->sem); - dTrace("node:%s, push %s msg:%p:%d cont:%p:%d, pos:%d remain:%d", queue->name, dmFuncStr(ftype), pHead, headLen, - pBody, bodyLen, pos, queue->items); + dTrace("node:%s, push %s msg:%p type:%d handle:%p len:%d code:0x%x, pos:%d remain:%d", queue->name, dmFuncStr(ftype), + pMsg, pMsg->msgType, pMsg->info.handle, pMsg->contLen, pMsg->code, pos, queue->items); return 0; } -static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHeadLen, void **ppBody, int32_t *pBodyLen, - EProcFuncType *pFuncType) { +static int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) { tsem_wait(&queue->sem); taosThreadMutexLock(&queue->mutex); @@ -217,8 +188,9 @@ static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHe int32_t bodyLen = CEIL8(rawBodyLen); void *pHead = taosAllocateQitem(headLen, DEF_QITEM); - void *pBody = rpcMallocCont(bodyLen); - if (pHead == NULL || pBody == NULL) { + void *pBody = NULL; + if (bodyLen > 0) pBody = rpcMallocCont(bodyLen); + if (pHead == NULL || (bodyLen > 0 && pBody == NULL)) { taosThreadMutexUnlock(&queue->mutex); tsem_post(&queue->sem); taosFreeQitem(pHead); @@ -230,31 +202,31 @@ static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHe const int32_t pos = queue->head; if (queue->head < queue->tail) { memcpy(pHead, queue->pBuffer + queue->head + 8, headLen); - memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, bodyLen); + if (bodyLen > 0) memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, bodyLen); queue->head = queue->head + 8 + headLen + bodyLen; } else { int32_t remain = queue->total - queue->head; if (remain == 0) { memcpy(pHead, queue->pBuffer + 8, headLen); - memcpy(pBody, queue->pBuffer + 8 + headLen, bodyLen); + if (bodyLen > 0) memcpy(pBody, queue->pBuffer + 8 + headLen, bodyLen); queue->head = 8 + headLen + bodyLen; } else if (remain == 8) { memcpy(pHead, queue->pBuffer, headLen); - memcpy(pBody, queue->pBuffer + headLen, bodyLen); + if (bodyLen > 0) memcpy(pBody, queue->pBuffer + headLen, bodyLen); queue->head = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(pHead, queue->pBuffer + queue->head + 8, remain - 8); memcpy((char *)pHead + remain - 8, queue->pBuffer, headLen - (remain - 8)); - memcpy(pBody, queue->pBuffer + headLen - (remain - 8), bodyLen); + if (bodyLen > 0) memcpy(pBody, queue->pBuffer + headLen - (remain - 8), bodyLen); queue->head = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(pHead, queue->pBuffer + queue->head + 8, headLen); - memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, remain - 8 - headLen); - memcpy((char *)pBody + remain - 8 - headLen, queue->pBuffer, bodyLen - (remain - 8 - headLen)); + if (bodyLen > 0) memcpy(pBody, queue->pBuffer + queue->head + 8 + headLen, remain - 8 - headLen); + if (bodyLen > 0) memcpy((char *)pBody + remain - 8 - headLen, queue->pBuffer, bodyLen - (remain - 8 - headLen)); queue->head = bodyLen - (remain - 8 - headLen); } else { memcpy(pHead, queue->pBuffer + queue->head + 8, headLen); - memcpy(pBody, queue->pBuffer + queue->head + headLen + 8, bodyLen); + if (bodyLen > 0) memcpy(pBody, queue->pBuffer + queue->head + headLen + 8, bodyLen); queue->head = queue->head + headLen + bodyLen + 8; } } @@ -263,14 +235,12 @@ static int32_t dmPopFromProcQueue(SProcQueue *queue, void **ppHead, int16_t *pHe queue->items--; taosThreadMutexUnlock(&queue->mutex); - *ppHead = pHead; - *ppBody = pBody; - *pHeadLen = rawHeadLen; - *pBodyLen = rawBodyLen; + *ppMsg = pHead; + (*ppMsg)->pCont = pBody; *pFuncType = (EProcFuncType)ftype; - dTrace("node:%s, pop %s msg:%p:%d cont:%p:%d, pos:%d remain:%d", queue->name, dmFuncStr(ftype), pHead, headLen, pBody, - bodyLen, pos, queue->items); + dTrace("node:%s, pop %s msg:%p type:%d handle:%p len:%d code:0x%x, pos:%d remain:%d", queue->name, dmFuncStr(ftype), + (*ppMsg), (*ppMsg)->msgType, (*ppMsg)->info.handle, (*ppMsg)->contLen, (*ppMsg)->code, pos, queue->items); return 1; } @@ -308,18 +278,14 @@ static void *dmConsumChildQueue(void *param) { SProc *proc = param; SMgmtWrapper *pWrapper = proc->wrapper; SProcQueue *queue = proc->cqueue; - void *pHead = NULL; - void *pBody = NULL; - int16_t headLen = 0; - int32_t bodyLen = 0; int32_t numOfMsgs = 0; int32_t code = 0; EProcFuncType ftype = DND_FUNC_REQ; - SRpcMsg *pReq = NULL; + SRpcMsg *pMsg = NULL; dDebug("node:%s, start to consume from cqueue", proc->name); do { - numOfMsgs = dmPopFromProcQueue(queue, &pHead, &headLen, &pBody, &bodyLen, &ftype); + numOfMsgs = dmPopFromProcQueue(queue, &pMsg, &ftype); if (numOfMsgs == 0) { dDebug("node:%s, get no msg from cqueue and exit thread", proc->name); break; @@ -332,25 +298,24 @@ static void *dmConsumChildQueue(void *param) { } if (ftype != DND_FUNC_REQ) { - dFatal("node:%s, get msg:%p from cqueue, invalid ftype:%d", proc->name, pHead, ftype); - taosFreeQitem(pHead); - rpcFreeCont(pBody); - } else { - pReq = pHead; - pReq->pCont = pBody; - code = dmProcessNodeMsg(pWrapper, pReq); - if (code != 0) { - dError("node:%s, failed to process msg:%p since %s, put into pqueue", proc->name, pReq, terrstr()); - SRpcMsg rspMsg = { - .info = pReq->info, - .pCont = pReq->info.rsp, - .contLen = pReq->info.rspLen, - }; - dmPutToProcPQueue(proc, &rspMsg, sizeof(SRpcMsg), rspMsg.pCont, rspMsg.contLen, DND_FUNC_RSP); - taosFreeQitem(pHead); - rpcFreeCont(pBody); - rpcFreeCont(rspMsg.pCont); - } + dError("node:%s, invalid ftype:%d from cqueue", proc->name, ftype); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + continue; + } + + code = dmProcessNodeMsg(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to process msg:%p since %s, put into pqueue", proc->name, pMsg, terrstr()); + SRpcMsg rsp = { + .code = (terrno != 0 ? terrno : code), + .pCont = pMsg->info.rsp, + .contLen = pMsg->info.rspLen, + .info = pMsg->info, + }; + dmPutToProcPQueue(proc, &rsp, DND_FUNC_RSP); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } } while (1); @@ -361,18 +326,14 @@ static void *dmConsumParentQueue(void *param) { SProc *proc = param; SMgmtWrapper *pWrapper = proc->wrapper; SProcQueue *queue = proc->pqueue; - void *pHead = NULL; - void *pBody = NULL; - int16_t headLen = 0; - int32_t bodyLen = 0; int32_t numOfMsgs = 0; int32_t code = 0; EProcFuncType ftype = DND_FUNC_REQ; - SRpcMsg *pRsp = NULL; + SRpcMsg *pMsg = NULL; dDebug("node:%s, start to consume from pqueue", proc->name); do { - numOfMsgs = dmPopFromProcQueue(queue, &pHead, &headLen, &pBody, &bodyLen, &ftype); + numOfMsgs = dmPopFromProcQueue(queue, &pMsg, &ftype); if (numOfMsgs == 0) { dDebug("node:%s, get no msg from pqueue and exit thread", proc->name); break; @@ -385,31 +346,19 @@ static void *dmConsumParentQueue(void *param) { } if (ftype == DND_FUNC_RSP) { - pRsp = pHead; - pRsp->pCont = pBody; - dTrace("node:%s, get rsp msg:%p from pqueue, code:0x%04x handle:%p", proc->name, pRsp, code, pRsp->info.handle); - dmRemoveProcRpcHandle(proc, pRsp->info.handle); - rpcSendResponse(pRsp); + dmRemoveProcRpcHandle(proc, pMsg->info.handle); + rpcSendResponse(pMsg); } else if (ftype == DND_FUNC_REGIST) { - pRsp = pHead; - pRsp->pCont = pBody; - dTrace("node:%s, get regist msg:%p from pqueue, code:0x%04x handle:%p", proc->name, pRsp, code, - pRsp->info.handle); - rpcRegisterBrokenLinkArg(pRsp); + rpcRegisterBrokenLinkArg(pMsg); } else if (ftype == DND_FUNC_RELEASE) { - pRsp = pHead; - pRsp->pCont = NULL; - dTrace("node:%s, get release msg:%p from pqueue, code:0x%04x handle:%p", proc->name, pRsp, code, - pRsp->info.handle); - dmRemoveProcRpcHandle(proc, pRsp->info.handle); - rpcReleaseHandle(pRsp->info.handle, (int8_t)pRsp->code); - rpcFreeCont(pBody); + dmRemoveProcRpcHandle(proc, pMsg->info.handle); + rpcReleaseHandle(pMsg->info.handle, (int8_t)pMsg->code); } else { - dFatal("node:%s, get msg:%p from pqueue, invalid ftype:%d", proc->name, pHead, ftype); - rpcFreeCont(pBody); + dError("node:%s, invalid ftype:%d from pqueue", proc->name, ftype); + rpcFreeCont(pMsg->pCont); } - taosFreeQitem(pHead); + taosFreeQitem(pMsg); } while (1); return NULL; @@ -468,51 +417,55 @@ void dmCleanupProc(struct SMgmtWrapper *pWrapper) { dmCleanupProcQueue(proc->cqueue); dmCleanupProcQueue(proc->pqueue); taosHashCleanup(proc->hash); + proc->hash = NULL; dDebug("node:%s, proc is cleaned up", pWrapper->name); } -int64_t dmRemoveProcRpcHandle(SProc *proc, void *handle) { +void dmRemoveProcRpcHandle(SProc *proc, void *handle) { int64_t h = (int64_t)handle; taosThreadMutexLock(&proc->cqueue->mutex); - - int64_t *pRef = taosHashGet(proc->hash, &h, sizeof(int64_t)); - int64_t ref = 0; - if (pRef != NULL) { - ref = *pRef; - } - taosHashRemove(proc->hash, &h, sizeof(int64_t)); taosThreadMutexUnlock(&proc->cqueue->mutex); - - return ref; } void dmCloseProcRpcHandles(SProc *proc) { taosThreadMutexLock(&proc->cqueue->mutex); - void *h = taosHashIterate(proc->hash, NULL); - while (h != NULL) { - void *handle = *((void **)h); - h = taosHashIterate(proc->hash, h); - - dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, handle); - SRpcMsg rpcMsg = {.info.handle = handle, .code = TSDB_CODE_NODE_OFFLINE}; + SRpcHandleInfo *pInfo = taosHashIterate(proc->hash, NULL); + while (pInfo != NULL) { + dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, pInfo->handle); + SRpcMsg rpcMsg = {.info = *pInfo, .code = TSDB_CODE_NODE_OFFLINE}; rpcSendResponse(&rpcMsg); + pInfo = taosHashIterate(proc->hash, pInfo); } taosHashClear(proc->hash); taosThreadMutexUnlock(&proc->cqueue->mutex); } -void dmPutToProcPQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - EProcFuncType ftype) { +void dmPutToProcPQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) { int32_t retry = 0; - while (dmPushToProcQueue(proc, proc->pqueue, pHead, headLen, pBody, bodyLen, 0, 0, ftype) != 0) { - dWarn("node:%s, failed to put msg:%p to pqueue since %s, retry:%d", proc->name, pHead, terrstr(), retry); - retry++; - taosMsleep(retry); + while (1) { + if (dmPushToProcQueue(proc, proc->pqueue, pMsg, ftype) == 0) { + break; + } + + if (retry == 10) { + pMsg->code = terrno; + if (pMsg->contLen > 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + pMsg->contLen = 0; + } + dError("node:%s, failed to push %s msg:%p type:%d handle:%p then discard data and return error", proc->name, + dmFuncStr(ftype), pMsg, pMsg->msgType, pMsg->info.handle); + } else { + dError("node:%s, failed to push %s msg:%p type:%d handle:%p len:%d since %s, retry:%d", proc->name, + dmFuncStr(ftype), pMsg, pMsg->msgType, pMsg->info.handle, pMsg->contLen, terrstr(), retry); + retry++; + taosMsleep(retry); + } } } -int32_t dmPutToProcCQueue(SProc *proc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - void *handle, int64_t ref, EProcFuncType ftype) { - return dmPushToProcQueue(proc, proc->cqueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, ref, ftype); +int32_t dmPutToProcCQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) { + return dmPushToProcQueue(proc, proc->cqueue, pMsg, ftype); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 2efa341704..c9100aab9d 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -128,9 +128,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } if (InParentProc(pWrapper)) { - code = dmPutToProcCQueue(&pWrapper->proc, pMsg, sizeof(SRpcMsg), pRpc->pCont, pRpc->contLen, - (IsReq(pRpc) && (pRpc->code == 0)) ? pRpc->info.handle : NULL, pRpc->info.refId, - DND_FUNC_REQ); + code = dmPutToProcCQueue(&pWrapper->proc, pMsg, DND_FUNC_REQ); } else { code = dmProcessNodeMsg(pWrapper, pMsg); } @@ -255,23 +253,23 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pReq) { } } -static inline void dmSendRsp(const SRpcMsg *pMsg) { +static inline void dmSendRsp(SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = pMsg->info.wrapper; - if (InChildProc(pWrapper)) { - dmPutToProcPQueue(&pWrapper->proc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, DND_FUNC_RSP); + if (pMsg->code == TSDB_CODE_NODE_REDIRECT) { + dmSendRpcRedirectRsp(pMsg); } else { - if (pMsg->code == TSDB_CODE_NODE_REDIRECT) { - dmSendRpcRedirectRsp(pMsg); + if (InChildProc(pWrapper)) { + dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP); } else { rpcSendResponse(pMsg); } } } -static inline void dmSendRedirectRsp(const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { - SMgmtWrapper *pWrapper = pRsp->info.wrapper; +static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { + SMgmtWrapper *pWrapper = pMsg->info.wrapper; if (InChildProc(pWrapper)) { - dmPutToProcPQueue(&pWrapper->proc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, DND_FUNC_RSP); + dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP); } else { SRpcMsg rsp = {0}; SMEpSet msg = {.epSet = *pNewEpSet}; @@ -281,7 +279,7 @@ static inline void dmSendRedirectRsp(const SRpcMsg *pRsp, const SEpSet *pNewEpSe tSerializeSMEpSet(rsp.pCont, len, &msg); rsp.code = TSDB_CODE_RPC_REDIRECT; - rsp.info = pRsp->info; + rsp.info = pMsg->info; rpcSendResponse(&rsp); } } @@ -289,7 +287,7 @@ static inline void dmSendRedirectRsp(const SRpcMsg *pRsp, const SEpSet *pNewEpSe static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = pMsg->info.wrapper; if (InChildProc(pWrapper)) { - dmPutToProcPQueue(&pWrapper->proc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, DND_FUNC_REGIST); + dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_REGIST); } else { rpcRegisterBrokenLinkArg(pMsg); } @@ -299,7 +297,7 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { SMgmtWrapper *pWrapper = pHandle->wrapper; if (InChildProc(pWrapper)) { SRpcMsg msg = {.code = type, .info = *pHandle}; - dmPutToProcPQueue(&pWrapper->proc, &msg, sizeof(SRpcMsg), NULL, 0, DND_FUNC_RELEASE); + dmPutToProcPQueue(&pWrapper->proc, &msg, DND_FUNC_RELEASE); } else { rpcReleaseHandle(pHandle->handle, type); } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 2756d2203e..3dfba4eca7 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -122,7 +122,7 @@ static void mndCleanupTimer(SMnode *pMnode) { pMnode->stopped = true; if (taosCheckPthreadValid(pMnode->thread)) { taosThreadJoin(pMnode->thread, NULL); - memset(&pMnode->thread, 0, sizeof(pMnode->thread)); + taosThreadClear(&pMnode->thread); } } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index efaa8bb7d3..82acd7fddc 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -16,10 +16,9 @@ #include "tcache.h" void reportStartup(const char *name, const char *desc) {} -void sendRsp(const SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } +void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { - // rpcFreeCont(pMsg->pCont); terrno = TSDB_CODE_INVALID_PTR; return -1; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 1117528b53..203a8a1e62 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -237,6 +237,7 @@ static int32_t syncIOStopInternal(SSyncIO *io) { int32_t ret = 0; atomic_store_8(&io->isStart, 0); taosThreadJoin(io->consumerTid, NULL); + taosThreadClear(&io->consumerTid); taosTmrCleanUp(io->timerMgr); return ret; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index cf40c998de..ada1f599f2 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -244,6 +244,7 @@ static void walStopThread() { if (taosCheckPthreadValid(tsWal.thread)) { taosThreadJoin(tsWal.thread, NULL); + taosThreadClear(&tsWal.thread); } wDebug("wal thread is stopped"); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 46403833f1..c1fc2c48c0 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -145,6 +145,7 @@ void taosCloseLog() { taosStopLog(); if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); + taosThreadClear(&tsLogObj.logHandle->asyncThread); } tsLogInited = 0; diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 2deba5077b..ee1f418561 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -209,6 +209,7 @@ void taosCleanUpScheduler(void *param) { for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { taosThreadJoin(pSched->qthread[i], NULL); + taosThreadClear(&pSched->qthread[i]); } } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 7fc390e70b..dc48fc3f8d 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -57,6 +57,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) { if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { taosThreadJoin(worker->thread, NULL); + taosThreadClear(&worker->thread); } } @@ -179,6 +180,7 @@ void tWWorkerCleanup(SWWorkerPool *pool) { SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { taosThreadJoin(worker->thread, NULL); + taosThreadClear(&worker->thread); taosFreeQall(worker->qall); taosCloseQset(worker->qset); } diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 4fca7d6245..c53ae0136c 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -436,6 +436,7 @@ int32_t main(int32_t argc, char *argv[]) { taosMsleep(300); for (int32_t i = 0; i < numOfThreads; i++) { taosThreadJoin(pInfo[i].thread, NULL); + taosThreadClear(&pInfo[i].thread); } int64_t maxDelay = 0; diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 55aa7a8d31..cf113369bc 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -537,6 +537,7 @@ int main(int32_t argc, char* argv[]) { for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stThreads[i].thread); } // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 69f15a164e..969332ba5f 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -56,6 +56,7 @@ void simFreeScript(SScript *script) { bgScript->killed = true; if (taosCheckPthreadValid(bgScript->bgPid)) { taosThreadJoin(bgScript->bgPid, NULL); + taosThreadClear(&bgScript->bgPid); } simDebug("script:%s, background thread joined", bgScript->fileName); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4825aae699..9f9c8821b0 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -985,6 +985,7 @@ int32_t shellExecute() { while (1) { taosThreadCreate(&shell.pid, NULL, shellThreadLoop, shell.conn); taosThreadJoin(shell.pid, NULL); + taosThreadClear(&shell.pid); } return 0; From bf44937dca41e55e44bed19749c7138c09561eac Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 18 May 2022 14:21:04 +0800 Subject: [PATCH 042/103] fix(os): win compile error --- contrib/CMakeLists.txt | 6 +++--- source/libs/index/src/indexComm.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 97bfcfb8c0..aba955ff3b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -226,10 +226,10 @@ endif(${BUILD_WITH_NURAFT}) if(${BUILD_PTHREAD}) set(CMAKE_BUILD_TYPE release) add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) + add_subdirectory(pthread EXCLUDE_FROM_ALL) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) - add_library(pthread STATIC IMPORTED GLOBAL) - SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) + add_library(pthread INTERFACE) + target_link_libraries(pthread INTERFACE libpthreadVC3) endif() # iconv diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 74e2861037..8763a8ab3f 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -361,7 +361,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - *dst = *dst - tlen; + *dst = (char*) * dst - tlen; break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY From de0793b6118cee51025755aaf68868c40b0aeb98 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 May 2022 14:40:14 +0800 Subject: [PATCH 043/103] fix: avoid invalid read/write --- source/libs/index/test/jsonUT.cc | 38 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index f0c6455442..86592f0cf3 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -580,38 +580,46 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) { } TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) { { - double val = 10.0; - std::string colName("test1"); + double val = 10.0; for (int i = 0; i < 1000; i++) { - WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); + WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); } } { - double val = 2.0; - std::string colName("test1"); + double val = 2.0; for (int i = 0; i < 1000; i++) { - WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); + WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); } } { SArray* res = NULL; std::string colName("test1"); double val = 1.9; - Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); EXPECT_EQ(2000, taosArrayGetSize(res)); } { - SArray* res = NULL; - std::string colName("test1"); - double val = 2.1; - Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + SArray* res = NULL; + double val = 2.1; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); EXPECT_EQ(1000, taosArrayGetSize(res)); } { - std::string colName("test1"); - SArray* res = NULL; - double val = 2.1; - Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + SArray* res = NULL; + double val = 2.1; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 10.0; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_EQUAL, &res); + EXPECT_EQ(2000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 10.0; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_THAN, &res); EXPECT_EQ(1000, taosArrayGetSize(res)); } } From 0c24651319d010929d2082e1954576e269d0439d Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 14:46:30 +0800 Subject: [PATCH 044/103] enh: add windows build test to ci --- Jenkinsfile2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index cbf663cdcf..bc2c9ff494 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -226,6 +226,13 @@ pipeline { stages { stage('run test') { parallel { + stage('windows test') { + agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} + steps { + pre_test_win() + pre_test_build_win() + } + } stage('linux test') { agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } From f4ef09bf6b8c164a23df529acf15f2c9f768e76c Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 15:11:54 +0800 Subject: [PATCH 045/103] enh: print ip address before build --- Jenkinsfile2 | 52 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index bc2c9ff494..20a86dbcea 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -117,27 +117,29 @@ def pre_test(){ def pre_test_win(){ bat ''' hostname + ipconfig + set date /t time /t - taskkill /f /t /im python.exe - taskkill /f /t /im bash.exe rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug - exit 0 ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git reset --hard git fetch || git fetch + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git reset --hard git fetch || git fetch - git checkout -f ''' script { if (env.CHANGE_TARGET == 'master') { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout master + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout master ''' @@ -145,6 +147,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 2.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 2.0 ''' @@ -152,6 +156,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 3.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 3.0 ''' @@ -159,6 +165,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout develop + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout develop ''' @@ -169,30 +177,50 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout -qf FETCH_HEAD - git log -5 ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout -qf FETCH_HEAD - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git log -5 ''' } else { - sh ''' - echo "unmatched reposiotry ${CHANGE_URL}" + bat ''' + echo "unmatched reposiotry %CHANGE_URL%" ''' } } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git log -5 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git log -5 + ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git submodule update --init --recursive From 292e9afd3da70583a26fde3fdc9923e9c6d54e5f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 15:14:23 +0800 Subject: [PATCH 046/103] fix: restart killed child process --- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index ff9d4089cd..f23db5da16 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -58,6 +58,7 @@ static int32_t dmNewProc(SMgmtWrapper *pWrapper, EDndNodeType ntype) { return -1; } + taosIgnSignal(SIGCHLD); pWrapper->proc.pid = pid; dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid); return 0; @@ -254,7 +255,7 @@ static void dmWatchNodes(SDnode *pDnode) { if (!OnlyInParentProc(pWrapper)) continue; if (proc->pid <= 0 || !taosProcExist(proc->pid)) { - dWarn("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid); + dError("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid); dmCloseProcRpcHandles(&pWrapper->proc); dmNewProc(pWrapper, ntype); } From 9e71d03d37dc1a3de0bcd8061bfb6bf7d5e6f705 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 18 May 2022 15:31:05 +0800 Subject: [PATCH 047/103] enh(tmq): commit when consumer_close called --- source/client/src/tmq.c | 39 +++++++++++++++++------ source/dnode/mnode/impl/src/mndConsumer.c | 27 +++++++++------- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ecfc991331..f7ddca8f69 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -119,7 +119,7 @@ enum { enum { TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__READY, - TMQ_CONSUMER_STATUS__NO_TOPIC, + /*TMQ_CONSUMER_STATUS__NO_TOPIC,*/ }; enum { @@ -753,12 +753,19 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { + if (topic_list == NULL) { + return TMQ_RESP_ERR__FAIL; + } const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; SCMSubscribeReq req = {0}; int32_t code = -1; + if (sz == 0) { + return TMQ_RESP_ERR__FAIL; + } + req.consumerId = tmq->consumerId; tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); @@ -830,10 +837,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } // init hb timer - tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + if (tmq->hbTimer == NULL) { + tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + } // init auto commit timer - if (tmq->autoCommit) { + if (tmq->autoCommit && tmq->commitTimer == NULL) { tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer); } @@ -1074,10 +1083,13 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { taosHashCleanup(pHash); tmq->clientTopics = newTopics; - if (taosArrayGetSize(tmq->clientTopics) == 0) - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); - else - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); + ASSERT(taosArrayGetSize(tmq->clientTopics) != 0); + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); + + /*if (taosArrayGetSize(tmq->clientTopics) == 0)*/ + /*atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC);*/ + /*else*/ + /*atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);*/ atomic_store_32(&tmq->epoch, epoch); return set; @@ -1456,9 +1468,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { - tmq_list_t* lst = tmq_list_new(); - tmq_resp_err_t rsp = tmq_subscribe(tmq, lst); + tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL); + if (rsp == TMQ_RESP_ERR__SUCCESS) { + // TODO: free resources + return TMQ_RESP_ERR__SUCCESS; + } else { + return TMQ_RESP_ERR__FAIL; + } + + tmq_list_t* lst = tmq_list_new(); + rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); + if (rsp == TMQ_RESP_ERR__SUCCESS) { // TODO: free resources return TMQ_RESP_ERR__SUCCESS; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 503c0f404a..274876567e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -627,21 +627,26 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); - SArray *tmp = pOldConsumer->rebNewTopics; - pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; - pNewConsumer->rebNewTopics = tmp; - tmp = pOldConsumer->rebRemovedTopics; - pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; - pNewConsumer->rebRemovedTopics = tmp; + if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { + pOldConsumer->status = MQ_CONSUMER_STATUS__READY; + } else { + SArray *tmp = pOldConsumer->rebNewTopics; + pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; + pNewConsumer->rebNewTopics = tmp; - tmp = pOldConsumer->assignedTopics; - pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; - pNewConsumer->assignedTopics = tmp; + tmp = pOldConsumer->rebRemovedTopics; + pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; + pNewConsumer->rebRemovedTopics = tmp; - pOldConsumer->subscribeTime = pNewConsumer->upTime; + tmp = pOldConsumer->assignedTopics; + pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; + pNewConsumer->assignedTopics = tmp; - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pOldConsumer->subscribeTime = pNewConsumer->upTime; + + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + } } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); From 3de22159d8872e9f1f6d1281c6a5d93c038150b3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 18 May 2022 15:32:23 +0800 Subject: [PATCH 048/103] fix: avoid invalid read/write --- source/libs/index/src/indexComm.c | 11 +++++++++++ source/libs/index/test/jsonUT.cc | 11 +++++++++++ source/libs/transport/src/transSrv.c | 1 + 3 files changed, 23 insertions(+) diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 74e2861037..f76d46aa77 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -316,46 +316,57 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { case TSDB_DATA_TYPE_TIMESTAMP: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int64_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint8_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_TINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int8_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_SMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int16_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_USMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint16_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_INT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int32_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint32_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%" PRIu64, *(uint64_t*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UBIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint64_t*)src, *dst, 1); + tlen = strlen(*dst); case TSDB_DATA_TYPE_FLOAT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(float*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_DOUBLE: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(double*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_NCHAR: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 86592f0cf3..ffb102ef2e 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -66,6 +66,17 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp indexMultiTermDestroy(terms); } + +static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, + int8_t operType = DEL_VALUE) { + SIndexTerm* term = + indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + tIndexJsonPut(index, terms, (int64_t)tableId); + + indexMultiTermDestroy(terms); +} static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType, SArray** result) { std::string colName(colNam); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index d1bd78f809..84adeb4bf6 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -264,6 +264,7 @@ static void uvHandleReq(SSrvConn* pConn) { CONN_SHOULD_RELEASE(pConn, pHead); STransMsg transMsg; + memset(&transMsg, 0, sizeof(transMsg)); transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = pHead->content; transMsg.msgType = pHead->msgType; From 5fdf57a1ea5083493ecedd14cb0babad3c2f65a6 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 15:33:15 +0800 Subject: [PATCH 049/103] enh: return error if cmake fails --- Jenkinsfile2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 20a86dbcea..29f4c45229 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -235,8 +235,8 @@ def pre_test_build_win() { cd debug call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 set CL=/MP8 - cmake .. -G "NMake Makefiles JOM" - jom -j 4 || exit 8 + cmake .. -G "NMake Makefiles JOM" || exit 7 + jom -j 6 || exit 8 time /t ''' return 1 From cd2f10542dcb4cf37d3388db359332bb425ee11e Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 15:51:59 +0800 Subject: [PATCH 050/103] enh: add some log --- Jenkinsfile2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 29f4c45229..4eb53c315f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -215,10 +215,12 @@ def pre_test_win(){ } bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git branch git log -5 ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git branch git log -5 ''' bat ''' @@ -233,9 +235,13 @@ def pre_test_build_win() { cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal mkdir debug cd debug + time /t call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 set CL=/MP8 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" + time /t cmake .. -G "NMake Makefiles JOM" || exit 7 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" jom -j 6 || exit 8 time /t ''' From 2f773114f8b042efa2d1433e36dc53ad248b6172 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 18 May 2022 15:56:17 +0800 Subject: [PATCH 051/103] fix(os): compile ptr error --- source/dnode/mgmt/node_mgmt/src/dmProc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 2e24e3fa1c..0a8b5135aa 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -137,13 +137,13 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg queue->tail = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, remain - 8); - memcpy(queue->pBuffer, pHead + remain - 8, rawHeadLen - (remain - 8)); + memcpy(queue->pBuffer, (char*)pHead + remain - 8, rawHeadLen - (remain - 8)); if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); queue->tail = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen); - if (rawBodyLen > 0) memcpy(queue->pBuffer, pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + if (rawBodyLen > 0) memcpy(queue->pBuffer, (char*)pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); queue->tail = bodyLen - (remain - 8 - headLen); } else { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); From ef684076c4ff86f836f5ce4db089f2269b84f9ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 07:57:29 +0000 Subject: [PATCH 052/103] refact: tdb api --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/meta.h | 24 +-- source/dnode/vnode/src/inc/sma.h | 10 +- source/dnode/vnode/src/meta/metaOpen.c | 54 ++--- source/dnode/vnode/src/meta/metaQuery.c | 40 ++-- source/dnode/vnode/src/meta/metaSma.c | 8 +- source/dnode/vnode/src/meta/metaTable.c | 186 ++++++++--------- source/dnode/vnode/src/sma/smaTDBImpl.c | 18 +- source/dnode/vnode/src/sma/smaTimeRange.c | 60 +++--- source/dnode/vnode/src/tsdb/tsdbSma.c | 4 +- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 18 +- source/libs/tdb/CMakeLists.txt | 2 +- source/libs/tdb/inc/tdb.h | 60 +++--- source/libs/tdb/src/db/tdbDb.c | 234 ++++++++++++---------- source/libs/tdb/src/db/tdbEnv.c | 178 ---------------- source/libs/tdb/src/db/tdbTable.c | 148 ++++++++++++++ source/libs/tdb/src/inc/tdbInt.h | 12 +- source/libs/tdb/test/tdbTest.cpp | 104 +++++----- 18 files changed, 576 insertions(+), 586 deletions(-) delete mode 100644 source/libs/tdb/src/db/tdbEnv.c create mode 100644 source/libs/tdb/src/db/tdbTable.c diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f7eac3a9d9..ecb022426b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -210,7 +210,7 @@ struct SMetaReader { }; struct SMTbCursor { - TDBC *pDbc; + TBC *pDbc; void *pKey; void *pVal; int kLen; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index d3abc95da9..693f4a0a2b 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -22,8 +22,8 @@ extern "C" { #endif -typedef struct SMetaIdx SMetaIdx; -typedef struct SMetaDB SMetaDB; +typedef struct SMetaIdx SMetaIdx; +typedef struct SMetaDB SMetaDB; // metaDebug ================== // clang-format off @@ -63,16 +63,16 @@ struct SMeta { char* path; SVnode* pVnode; - TENV* pEnv; + TDB* pEnv; TXN txn; - TDB* pTbDb; - TDB* pSkmDb; - TDB* pUidIdx; - TDB* pNameIdx; - TDB* pCtbIdx; - TDB* pTagIdx; - TDB* pTtlIdx; - TDB* pSmaIdx; + TTB* pTbDb; + TTB* pSkmDb; + TTB* pUidIdx; + TTB* pNameIdx; + TTB* pCtbIdx; + TTB* pTagIdx; + TTB* pTtlIdx; + TTB* pSmaIdx; SMetaIdx* pIdx; }; @@ -118,7 +118,7 @@ typedef struct { int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); +int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 2efe600b3d..0601df61e7 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -46,7 +46,7 @@ struct SSmaEnv { TXN txn; void *pPool; // SPoolMem SDiskID did; - TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? char *path; // relative path SSmaStat *pStat; }; @@ -93,16 +93,16 @@ typedef struct SDBFile SDBFile; struct SDBFile { int32_t fid; - TDB *pDB; + TTB *pDB; char *path; }; int32_t tdSmaBeginCommit(SSmaEnv *pEnv); int32_t tdSmaEndCommit(SSmaEnv *pEnv); -int32_t smaOpenDBEnv(TENV **ppEnv, const char *path); -int32_t smaCloseDBEnv(TENV *pEnv); -int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF); +int32_t smaOpenDBEnv(TDB **ppEnv, const char *path); +int32_t smaCloseDBEnv(TDB *pEnv); +int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF); int32_t smaCloseDBF(SDBFile *pDBF); int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 127eb43f2b..135f44e939 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -50,63 +50,63 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { taosMkDir(pMeta->path); // open env - ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); + ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); if (ret < 0) { metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTbDb - ret = tdbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); + ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); if (ret < 0) { metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSkmDb - ret = tdbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); + ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); if (ret < 0) { metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pUidIdx - ret = tdbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); + ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); if (ret < 0) { metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pNameIdx - ret = tdbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); + ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); if (ret < 0) { metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pCtbIdx - ret = tdbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); + ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTagIdx - ret = tdbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); + ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTtlIdx - ret = tdbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); + ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); if (ret < 0) { metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSmaIdx - ret = tdbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); + ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); if (ret < 0) { metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -125,15 +125,15 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { _err: if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); return -1; @@ -142,15 +142,15 @@ _err: int metaClose(SMeta *pMeta) { if (pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index f02b6402c4..2bcb68c82a 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -35,7 +35,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u STbDbKey tbDbKey = {.version = version, .uid = uid}; // query table.db - if (tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _err; } @@ -58,7 +58,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { int64_t version; // query uid.idx - if (tdbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -72,7 +72,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { tb_uid_t uid; // query name.idx - if (tdbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -100,9 +100,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { metaReaderInit(&pTbCur->mr, pMeta, 0); - tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); + tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); - tdbDbcMoveToFirst(pTbCur->pDbc); + tdbTbcMoveToFirst(pTbCur->pDbc); return pTbCur; } @@ -113,7 +113,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { tdbFree(pTbCur->pVal); metaReaderClear(&pTbCur->mr); if (pTbCur->pDbc) { - tdbDbcClose(pTbCur->pDbc); + tdbTbcClose(pTbCur->pDbc); } taosMemoryFree(pTbCur); } @@ -125,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { STbCfg tbCfg; for (;;) { - ret = tdbDbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -159,7 +159,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo pKey = &skmDbKey; kLen = sizeof(skmDbKey); metaRLock(pMeta); - ret = tdbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); + ret = tdbTbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); metaULock(pMeta); if (ret < 0) { return NULL; @@ -184,7 +184,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo struct SMCtbCursor { SMeta *pMeta; - TDBC *pCur; + TBC *pCur; tb_uid_t suid; void *pKey; void *pVal; @@ -207,7 +207,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { pCtbCur->suid = uid; metaRLock(pMeta); - ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); + ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); if (ret < 0) { metaULock(pMeta); taosMemoryFree(pCtbCur); @@ -217,9 +217,9 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { // move to the suid ctbIdxKey.suid = uid; ctbIdxKey.uid = INT64_MIN; - tdbDbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); + tdbTbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); if (c > 0) { - tdbDbcMoveToNext(pCtbCur->pCur); + tdbTbcMoveToNext(pCtbCur->pCur); } return pCtbCur; @@ -229,7 +229,7 @@ void metaCloseCtbCursor(SMCtbCursor *pCtbCur) { if (pCtbCur) { if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta); if (pCtbCur->pCur) { - tdbDbcClose(pCtbCur->pCur); + tdbTbcClose(pCtbCur->pCur); tdbFree(pCtbCur->pKey); tdbFree(pCtbCur->pVal); @@ -243,7 +243,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { int ret; SCtbIdxKey *pCtbIdxKey; - ret = tdbDbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + ret = tdbTbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); if (ret < 0) { return 0; } @@ -299,7 +299,7 @@ int metaGetTbNum(SMeta *pMeta) { typedef struct { SMeta *pMeta; - TDBC *pCur; + TBC *pCur; tb_uid_t uid; void *pKey; void *pVal; @@ -323,7 +323,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { pSmaCur->uid = uid; metaRLock(pMeta); - ret = tdbDbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); + ret = tdbTbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); if (ret < 0) { metaULock(pMeta); taosMemoryFree(pSmaCur); @@ -333,9 +333,9 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // move to the suid smaIdxKey.uid = uid; smaIdxKey.smaUid = INT64_MIN; - tdbDbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); + tdbTbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); if (c > 0) { - tdbDbcMoveToNext(pSmaCur->pCur); + tdbTbcMoveToNext(pSmaCur->pCur); } return pSmaCur; @@ -345,7 +345,7 @@ void metaCloseSmaCursor(SMSmaCursor *pSmaCur) { if (pSmaCur) { if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta); if (pSmaCur->pCur) { - tdbDbcClose(pSmaCur->pCur); + tdbTbcClose(pSmaCur->pCur); tdbFree(pSmaCur->pKey); tdbFree(pSmaCur->pVal); @@ -359,7 +359,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) { int ret; SSmaIdxKey *pSmaIdxKey; - ret = tdbDbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); + ret = tdbTbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); if (ret < 0) { return 0; } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 1fd81bc2cb..75595d83a6 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -130,17 +130,17 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; - return tdbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); } static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 273f7885e6..3d1854927c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -71,9 +71,9 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - TDBC *pNameIdxc = NULL; - TDBC *pUidIdxc = NULL; - TDBC *pCtbIdxc = NULL; + TBC *pNameIdxc = NULL; + TBC *pUidIdxc = NULL; + TBC *pCtbIdxc = NULL; SCtbIdxKey *pCtbIdxKey; const void *pKey = NULL; int nKey; @@ -82,43 +82,43 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { int c, ret; // prepare uid idx cursor - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c != 0) { terrno = TSDB_CODE_VND_TB_NOT_EXIST; - tdbDbcClose(pUidIdxc); + tdbTbcClose(pUidIdxc); goto _err; } // prepare name idx cursor - tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); if (ret < 0 || c != 0) { ASSERT(0); } - tdbDbcDelete(pUidIdxc); - tdbDbcDelete(pNameIdxc); - tdbDbcClose(pUidIdxc); - tdbDbcClose(pNameIdxc); + tdbTbcDelete(pUidIdxc); + tdbTbcDelete(pNameIdxc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pNameIdxc); // loop to drop each child table - tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { - tdbDbcClose(pCtbIdxc); + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || (c < 0 && tdbTbcMoveToNext(pCtbIdxc) < 0)) { + tdbTbcClose(pCtbIdxc); goto _exit; } for (;;) { - tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); + tdbTbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); pCtbIdxKey = (SCtbIdxKey *)pKey; if (pCtbIdxKey->suid > pReq->suid) break; // drop the child table (TODO) - if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; + if (tdbTbcMoveToNext(pCtbIdxc) < 0) break; } _exit: @@ -134,8 +134,8 @@ _err: int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TDBC *pUidIdxc = NULL; - TDBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -143,14 +143,14 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t ret; int32_t c; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c) { ASSERT(0); return -1; } - ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -158,11 +158,11 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { oversion = *(int64_t *)pData; - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(ret == 0 && c == 0); - ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ASSERT(ret == 0); tDecoderInit(&dc, pData, nData); @@ -191,12 +191,12 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { metaSaveToTbDb(pMeta, &nStbEntry); // update uid index - tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); + tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); metaULock(pMeta); tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; } @@ -256,9 +256,9 @@ _err: } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { - TDBC *pTbDbc = NULL; - TDBC *pUidIdxc = NULL; - TDBC *pNameIdxc = NULL; + TBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pNameIdxc = NULL; const void *pData; int nData; tb_uid_t uid; @@ -271,15 +271,15 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int c = 0, ret; // search & delete the name idx - tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); - if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { - tdbDbcClose(pNameIdxc); + tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || !tdbTbcIsValid(pNameIdxc) || c) { + tdbTbcClose(pNameIdxc); terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; } - ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pNameIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -287,36 +287,36 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { uid = *(tb_uid_t *)pData; - tdbDbcDelete(pNameIdxc); - tdbDbcClose(pNameIdxc); + tdbTbcDelete(pNameIdxc); + tdbTbcClose(pNameIdxc); // search & delete uid idx - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; } tver = *(int64_t *)pData; - tdbDbcDelete(pUidIdxc); - tdbDbcClose(pUidIdxc); + tdbTbcDelete(pUidIdxc); + tdbTbcClose(pUidIdxc); // search and get meta entry - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbTbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -345,21 +345,21 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { taosMemoryFree(pDataCopy); tDecoderClear(&coder); - tdbDbcClose(pTbDbc); + tdbTbcClose(pTbDbc); if (type == TSDB_CHILD_TABLE) { // remove the pCtbIdx - TDBC *pCtbIdxc = NULL; - tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); + ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - tdbDbcDelete(pCtbIdxc); - tdbDbcClose(pCtbIdxc); + tdbTbcDelete(pCtbIdxc); + tdbTbcClose(pCtbIdxc); // remove tags from pTagIdx (todo) } else if (type == TSDB_NORMAL_TABLE) { @@ -389,7 +389,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl int c; // search name index - ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -400,22 +400,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl pVal = NULL; // search uid index - TDBC *pUidIdxc = NULL; + TBC *pUidIdxc = NULL; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); - tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = *(int64_t *)pData; // search table.db - TDBC *pTbDbc = NULL; + TBC *pTbDbc = NULL; - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); - tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry SDecoder dc = {0}; @@ -505,21 +505,21 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // save to table db metaSaveToTbDb(pMeta, &entry); - tdbDbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); + tdbTbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); metaSaveToSkmDb(pMeta, &entry); metaULock(pMeta); tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; _err: tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return -1; } @@ -536,7 +536,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA int nData = 0; // search name index - ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -547,24 +547,24 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA pVal = NULL; // search uid index - TDBC *pUidIdxc = NULL; + TBC *pUidIdxc = NULL; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); - tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = *(int64_t *)pData; // search table.db - TDBC *pTbDbc = NULL; + TBC *pTbDbc = NULL; SDecoder dc = {0}; /* get ctbEntry */ - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); - tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ctbEntry.pBuf = taosMemoryMalloc(nData); memcpy(ctbEntry.pBuf, pData, nData); @@ -573,9 +573,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc); /* get stbEntry*/ - tdbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); - tdbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), - (void **)&stbEntry.pBuf, &nVal); + tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); + tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), + (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); tDecoderInit(&dc, stbEntry.pBuf, nVal); metaDecodeEntry(&dc, &stbEntry); @@ -632,19 +632,19 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaSaveToTbDb(pMeta, &ctbEntry); // save to uid.idx - tdbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); + tdbTbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; _err: if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return -1; } @@ -708,7 +708,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -721,11 +721,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -748,12 +748,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, @@ -801,10 +801,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SDecoder dc = {0}; // get super table - tdbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = *(int64_t *)pData; - tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); tDecoderInit(&dc, pData, nData); metaDecodeEntry(&dc, &stbEntry); @@ -817,7 +817,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } - tdbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); tDecoderClear(&dc); @@ -859,7 +859,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderInit(&coder, pVal, vLen); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c index 6e57656802..cac986d053 100644 --- a/source/dnode/vnode/src/sma/smaTDBImpl.c +++ b/source/dnode/vnode/src/sma/smaTDBImpl.c @@ -17,12 +17,12 @@ #include "sma.h" -int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { +int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) { int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { smaError("failed to create tsdb db env, ret = %d", ret); @@ -32,7 +32,7 @@ int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { return 0; } -int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } +int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { const SSmaKey *pKey1 = (const SSmaKey *)arg1; @@ -54,21 +54,21 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int return 0; } -static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { +static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { tdb_cmpr_fn_t compFunc; // Create a database compFunc = tdSmaKeyCmpr; - if (tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { + if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { return -1; } return 0; } -static int32_t smaCloseDBDb(TDB *pDB) { return tdbClose(pDB); } +static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } -int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { +int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile if (!pEnv || !pDBF) { terrno = TSDB_CODE_INVALID_PTR; @@ -99,7 +99,7 @@ int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, in int32_t ret; printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); - ret = tdbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { smaError("failed to upsert tsma data into db, ret = %d", ret); return -1; @@ -112,7 +112,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_ void *pVal = NULL; int ret; - ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { smaError("failed to get tsma data from db, ret = %d", ret); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 1d54d75ad5..f771e73c8a 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -55,7 +55,6 @@ typedef enum { SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma } ESmaStorageLevel; - // static func static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); @@ -69,18 +68,15 @@ static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t f static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); + TXN *txn); // expired window - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version); +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); // read data - // implementation /** @@ -157,7 +153,6 @@ static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { return false; } - /** * @brief Approximate value for week/month/year. * @@ -239,9 +234,8 @@ static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, return interval; } - static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { + int8_t intervalUnit) { pSmaH->pSma = pSma; pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); pSmaH->pDataBlocks = pDataBlocks; @@ -493,11 +487,12 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { smaCloseDBF(&tSmaH.dFile); } tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64, + smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 + " queryKey:%" PRIi64, SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tdDestroyTSmaWriteH(&tSmaH); tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; @@ -523,9 +518,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { tdResetExpiredWindow(pSma, pStat, indexUid, skey); } else { smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - SMA_VID(pSma), skey, tlen, indexUid); + SMA_VID(pSma), skey, tlen, indexUid); } - } } tdSmaEndCommit(pEnv); // TODO: not commit for every insert @@ -557,7 +551,7 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; - // TODO: insert tsma data blocks into B+Tree(TDB) + // TODO: insert tsma data blocks into B+Tree(TTB) if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); @@ -600,12 +594,12 @@ static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUi if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { // error handling tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), - skey, indexUid); + smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, + indexUid); return TSDB_CODE_FAILED; } smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), - skey, indexUid); + skey, indexUid); // TODO: use a standalone interface to received state upate notification from stream computing module. /** * @brief state @@ -666,8 +660,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); taosSsleep(1); if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, - refVal); + smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); break; }; } @@ -730,17 +723,17 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query tdUnRefSmaStat(pSma, pStat); terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, - tstrerror(terrno), smaStat); + tstrerror(terrno), smaStat); return TSDB_CODE_FAILED; } if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { // TODO: mark this window as expired. - smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), - querySKey, indexUid); + smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); } else { smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); + indexUid); } STSma *pTSma = pItem->pTSma; @@ -755,7 +748,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query tdInitTSmaFile(&tReadH, indexUid, querySKey); smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, - SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); + SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); return TSDB_CODE_FAILED; @@ -766,18 +759,18 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query int64_t queryGroupId = 0; tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), - tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); + smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, + *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); void *result = NULL; int32_t valueSize = 0; if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); + SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); smaCloseDBF(&tReadH.dFile); return TSDB_CODE_FAILED; } - #endif +#endif #ifdef _TEST_SMA_PRINT_DEBUG_LOG_ for (uint32_t v = 0; v < valueSize; v += 8) { @@ -878,7 +871,7 @@ static SSmaStatItem *tdNewSmaStatItem(int8_t state) { } static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { + int64_t version) { SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); if (!pItem) { // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later @@ -923,17 +916,15 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde taosMemoryFreeClear(pItem->pTSma); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, - winSKey); + winSKey); return TSDB_CODE_FAILED; } smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, - winSKey); + winSKey); return TSDB_CODE_SUCCESS; } - - /** * @brief Update expired window according to msg from stream computing module. * @@ -1035,7 +1026,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) } } else { smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - SMA_VID(pSma), pTSma->indexUid, winSKey); + SMA_VID(pSma), pTSma->indexUid, winSKey); } } } @@ -1044,4 +1035,3 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) return TSDB_CODE_SUCCESS; } - diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index dc782cc022..18cf18dbad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -53,7 +53,7 @@ struct SSmaEnv { TXN txn; SPoolMem *pPool; SDiskID did; - TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? char *path; // relative path SSmaStat *pStat; }; @@ -876,7 +876,7 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; - // TODO: insert tsma data blocks into B+Tree(TDB) + // TODO: insert tsma data blocks into B+Tree(TTB) if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index bed61a186f..a553f32bee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -17,12 +17,12 @@ #include "tsdb.h" -int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { +int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) { int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { tsdbError("Failed to create tsdb db env, ret = %d", ret); @@ -32,7 +32,7 @@ int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { return 0; } -int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } +int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { const SSmaKey *pKey1 = (const SSmaKey *)arg1; @@ -54,20 +54,20 @@ static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, i return 0; } -static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { +static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { int ret; tdb_cmpr_fn_t compFunc; // Create a database compFunc = tsdbSmaKeyCmpr; - ret = tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); + ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); return 0; } -static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbClose(pDB); } +static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } -int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { +int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile if (!pEnv || !pDBF) { terrno = TSDB_CODE_INVALID_PTR; @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; @@ -110,7 +110,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32 void *pVal = NULL; int ret; - ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { tsdbError("Failed to get sma data from db, ret = %d", ret); diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 01490030f2..405fb1c5a0 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -7,7 +7,7 @@ target_sources(tdb "src/db/tdbUtil.c" "src/db/tdbBtree.c" "src/db/tdbDb.c" - "src/db/tdbEnv.c" + "src/db/tdbTable.c" "src/db/tdbTxn.c" "src/db/tdbPage.c" "src/db/tdbOs.c" diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 8ba66f1d26..521b61e3c8 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -25,40 +25,40 @@ extern "C" { typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); // exposed types -typedef struct STEnv TENV; -typedef struct STDB TDB; -typedef struct STDBC TDBC; -typedef struct STxn TXN; - -// TENV -int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv); -int tdbEnvClose(TENV *pEnv); -int tdbBegin(TENV *pEnv, TXN *pTxn); -int tdbCommit(TENV *pEnv, TXN *pTxn); +typedef struct STDB TDB; +typedef struct STTB TTB; +typedef struct STBC TBC; +typedef struct STxn TXN; // TDB -int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); +int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb); int tdbClose(TDB *pDb); -int tdbDrop(TDB *pDb); -int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); -int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int tdbBegin(TDB *pDb, TXN *pTxn); +int tdbCommit(TDB *pDb, TXN *pTxn); -// TDBC -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); -int tdbDbcClose(TDBC *pDbc); -int tdbDbcIsValid(TDBC *pDbc); -int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); -int tdbDbcMoveToFirst(TDBC *pDbc); -int tdbDbcMoveToLast(TDBC *pDbc); -int tdbDbcMoveToNext(TDBC *pDbc); -int tdbDbcMoveToPrev(TDBC *pDbc); -int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); -int tdbDbcDelete(TDBC *pDbc); -int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); +// TTB +int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int tdbTbClose(TTB *pTb); +int tdbTbDrop(TTB *pTb); +int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); +int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); + +// TBC +int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); +int tdbTbcClose(TBC *pTbc); +int tdbTbcIsValid(TBC *pTbc); +int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); +int tdbTbcMoveToFirst(TBC *pTbc); +int tdbTbcMoveToLast(TBC *pTbc); +int tdbTbcMoveToNext(TBC *pTbc); +int tdbTbcMoveToPrev(TBC *pTbc); +int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); +int tdbTbcDelete(TBC *pTbc); +int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index fdad797333..912c1ecaaf 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,134 +15,164 @@ #include "tdbInt.h" -struct STDB { - TENV *pEnv; - SBTree *pBt; -}; - -struct STDBC { - SBTC btc; -}; - -int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) { - TDB *pDb; - SPager *pPager; - int ret; - char fFullName[TDB_FILENAME_LEN]; - SPage *pPage; - SPgno pgno; +int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { + TDB *pDb; + int dsize; + int zsize; + int tsize; + u8 *pPtr; + int ret; *ppDb = NULL; - pDb = (TDB *)tdbOsCalloc(1, sizeof(*pDb)); - if (pDb == NULL) { + dsize = strlen(rootDir); + zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; + + pPtr = (uint8_t *)tdbOsCalloc(1, zsize); + if (pPtr == NULL) { return -1; } - // pDb->pEnv - pDb->pEnv = pEnv; + pDb = (TDB *)pPtr; + pPtr += sizeof(*pDb); + // pDb->rootDir + pDb->rootDir = pPtr; + memcpy(pDb->rootDir, rootDir, dsize); + pDb->rootDir[dsize] = '\0'; + pPtr = pPtr + dsize + 1; + // pDb->jfname + pDb->jfname = pPtr; + memcpy(pDb->jfname, rootDir, dsize); + pDb->jfname[dsize] = '/'; + memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; - pPager = tdbEnvGetPager(pEnv, fname); - if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); - ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); - if (ret < 0) { - return -1; - } + pDb->jfd = -1; - tdbEnvAddPager(pEnv, pPager); - } - - ASSERT(pPager != NULL); - - // pDb->pBt - ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt)); + ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); if (ret < 0) { return -1; } + pDb->nPgrHash = 8; + tsize = sizeof(SPager *) * pDb->nPgrHash; + pDb->pgrHash = tdbOsMalloc(tsize); + if (pDb->pgrHash == NULL) { + return -1; + } + memset(pDb->pgrHash, 0, tsize); + + mkdir(rootDir, 0755); + *ppDb = pDb; return 0; } int tdbClose(TDB *pDb) { + SPager *pPager; + if (pDb) { - tdbBtreeClose(pDb->pBt); + for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) { + pDb->pgrList = pPager->pNext; + tdbPagerClose(pPager); + } + + tdbPCacheClose(pDb->pCache); + tdbOsFree(pDb->pgrHash); tdbOsFree(pDb); } + return 0; } -int tdbDrop(TDB *pDb) { - // TODO - return 0; -} +int tdbBegin(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; -int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { - return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); -} - -int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } - -int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { - return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); -} - -int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { - return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); -} - -int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { - return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); -} - -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn) { - int ret; - TDBC *pDbc = NULL; - - *ppDbc = NULL; - pDbc = (TDBC *)tdbOsMalloc(sizeof(*pDbc)); - if (pDbc == NULL) { - return -1; - } - - tdbBtcOpen(&pDbc->btc, pDb->pBt, pTxn); - - *ppDbc = pDbc; - return 0; -} - -int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pDbc->btc, pKey, kLen, c); } - -int tdbDbcMoveToFirst(TDBC *pDbc) { return tdbBtcMoveToFirst(&pDbc->btc); } - -int tdbDbcMoveToLast(TDBC *pDbc) { return tdbBtcMoveToLast(&pDbc->btc); } - -int tdbDbcMoveToNext(TDBC *pDbc) { return tdbBtcMoveToNext(&pDbc->btc); } - -int tdbDbcMoveToPrev(TDBC *pDbc) { return tdbBtcMoveToPrev(&pDbc->btc); } - -int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { - return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); -} - -int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } - -int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); -} - -int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { - return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert); -} - -int tdbDbcClose(TDBC *pDbc) { - if (pDbc) { - tdbBtcClose(&pDbc->btc); - tdbOsFree(pDbc); + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerBegin(pPager, pTxn); + if (ret < 0) { + ASSERT(0); + return -1; + } } return 0; } -int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } \ No newline at end of file +int tdbCommit(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; + + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerCommit(pPager, pTxn); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + + return 0; +} + +SPager *tdbEnvGetPager(TDB *pDb, const char *fname) { + u32 hash; + SPager **ppPager; + + hash = tdbCstringHash(fname); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) { + } + + return *ppPager; +} + +void tdbEnvAddPager(TDB *pDb, SPager *pPager) { + u32 hash; + SPager **ppPager; + + // rehash if neccessary + if (pDb->nPager + 1 > pDb->nPgrHash) { + // TODO + } + + // add to list + pPager->pNext = pDb->pgrList; + pDb->pgrList = pPager; + + // add to hash + hash = tdbCstringHash(pPager->dbFileName); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + pPager->pHashNext = *ppPager; + *ppPager = pPager; + + // increase the counter + pDb->nPager++; +} + +void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { + u32 hash; + SPager **ppPager; + + // remove from the list + for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { + } + ASSERT(*ppPager == pPager); + *ppPager = pPager->pNext; + + // remove from hash + hash = tdbCstringHash(pPager->dbFileName); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { + } + ASSERT(*ppPager == pPager); + *ppPager = pPager->pNext; + + // decrease the counter + pDb->nPager--; + + // rehash if necessary + if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) { + // TODO + } +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c deleted file mode 100644 index c0c1343a4f..0000000000 --- a/source/libs/tdb/src/db/tdbEnv.c +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) { - TENV *pEnv; - int dsize; - int zsize; - int tsize; - u8 *pPtr; - int ret; - - *ppEnv = NULL; - - dsize = strlen(rootDir); - zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; - - pPtr = (uint8_t *)tdbOsCalloc(1, zsize); - if (pPtr == NULL) { - return -1; - } - - pEnv = (TENV *)pPtr; - pPtr += sizeof(*pEnv); - // pEnv->rootDir - pEnv->rootDir = pPtr; - memcpy(pEnv->rootDir, rootDir, dsize); - pEnv->rootDir[dsize] = '\0'; - pPtr = pPtr + dsize + 1; - // pEnv->jfname - pEnv->jfname = pPtr; - memcpy(pEnv->jfname, rootDir, dsize); - pEnv->jfname[dsize] = '/'; - memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); - pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; - - pEnv->jfd = -1; - - ret = tdbPCacheOpen(szPage, pages, &(pEnv->pCache)); - if (ret < 0) { - return -1; - } - - pEnv->nPgrHash = 8; - tsize = sizeof(SPager *) * pEnv->nPgrHash; - pEnv->pgrHash = tdbOsMalloc(tsize); - if (pEnv->pgrHash == NULL) { - return -1; - } - memset(pEnv->pgrHash, 0, tsize); - - mkdir(rootDir, 0755); - - *ppEnv = pEnv; - return 0; -} - -int tdbEnvClose(TENV *pEnv) { - SPager *pPager; - - if (pEnv) { - for (pPager = pEnv->pgrList; pPager; pPager = pEnv->pgrList) { - pEnv->pgrList = pPager->pNext; - tdbPagerClose(pPager); - } - - tdbPCacheClose(pEnv->pCache); - tdbOsFree(pEnv->pgrHash); - tdbOsFree(pEnv); - } - - return 0; -} - -int tdbBegin(TENV *pEnv, TXN *pTxn) { - SPager *pPager; - int ret; - - for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { - ret = tdbPagerBegin(pPager, pTxn); - if (ret < 0) { - ASSERT(0); - return -1; - } - } - - return 0; -} - -int tdbCommit(TENV *pEnv, TXN *pTxn) { - SPager *pPager; - int ret; - - for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { - ret = tdbPagerCommit(pPager, pTxn); - if (ret < 0) { - ASSERT(0); - return -1; - } - } - - return 0; -} - -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) { - u32 hash; - SPager **ppPager; - - hash = tdbCstringHash(fname); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) { - } - - return *ppPager; -} - -void tdbEnvAddPager(TENV *pEnv, SPager *pPager) { - u32 hash; - SPager **ppPager; - - // rehash if neccessary - if (pEnv->nPager + 1 > pEnv->nPgrHash) { - // TODO - } - - // add to list - pPager->pNext = pEnv->pgrList; - pEnv->pgrList = pPager; - - // add to hash - hash = tdbCstringHash(pPager->dbFileName); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - pPager->pHashNext = *ppPager; - *ppPager = pPager; - - // increase the counter - pEnv->nPager++; -} - -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) { - u32 hash; - SPager **ppPager; - - // remove from the list - for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { - } - ASSERT(*ppPager == pPager); - *ppPager = pPager->pNext; - - // remove from hash - hash = tdbCstringHash(pPager->dbFileName); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { - } - ASSERT(*ppPager == pPager); - *ppPager = pPager->pNext; - - // decrease the counter - pEnv->nPager--; - - // rehash if necessary - if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) { - // TODO - } -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c new file mode 100644 index 0000000000..ce7da0b471 --- /dev/null +++ b/source/libs/tdb/src/db/tdbTable.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct STTB { + TDB *pEnv; + SBTree *pBt; +}; + +struct STBC { + SBTC btc; +}; + +int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { + TTB *pTb; + SPager *pPager; + int ret; + char fFullName[TDB_FILENAME_LEN]; + SPage *pPage; + SPgno pgno; + + *ppTb = NULL; + + pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb)); + if (pTb == NULL) { + return -1; + } + + // pTb->pEnv + pTb->pEnv = pEnv; + + pPager = tdbEnvGetPager(pEnv, fname); + if (pPager == NULL) { + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); + if (ret < 0) { + return -1; + } + + tdbEnvAddPager(pEnv, pPager); + } + + ASSERT(pPager != NULL); + + // pTb->pBt + ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pTb->pBt)); + if (ret < 0) { + return -1; + } + + *ppTb = pTb; + return 0; +} + +int tdbTbClose(TTB *pTb) { + if (pTb) { + tdbBtreeClose(pTb->pBt); + tdbOsFree(pTb); + } + return 0; +} + +int tdbTbDrop(TTB *pTb) { + // TODO + return 0; +} + +int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { + return tdbBtreeInsert(pTb->pBt, pKey, keyLen, pVal, valLen, pTxn); +} + +int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pTb->pBt, pKey, kLen, pTxn); } + +int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + return tdbBtreeUpsert(pTb->pBt, pKey, kLen, pVal, vLen, pTxn); +} + +int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen) { + return tdbBtreeGet(pTb->pBt, pKey, kLen, ppVal, vLen); +} + +int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { + return tdbBtreePGet(pTb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); +} + +int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) { + int ret; + TBC *pTbc = NULL; + + *ppTbc = NULL; + pTbc = (TBC *)tdbOsMalloc(sizeof(*pTbc)); + if (pTbc == NULL) { + return -1; + } + + tdbBtcOpen(&pTbc->btc, pTb->pBt, pTxn); + + *ppTbc = pTbc; + return 0; +} + +int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pTbc->btc, pKey, kLen, c); } + +int tdbTbcMoveToFirst(TBC *pTbc) { return tdbBtcMoveToFirst(&pTbc->btc); } + +int tdbTbcMoveToLast(TBC *pTbc) { return tdbBtcMoveToLast(&pTbc->btc); } + +int tdbTbcMoveToNext(TBC *pTbc) { return tdbBtcMoveToNext(&pTbc->btc); } + +int tdbTbcMoveToPrev(TBC *pTbc) { return tdbBtcMoveToPrev(&pTbc->btc); } + +int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { + return tdbBtcGet(&pTbc->btc, ppKey, pkLen, ppVal, pvLen); +} + +int tdbTbcDelete(TBC *pTbc) { return tdbBtcDelete(&pTbc->btc); } + +int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + return tdbBtreeNext(&pTbc->btc, ppKey, kLen, ppVal, vLen); +} + +int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { + return tdbBtcUpsert(&pTbc->btc, pKey, nKey, pData, nData, insert); +} + +int tdbTbcClose(TBC *pTbc) { + if (pTbc) { + tdbBtcClose(&pTbc->btc); + tdbOsFree(pTbc); + } + + return 0; +} + +int tdbTbcIsValid(TBC *pTbc) { return tdbBtcIsValid(&pTbc->btc); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index c00706ce0c..bb59a7fc61 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -103,9 +103,9 @@ typedef struct SPage SPage; #define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED) // tdbEnv.c ==================================== -void tdbEnvAddPager(TENV *pEnv, SPager *pPager); -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); +void tdbEnvAddPager(TDB *pEnv, SPager *pPager); +void tdbEnvRemovePager(TDB *pEnv, SPager *pPager); +SPager *tdbEnvGetPager(TDB *pEnv, const char *fname); // tdbBtree.c ==================================== typedef struct SBTree SBTree; @@ -334,7 +334,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { return pCell; } -struct STEnv { +struct STDB { char *rootDir; char *jfname; int jfd; @@ -357,8 +357,8 @@ struct SPager { SPgno dbOrigSize; SPage *pDirty; u8 inTran; - SPager *pNext; // used by TENV - SPager *pHashNext; // used by TENV + SPager *pNext; // used by TDB + SPager *pHashNext; // used by TDB }; #ifdef __cplusplus diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 3c8bcee3f7..a161f1c589 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -117,8 +117,8 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in TEST(tdb_test, simple_insert1) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; TXN txn; @@ -126,12 +126,12 @@ TEST(tdb_test, simple_insert1) { taosRemoveDir("tdb"); // Open Env - ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + ret = tdbOpen("tdb", 4096, 64, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database compFunc = tKeyCmpr; - ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -152,7 +152,7 @@ TEST(tdb_test, simple_insert1) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -181,7 +181,7 @@ TEST(tdb_test, simple_insert1) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); - ret = tdbGet(pDb, key, strlen(key), &pVal, &vLen); + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); ASSERT(ret == 0); GTEST_ASSERT_EQ(ret, 0); @@ -193,19 +193,19 @@ TEST(tdb_test, simple_insert1) { } { // Iterate to query the DB data - TDBC *pDBC; + TBC *pDBC; void *pKey = NULL; void *pVal = NULL; int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC, NULL); + ret = tdbTbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); - tdbDbcMoveToFirst(pDBC); + tdbTbcMoveToFirst(pDBC); for (;;) { - ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -217,28 +217,28 @@ TEST(tdb_test, simple_insert1) { GTEST_ASSERT_EQ(count, nData); - tdbDbcClose(pDBC); + tdbTbcClose(pDBC); tdbFree(pKey); tdbFree(pVal); } } - ret = tdbDrop(pDb); + ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbClose(pDb); + tdbTbClose(pDb); // Close Env - ret = tdbEnvClose(pEnv); + ret = tdbClose(pEnv); GTEST_ASSERT_EQ(ret, 0); } TEST(tdb_test, simple_insert2) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; TXN txn; @@ -246,12 +246,12 @@ TEST(tdb_test, simple_insert2) { taosRemoveDir("tdb"); // Open Env - ret = tdbEnvOpen("tdb", 1024, 10, &pEnv); + ret = tdbOpen("tdb", 1024, 10, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database compFunc = tDefaultKeyCmpr; - ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -271,24 +271,24 @@ TEST(tdb_test, simple_insert2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } { // Iterate to query the DB data - TDBC *pDBC; + TBC *pDBC; void *pKey = NULL; void *pVal = NULL; int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC, NULL); + ret = tdbTbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); - tdbDbcMoveToFirst(pDBC); + tdbTbcMoveToFirst(pDBC); for (;;) { - ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -300,7 +300,7 @@ TEST(tdb_test, simple_insert2) { GTEST_ASSERT_EQ(count, nData); - tdbDbcClose(pDBC); + tdbTbcClose(pDBC); tdbFree(pKey); tdbFree(pVal); @@ -311,29 +311,29 @@ TEST(tdb_test, simple_insert2) { tdbCommit(pEnv, &txn); tdbTxnClose(&txn); - ret = tdbDrop(pDb); + ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbClose(pDb); + tdbTbClose(pDb); // Close Env - ret = tdbEnvClose(pEnv); + ret = tdbClose(pEnv); GTEST_ASSERT_EQ(ret, 0); } TEST(tdb_test, simple_delete1) { int ret; - TDB *pDb; + TTB *pDb; char key[128]; char data[128]; TXN txn; - TENV *pEnv; + TDB *pEnv; SPoolMem *pPool; void *pKey = NULL; void *pData = NULL; int nKey; - TDBC *pDbc; + TBC *pDbc; int nData; int nKV = 69; @@ -342,11 +342,11 @@ TEST(tdb_test, simple_delete1) { pPool = openPool(); // open env - ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + ret = tdbOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); @@ -356,7 +356,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -365,7 +365,7 @@ TEST(tdb_test, simple_delete1) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); } @@ -374,7 +374,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); - ret = tdbDelete(pDb, key, strlen(key), &txn); + ret = tdbTbDelete(pDb, key, strlen(key), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -382,7 +382,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); if (iData <= 30) { GTEST_ASSERT_EQ(ret, 0); } else { @@ -391,15 +391,15 @@ TEST(tdb_test, simple_delete1) { } // loop to iterate the data - tdbDbcOpen(pDb, &pDbc, NULL); + tdbTbcOpen(pDb, &pDbc, NULL); - ret = tdbDbcMoveToFirst(pDbc); + ret = tdbTbcMoveToFirst(pDbc); GTEST_ASSERT_EQ(ret, 0); pKey = NULL; pData = NULL; for (;;) { - ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); + ret = tdbTbcNext(pDbc, &pKey, &nKey, &pData, &nData); if (ret < 0) break; std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; @@ -407,20 +407,20 @@ TEST(tdb_test, simple_delete1) { std::cout << std::endl; } - tdbDbcClose(pDbc); + tdbTbcClose(pDbc); tdbCommit(pEnv, &txn); closePool(pPool); - tdbClose(pDb); - tdbEnvClose(pEnv); + tdbTbClose(pDb); + tdbClose(pEnv); } TEST(tdb_test, simple_upsert1) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; int nData = 100000; char key[64]; char data[64]; @@ -431,11 +431,11 @@ TEST(tdb_test, simple_upsert1) { taosRemoveDir("tdb"); // open env - ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + ret = tdbOpen("tdb", 4096, 64, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, NULL, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); pPool = openPool(); @@ -446,7 +446,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -454,7 +454,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } @@ -463,7 +463,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -473,11 +473,11 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } - tdbClose(pDb); - tdbEnvClose(pEnv); + tdbTbClose(pDb); + tdbClose(pEnv); } \ No newline at end of file From 26c618e5ad0b6d9b20c9e43a7e82ec87be365602 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 May 2022 16:03:59 +0800 Subject: [PATCH 053/103] enh(query): support sum(NULL), max(NULL), min(NULL) --- include/common/ttypes.h | 2 ++ source/common/src/tdatablock.c | 4 +++- source/libs/function/src/builtins.c | 9 ++++++--- source/libs/function/src/builtinsimpl.c | 24 +++++++++++++++++++----- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index cab429d136..14428bfc43 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -179,6 +179,8 @@ typedef struct { } \ } while (0) +//TODO: use varchar(0) to represent NULL type +#define IS_NULL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_NULL) #define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 43dcf2dfa9..f239b04f55 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -322,7 +322,9 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } pColumnInfoData->pData = tmp; - memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); + if (pColumnInfoData->info.type != TSDB_DATA_TYPE_NULL) { + memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); + } } pColumnInfoData->hasNull = pSource->hasNull; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 24d6c1ade9..d58df3e876 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -46,8 +46,10 @@ static int32_t translateInOutNum(SFunctionNode* pFunc, char* pErrBuf, int32_t le } uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType)) { + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } else if (IS_NULL_TYPE(paraType)) { + paraType = TSDB_DATA_TYPE_BIGINT; } pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType}; @@ -114,18 +116,19 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType)) { + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } uint8_t resType = 0; - if (IS_SIGNED_NUMERIC_TYPE(paraType) || paraType == TSDB_DATA_TYPE_BOOL) { + if (IS_SIGNED_NUMERIC_TYPE(paraType) || TSDB_DATA_TYPE_BOOL == paraType || IS_NULL_TYPE(paraType)) { resType = TSDB_DATA_TYPE_BIGINT; } else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) { resType = TSDB_DATA_TYPE_UBIGINT; } else if (IS_FLOAT_TYPE(paraType)) { resType = TSDB_DATA_TYPE_DOUBLE; } + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4ab1a52897..d8d7a2f62e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -255,7 +255,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + //pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -377,11 +377,17 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; - int32_t type = pInput->pData[0]->info.type; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; + int32_t type = pInput->pData[0]->info.type; SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 1; + goto _sum_over; + } + if (pInput->colDataAggIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; ASSERT(numOfElem >= 0); @@ -426,6 +432,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { } } +_sum_over: // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; @@ -782,6 +789,12 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SMinmaxResInfo *pBuf = GET_ROWCELL_INTERBUF(pResInfo); + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElems = 1; + goto _min_max_over; + } + // data in current data block are qualified to the query if (pInput->colDataAggIsSet) { numOfElems = pInput->numOfRows - pAgg->numOfNull; @@ -1202,6 +1215,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } } +_min_max_over: return numOfElems; } @@ -1234,9 +1248,9 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { float v = *(double*) &pRes->v; - colDataAppend(pCol, currentRow, (const char*)&v, false); + colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes); } else { - colDataAppend(pCol, currentRow, (const char*)&pRes->v, false); + colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes); } setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); From c7088f08126a7ccd45f914a3f885b6a60fb5f9df Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 18 May 2022 16:10:35 +0800 Subject: [PATCH 054/103] enh: print time during windows build --- Jenkinsfile2 | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 4eb53c315f..7a6f793399 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -242,6 +242,7 @@ def pre_test_build_win() { time /t cmake .. -G "NMake Makefiles JOM" || exit 7 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" + time /t jom -j 6 || exit 8 time /t ''' From a52e0027cf6508a562a4ed04ba30c90763c46fe4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 May 2022 16:03:59 +0800 Subject: [PATCH 055/103] enh(query): support count(NULL) = 0 --- source/libs/function/src/builtinsimpl.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d8d7a2f62e..77126225bf 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -330,8 +330,18 @@ static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) { int32_t countFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = getNumofElem(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); - *((int64_t*)buf) += numOfElem; + + SInputColumnInfoData* pInput = &pCtx->input; + int32_t type = pInput->pData[0]->info.type; + + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + if (IS_NULL_TYPE(type)) { + //select count(NULL) returns 0 + numOfElem = 1; + *((int64_t*)buf) = 0; + } else { + *((int64_t*)buf) += numOfElem; + } SET_VAL(pResInfo, numOfElem, 1); return TSDB_CODE_SUCCESS; From dbeee11c284d1971cc808659a456fe76b5cab456 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 18 May 2022 16:16:07 +0800 Subject: [PATCH 056/103] test: add case for tmq --- tests/system-test/7-tmq/subscribeDb.py | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index d2cccd0532..b536a70515 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -219,6 +219,41 @@ class TDTestCase: tdSql.query("drop topic %s"%topicName1) + tdLog.info("creat the same topic name , and start to consume") + self.initConsumerTable() + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self, cfgPath, buildPath): From 4e640b6d67433c52c90d981ef6fa0cbcc2e4a100 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 18 May 2022 16:13:58 +0800 Subject: [PATCH 057/103] enh(tmq): update tb list for existing topic --- include/libs/executor/executor.h | 6 +++--- source/client/src/tmq.c | 20 +++++--------------- source/dnode/vnode/src/inc/vnodeInt.h | 5 +++-- source/dnode/vnode/src/tq/tq.c | 15 +++++++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 7 ++++++- source/libs/executor/src/executor.c | 8 ++++---- 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 162b6fb2ed..8f300c96c5 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -42,7 +42,7 @@ typedef struct SReadHandle { #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2 typedef enum { - OPTR_EXEC_MODEL_BATCH = 0x1, + OPTR_EXEC_MODEL_BATCH = 0x1, OPTR_EXEC_MODEL_STREAM = 0x2, } EOPTR_EXEC_MODEL; @@ -81,7 +81,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO * @param isAdd * @return */ -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json @@ -169,7 +169,7 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type); void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); -int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes); +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes); #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index f7ddca8f69..2a2fdea537 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -119,7 +119,7 @@ enum { enum { TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__READY, - /*TMQ_CONSUMER_STATUS__NO_TOPIC,*/ + TMQ_CONSUMER_STATUS__NO_TOPIC, }; enum { @@ -753,19 +753,12 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - if (topic_list == NULL) { - return TMQ_RESP_ERR__FAIL; - } const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; SCMSubscribeReq req = {0}; int32_t code = -1; - if (sz == 0) { - return TMQ_RESP_ERR__FAIL; - } - req.consumerId = tmq->consumerId; tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); @@ -1083,13 +1076,10 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { taosHashCleanup(pHash); tmq->clientTopics = newTopics; - ASSERT(taosArrayGetSize(tmq->clientTopics) != 0); - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - - /*if (taosArrayGetSize(tmq->clientTopics) == 0)*/ - /*atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC);*/ - /*else*/ - /*atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);*/ + if (taosArrayGetSize(tmq->clientTopics) == 0) + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); + else + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); atomic_store_32(&tmq->epoch, epoch); return set; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9727d9df9f..6a5ad63822 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -96,8 +96,8 @@ STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy) SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); SArray* metaGetSmaTbUids(SMeta* pMeta); -int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); -int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); +int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); +int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); // tsdb int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); @@ -117,6 +117,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqCommit(STQ*); +int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6cc986d54b..80aa350e4d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -101,6 +101,21 @@ static void tdSRowDemo() { taosMemoryFree(pTSChema); } +int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) { + void* pIter = NULL; + STqExec* pExec = NULL; + while (1) { + pIter = taosHashIterate(pTq->execs, pIter); + if (pIter == NULL) break; + pExec = (STqExec*)pIter; + for (int32_t i = 0; i < 5; i++) { + int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true); + ASSERT(code == 0); + } + } + return 0; +} + int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType != TDMT_VND_SUBMIT) return 0; void* pIter = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464331319e..5c8cd362fd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -333,6 +333,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, SVCreateTbRsp cRsp = {0}; char tbName[TSDB_TABLE_FNAME_LEN]; STbUidStore *pStore = NULL; + SArray *tbUids = NULL; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -348,7 +349,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); - if (rsp.pArray == NULL) { + tbUids = taosArrayInit(req.nReqs, sizeof(int64_t)); + if (rsp.pArray == NULL || tbUids == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -376,6 +378,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } else { cRsp.code = TSDB_CODE_SUCCESS; tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); + taosArrayPush(tbUids, &pCreateReq->uid); } taosArrayPush(rsp.pArray, &cRsp); @@ -383,6 +386,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tDecoderClear(&decoder); + tqUpdateTbUidList(pVnode->pTq, tbUids); tdUpdateTbUidList(pVnode->pSma, pStore); tdUidStoreFree(pStore); @@ -402,6 +406,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, _exit: taosArrayDestroy(rsp.pArray); + taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); return rcode; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 320450eb6e..66073b70eb 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -126,7 +126,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return pTaskInfo; } -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; // traverse to the stream scanner node to add this table id @@ -141,12 +141,12 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA SMetaReader mr = {0}; metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); - for(int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); int32_t code = metaGetTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%"PRIu64" code:%s", *id, tstrerror(terrno)); + qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); continue; } @@ -160,7 +160,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA metaReaderClear(&mr); - qDebug(" %d qualified child tables added into stream scanner", (int32_t) taosArrayGetSize(qa)); + qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); if (code != TSDB_CODE_SUCCESS) { return code; From 19d64d7c25b631ca6d9c21ed59084fb4d270f951 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 16:30:50 +0800 Subject: [PATCH 058/103] feat: check ts in submit req msg --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tsdb/tsdbWrite.c | 45 +++++++++++++++++++++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9727d9df9f..0327c4be13 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,6 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 341ab94ca4..c4d18ef9e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -15,7 +15,7 @@ #include "tsdb.h" -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); +// static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { SSubmitMsgIter msgIter = {0}; @@ -54,7 +54,38 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * return 0; } -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +#if 0 +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, + rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} +#endif + +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), uid, now, minKey, maxKey, rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; @@ -112,14 +143,14 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { return -1; } } - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) { +#endif + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { + if (tsdbCheckRowRange(pTsdb, msgIter.uid, row, minKey, maxKey, now) < 0) { return -1; } } -#endif + } if (terrno != TSDB_CODE_SUCCESS) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464331319e..80157ec0ba 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -626,6 +626,11 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); #endif + if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { + pRsp->code = terrno; + goto _exit; + } + // handle the request if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; From fb0711fa3e318abf0b9faccb6055af34b2261784 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 18 May 2022 16:18:36 +0800 Subject: [PATCH 059/103] fix(stream):set window info --- source/libs/executor/src/scanoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eaa1197a84..d042e463f0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -668,6 +668,7 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti } pDataBlock->info.rows = size; pDataBlock->info.type = STREAM_REPROCESS; + blockDataUpdateTsWindow(pDataBlock); taosArrayClear(pInfo->tsArray); return pDataBlock; } @@ -768,6 +769,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } rows = pBlockInfo->rows; doFilter(pInfo->pCondition, pInfo->pRes, NULL); + blockDataUpdateTsWindow(pInfo->pRes); break; } From e948d5a2483f3d6da8ee822d6b66cba51a2f6ab7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 May 2022 16:35:13 +0800 Subject: [PATCH 060/103] enh(query): support avg(NULL), stddev(NULL) --- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index d58df3e876..306b438344 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -63,7 +63,7 @@ static int32_t translateInNumOutDou(SFunctionNode* pFunc, char* pErrBuf, int32_t } uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType)) { + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 77126225bf..90632256b2 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -542,6 +542,12 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 1; + goto _avg_over; + } + switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t* plist = (int8_t*)pCol->pData; @@ -633,6 +639,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { break; } +_avg_over: // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; @@ -1330,6 +1337,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 1; + goto _stddev_over; + } + switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t* plist = (int8_t*)pCol->pData; @@ -1427,6 +1440,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { break; } +_stddev_over: // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; From c85232e14c7168ecd56c61950481abaac9169461 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 May 2022 16:37:09 +0800 Subject: [PATCH 061/103] update table meta based on sversion --- include/libs/catalog/catalog.h | 5 +++ source/client/src/clientEnv.c | 10 ++++++ source/client/src/clientImpl.c | 57 ++++++++++++++++++++++++++++--- source/libs/catalog/src/catalog.c | 5 +++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 04a24c4f32..a29dc3a90a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -59,6 +59,11 @@ typedef struct SMetaData { SArray *pQnodeList; // qnode list, SArray } SMetaData; +typedef struct STbSVersion { + char* tbFName; + int32_t sver; +} STbSVersion; + typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c82cce9af3..30997def74 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -83,6 +83,15 @@ void closeTransporter(STscObj *pTscObj) { rpcClose(pTscObj->pAppInfo->pTransporter); } +static bool clientRpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + + // TODO refactor void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; @@ -91,6 +100,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; + rpcInit.rfp = clientRpcRfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 23787651b2..5d3a139aeb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -310,9 +310,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } } - if (pRes) { - *pRes = res.res; - } + *pRes = res.res; pRequest->code = res.code; terrno = res.code; @@ -324,7 +322,49 @@ int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList); } +int32_t validateSversion(SRequestObj* pRequest, void* res) { + SArray* pArray = NULL; + int32_t code = 0; + + if (TDMT_VND_SUBMIT == pRequest->type) { + SSubmitRsp* pRsp = (SSubmitRsp*)res; + if (pRsp->nBlocks <= 0) { + return TSDB_CODE_SUCCESS; + } + + pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); + if (NULL == pArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pRsp->nBlocks; ++i) { + SSubmitBlkRsp *blk = pRsp->pBlocks + i; + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; + taosArrayPush(pArray, &tbSver); + } + } else if (TDMT_VND_QUERY == pRequest->type) { + + } + + SCatalog* pCatalog = NULL; + CHECK_CODE_GOTO(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog), _return); + + SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + code = catalogChkTbMetaVersion(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, &epset, pArray); + +_return: + + taosArrayDestroy(pArray); + + return code; +} + + SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { + void* pRes = NULL; + if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -337,7 +377,10 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); if (TSDB_CODE_SUCCESS == code) { - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, res); + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, &pRes); + if (NULL != pRes) { + code = validateSversion(pRequest, pRes); + } } taosArrayDestroy(pNodeList); break; @@ -356,6 +399,12 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; + freeRequestRes(pRequest, pRes); + pRes = NULL; + } + + if (res) { + *res = pRes; } return pRequest; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d3045f68a1..a620b84f6d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2883,6 +2883,11 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + +} + + int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { CTG_API_ENTER(); From eaba4dba180f72e36a7314f9aa069e0368cb5df0 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 16:40:21 +0800 Subject: [PATCH 062/103] enh: add const for param --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbWrite.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0327c4be13..c2fe63d79c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,7 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index c4d18ef9e6..a67f413ba7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -85,7 +85,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro return 0; } -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; From dbe43425f172eb855d5c52a32944c1d0fb5a34e1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 May 2022 16:53:17 +0800 Subject: [PATCH 063/103] fix(query): fix error --- source/common/src/tdatablock.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f239b04f55..43dcf2dfa9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -322,9 +322,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } pColumnInfoData->pData = tmp; - if (pColumnInfoData->info.type != TSDB_DATA_TYPE_NULL) { - memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); - } + memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } pColumnInfoData->hasNull = pSource->hasNull; From 30574a17b64616c730e43eaef9cdb47f8edb5f7e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 16:58:28 +0800 Subject: [PATCH 064/103] refactor: adjust msg logs --- source/dnode/mgmt/mgmt_bnode/src/bmWorker.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 8 ++++---- source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 6 +++--- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 8 ++++---- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 6 +++--- source/dnode/mnode/impl/src/mndTrans.c | 8 ++++---- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index d97949d3f7..54c57fbb0b 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -63,7 +63,7 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { bmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7afa86a377..7894eb5acb 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -86,7 +86,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; SRpcMsg rpcRsp = {0}; - dTrace("send status msg to mnode, app:%p", rpcMsg.info.ahandle); + dTrace("send status msg to mnode"); SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 787c6c98c8..57737293c5 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -160,7 +160,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { rpcSendResponse(&rsp); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index ee65335382..2d690f3306 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -29,7 +29,7 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; - dTrace("msg:%p, get from mnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + dTrace("msg:%p, get from mnode queue", pMsg); switch (pMsg->msgType) { case TDMT_DND_ALTER_MNODE: @@ -51,7 +51,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { mmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -73,13 +73,13 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { } } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index d5c82a043c..6b08cc4b62 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -44,7 +44,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } @@ -60,7 +60,7 @@ static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -76,7 +76,7 @@ static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 10bf6ff9bf..a2e88eb18e 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -44,7 +44,7 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { smSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d7298804fe..4839266c0e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -59,7 +59,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { vmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -73,7 +73,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -88,7 +88,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -238,7 +238,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index c9100aab9d..1c34a518d4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -44,7 +44,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { } pMsg->info.wrapper = pWrapper; - dTrace("msg:%p, will be processed by %s, handle:%p", pMsg, pWrapper->name, pMsg->info.handle); + dTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); return (*msgFp)(pWrapper->pMgmt, pMsg); } @@ -56,8 +56,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; SMgmtWrapper *pWrapper = NULL; - dTrace("msg:%s is received, handle:%p cont:%p len:%d code:0x%04x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), - pRpc->info.handle, pRpc->pCont, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); + dTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), + pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); pRpc->info.noResp = 0; pRpc->info.persistHandle = 0; pRpc->info.wrapper = NULL; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7bd1dd80fb..c9b68f27e7 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -842,7 +842,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, pTrans->rpcInfo.ahandle); SRpcMsg rspMsg = { .info = pTrans->rpcInfo, @@ -899,7 +899,7 @@ void mndTransProcessRsp(SRpcMsg *pRsp) { } } - mDebug("trans:%d, action:%d response is received, code:0x%04x, accept:0x%04x", transId, action, pRsp->code, + mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%04x", transId, action, pRsp->code, pAction->acceptableCode); mndTransExecute(pMnode, pTrans); @@ -1031,7 +1031,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); return 0; } else { - mError("trans:%d, all %d actions executed, code:0x%04x", pTrans->id, numOfActions, errCode & 0XFFFF); + mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode & 0XFFFF); mndTransResetActions(pMnode, pTrans, pArray); terrno = errCode; return errCode; @@ -1222,7 +1222,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); } - mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); + mDebug("trans:%d, finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); return continueExec; } From c50bb310def343e472b3a0e3604d797af2e19653 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 17:22:43 +0800 Subject: [PATCH 065/103] refactor: adjust msg logs --- source/dnode/mgmt/mgmt_bnode/src/bmWorker.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 37 ++++++---- source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 8 +-- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 10 +-- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 76 +++++++++------------ 6 files changed, 65 insertions(+), 70 deletions(-) diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index 54c57fbb0b..14c6250b2a 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -63,7 +63,7 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { bmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 57737293c5..be8ddf1430 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -160,7 +160,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { rpcSendResponse(&rsp); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 2d690f3306..98e234af3c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -51,7 +51,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { mmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -73,24 +73,31 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { } } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); } +int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); +} -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); } +int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); +} -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); } +int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); +} -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); +int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); } int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { @@ -101,25 +108,25 @@ static inline int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); taosWriteQitem(pWorker->queue, pMsg); return 0; } -int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pRpc); +int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pMsg); } -int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pRpc); +int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pMsg); } -int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return mmPutRpcMsgToWorker(&pMgmt->readWorker, pRpc); +int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutRpcMsgToWorker(&pMgmt->readWorker, pMsg); } -int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pRpc); } +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg qCfg = { diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 6b08cc4b62..444c42717a 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -44,7 +44,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } @@ -60,7 +60,7 @@ static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -76,7 +76,7 @@ static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -105,7 +105,7 @@ static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SR return -1; } - dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); taosWriteQitem(pWorker->queue, pMsg); return 0; diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index a2e88eb18e..c94cf527c7 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -44,7 +44,7 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { smSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } @@ -166,7 +166,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -174,7 +174,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -187,7 +187,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -195,7 +195,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->sharedWorker; - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 4839266c0e..9f77180cd8 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -51,7 +51,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, not processed in vnode-mgmt/monitor queue", pMsg); + dError("msg:%p, not processed in vnode queue", pMsg); } if (msgType & 1u) { @@ -59,7 +59,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { vmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -67,13 +67,13 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeObj *pVnode = pInfo->ahandle; - dTrace("msg:%p, will be processed in vnode-query queue", pMsg); + dTrace("msg:%p, get from vnode-query queue", pMsg); int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -82,13 +82,13 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeObj *pVnode = pInfo->ahandle; - dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg); + dTrace("msg:%p, get from vnode-fetch queue", pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -96,7 +96,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg rsp; SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); if (pArray == NULL) { @@ -108,7 +107,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO SRpcMsg *pMsg = NULL; if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; - dTrace("msg:%p, will be processed in vnode-write queue", pMsg); + dTrace("msg:%p, get from vnode-write queue", pMsg); if (taosArrayPush(pArray, &pMsg) == NULL) { dTrace("msg:%p, failed to process since %s", pMsg, terrstr()); vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); @@ -116,21 +115,12 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } for (int i = 0; i < taosArrayGetSize(pArray); i++) { - SRpcMsg *pMsg; - SRpcMsg *pRpc; + SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + SRpcMsg rsp = {.info = pMsg->info, .pCont = NULL, .contLen = 0}; - pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - pRpc = pMsg; - - rsp.info = pRpc->info; - rsp.pCont = NULL; - rsp.contLen = 0; - - int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false); + int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - // rsp.code = TSDB_CODE_SYN_NOT_LEADER; - // tmsgSendRsp(&rsp); - dTrace("syncPropose not leader redirect, vgId:%d ", syncGetVgId(vnodeGetSyncHandle(pVnode->pImpl))); + dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId); rsp.code = TSDB_CODE_RPC_REDIRECT; SEpSet newEpSet; syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); @@ -160,7 +150,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; SRpcMsg rsp; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -181,7 +171,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // apply data into tsdb if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) { rsp.code = terrno; - dTrace("vnodeProcessWriteReq error, code:%d", terrno); + dTrace("msg:%p, process write error since %s", pMsg, terrstr()); } syncApplyMsgDestroy(pSyncApplyMsg); @@ -215,7 +205,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf SRpcMsg rsp = {0}; rsp.code = terrno; rsp.info = pMsg->info; - dTrace("vmProcessSyncQueue error, code:%d", terrno); + dTrace("msg:%p, process sync queue error since code:%s", pMsg, terrstr()); tmsgSendRsp(&rsp); } } @@ -227,18 +217,18 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); - dTrace("msg:%p, will be processed in vnode-merge queue", pMsg); + dTrace("msg:%p, get from vnode-merge queue", pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%x:%s", pMsg, code, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -255,29 +245,29 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr()); + dError("vgId:%d, failed to put msg:%p into vnode-queue since %s", pHead->vgId, pMsg, terrstr()); return terrno != 0 ? terrno : -1; } switch (qtype) { case QUERY_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-query queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-fetch queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-write queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-sync queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pSyncQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-merge queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pMergeQ, pMsg); break; default: @@ -312,7 +302,7 @@ int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->mgmtWorker; - dTrace("msg:%p, will be put into vnode-mgmt queue, worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into vnode-mgmt worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -320,7 +310,7 @@ int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; - dTrace("msg:%p, will be put into vnode-monitor queue, worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into vnode-monitor worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -332,35 +322,33 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q if (pVnode == NULL) return -1; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - int32_t code = 0; + int32_t code = 0; if (pMsg != NULL) { - dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - // if (pMsg->handle != NULL) assert(pMsg->refId != 0); switch (qtype) { case WRITE_QUEUE: - dTrace("msg:%p, will be put into vnode-write queue", pMsg); + dTrace("msg:%p, create and put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case QUERY_QUEUE: - dTrace("msg:%p, will be put into vnode-query queue", pMsg); + dTrace("msg:%p, create and put into vnode-query queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); + dTrace("msg:%p, create and put into vnode-fetch queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case APPLY_QUEUE: - dTrace("msg:%p, will be put into vnode-apply queue", pMsg); + dTrace("msg:%p, create and put into vnode-apply queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pApplyQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, will be put into vnode-merge queue", pMsg); + dTrace("msg:%p, create and put into vnode-merge queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pMergeQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, will be put into vnode-sync queue", pMsg); + dTrace("msg:%p, create and put into vnode-sync queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pSyncQ, pMsg); break; default: From 3800079dbed6d9e72a63465b7753a149a12c1ba2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 17:42:50 +0800 Subject: [PATCH 066/103] refactor: adjust msg logs --- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 16 ++++++++-------- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 11 ++++++----- source/dnode/mgmt/node_mgmt/src/dmProc.c | 4 ++-- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 5d9f61b846..5dc5b6cef2 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -274,25 +274,25 @@ static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus) } } -void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("net test req is received"); - SRpcMsg rsp = {.code = 0, .info = pReq->info}; - rsp.pCont = rpcMallocCont(pReq->contLen); +void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) { + dDebug("start to process net test req"); + SRpcMsg rsp = {.code = 0, .info = pMsg->info}; + rsp.pCont = rpcMallocCont(pMsg->contLen); if (rsp.pCont == NULL) { rsp.code = TSDB_CODE_OUT_OF_MEMORY; } else { - rsp.contLen = pReq->contLen; + rsp.contLen = pMsg->contLen; } rpcSendResponse(&rsp); } -void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("server startup status req is received"); +void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) { + dDebug("start to process server startup status req"); SServerStatusRsp statusRsp = {0}; dmGetServerStartupStatus(pDnode, &statusRsp); - SRpcMsg rspMsg = {.info = pReq->info}; + SRpcMsg rspMsg = {.info = pMsg->info}; int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); if (rspLen < 0) { rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index f23db5da16..3a26388d14 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -37,6 +37,7 @@ static int32_t dmCreateShm(SMgmtWrapper *pWrapper) { dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); return -1; } + dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->proc.shm.id, shmsize); return 0; } @@ -60,7 +61,7 @@ static int32_t dmNewProc(SMgmtWrapper *pWrapper, EDndNodeType ntype) { taosIgnSignal(SIGCHLD); pWrapper->proc.pid = pid; - dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid); + dInfo("node:%s, continue running in new pid:%d", pWrapper->name, pid); return 0; } @@ -177,11 +178,11 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { if (OnlyInParentProc(pWrapper)) { int32_t pid = pWrapper->proc.pid; if (pid > 0 && taosProcExist(pid)) { - dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pid); + dInfo("node:%s, send kill signal to the child pid:%d", pWrapper->name, pid); taosKillProc(pid); - dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pid); + dInfo("node:%s, wait for child pid:%d to stop", pWrapper->name, pid); taosWaitProc(pid); - dInfo("node:%s, child process:%d is stopped", pWrapper->name, pid); + dInfo("node:%s, child pid:%d is stopped", pWrapper->name, pid); } } @@ -255,7 +256,7 @@ static void dmWatchNodes(SDnode *pDnode) { if (!OnlyInParentProc(pWrapper)) continue; if (proc->pid <= 0 || !taosProcExist(proc->pid)) { - dError("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid); + dError("node:%s, pid:%d is killed and needs to restart", pWrapper->name, proc->pid); dmCloseProcRpcHandles(&pWrapper->proc); dmNewProc(pWrapper, ntype); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 2e24e3fa1c..0377aaeb79 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -162,7 +162,7 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg return 0; } -static int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) { +static inline int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) { tsem_wait(&queue->sem); taosThreadMutexLock(&queue->mutex); @@ -412,7 +412,7 @@ void dmCleanupProc(struct SMgmtWrapper *pWrapper) { SProc *proc = &pWrapper->proc; if (proc->name == NULL) return; - dDebug("node:%s, start to clean up proc", pWrapper->name); + dDebug("node:%s, start to cleanup proc", pWrapper->name); dmStopProc(proc); dmCleanupProcQueue(proc->cqueue); dmCleanupProcQueue(proc->pqueue); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 1c34a518d4..f66bc84def 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -43,8 +43,8 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { return -1; } - pMsg->info.wrapper = pWrapper; dTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); + pMsg->info.wrapper = pWrapper; return (*msgFp)(pWrapper->pMgmt, pMsg); } From 36dcc70f0d9eb58db7a9571619e09d962dcdaaad Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 18 May 2022 18:21:32 +0800 Subject: [PATCH 067/103] fix: improve udf function misuse stability --- source/libs/executor/src/executorMain.c | 1 + source/libs/function/src/tudf.c | 64 ++++++++++++++----------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 354f4d8752..d4d8696aba 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -133,6 +133,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; + cleanUpUdfs(); qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); return pTaskInfo->code; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 068998e469..5f20d2e50a 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1364,9 +1364,12 @@ void releaseUdfFuncHandle(char* udfName) { SUdfcFuncStub key = {0}; strcpy(key.udfName, udfName); SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - ASSERT(foundStub); - --foundStub->refCount; - ASSERT(foundStub->refCount>=0); + if (!foundStub) { + return; + } + if (foundStub->refCount > 0) { + --foundStub->refCount; + } uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); } @@ -1377,7 +1380,7 @@ int32_t cleanUpUdfs() { while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); if (stub->refCount == 0) { - fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle); + fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); doTeardownUdf(stub->handle); } else { fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", @@ -1530,12 +1533,15 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, } SUdfcUvSession *session = handle; code = doCallUdfScalarFunc(handle, input, numOfCols, output); - if (session->outputType != output->columnData->info.type - || session->outputLen != output->columnData->info.bytes) { - fnError("udfc scalar function calculate error, session type: %d(%d), output type: %d(%d)", - session->outputType, session->outputLen, - output->columnData->info.type, output->columnData->info.bytes); + if (output->columnData == NULL) { + fnError("udfc scalar function calculate error. no column data"); code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } else { + if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) { + fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", session->outputType, + session->outputLen, output->columnData->info.type, output->columnData->info.bytes); + code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } } releaseUdfFuncHandle(udfName); return code; @@ -1565,7 +1571,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); - taosMemoryFree(task->session); + taosMemoryFree(session); taosMemoryFree(task); return err; @@ -1573,7 +1579,6 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { //memory layout |---SUdfAggRes----|-----final result-----|---inter result----| typedef struct SUdfAggRes { - SUdfcUvSession *session; int8_t finalResNum; int8_t interResNum; char* finalResBuf; @@ -1606,7 +1611,6 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - udfRes->session = (SUdfcUvSession *)handle; SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); @@ -1621,22 +1625,26 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult releaseUdfFuncHandle(pCtx->udfName); return false; } + releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&buf); return true; } int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - SInputColumnInfoData* pInput = &pCtx->input; - int32_t numOfCols = pInput->numOfInputCols; - - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SUdfcUvSession *session = udfRes->session; - if (session == NULL) { - return TSDB_CODE_UDF_NO_FUNC_HANDLE; + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + SInputColumnInfoData* pInput = &pCtx->input; + int32_t numOfCols = pInput->numOfInputCols; int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; @@ -1664,7 +1672,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { .numOfResult = udfRes->interResNum}; SUdfInterBuf newState = {0}; - int32_t udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); + udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); if (udfCode != 0) { fnError("udfAggProcess error. code: %d", udfCode); newState.numOfResult = 0; @@ -1684,19 +1692,21 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { blockDataDestroy(inputBlock); taosArrayDestroy(tempBlock.pDataBlock); - if (udfCode != 0) { - releaseUdfFuncHandle(pCtx->udfName); - } + releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&newState); return udfCode; } int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SUdfcUvSession *session = udfRes->session; - if (session == NULL) { - return TSDB_CODE_UDF_NO_FUNC_HANDLE; + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; From 7e89f1c427d861d322e2ff2800ddbe030c1d0079 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 18:27:42 +0800 Subject: [PATCH 068/103] refactor: remove auth func --- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 5 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 61 ------------------- source/dnode/mnode/impl/src/mndProfile.c | 3 +- source/dnode/mnode/impl/src/mnode.c | 21 ++++--- 4 files changed, 14 insertions(+), 76 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 5dc5b6cef2..30d7750f79 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -275,7 +275,7 @@ static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus) } void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) { - dDebug("start to process net test req"); + dDebug("msg:%p, net test req will be processed", pMsg); SRpcMsg rsp = {.code = 0, .info = pMsg->info}; rsp.pCont = rpcMallocCont(pMsg->contLen); if (rsp.pCont == NULL) { @@ -287,8 +287,7 @@ void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) { } void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) { - dDebug("start to process server startup status req"); - + dDebug("msg:%p, server startup status req will be processed", pMsg); SServerStatusRsp statusRsp = {0}; dmGetServerStartupStatus(pDnode, &statusRsp); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index f66bc84def..b4c8a0807c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -344,66 +344,6 @@ void dmCleanupClient(SDnode *pDnode) { } } -static inline int32_t dmGetHideUserAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - int32_t code = 0; - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - - if (strcmp(user, INTERNAL_USER) == 0) { - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - } else { - code = -1; - } - - if (code == 0) { - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - } - - return code; -} - -static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, - char *ckey) { - if (dmGetHideUserAuth(user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - SAuthReq authReq = {0}; - tstrncpy(authReq.user, user, TSDB_USER_LEN); - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSAuthReq(pReq, contLen, &authReq); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .info.ahandle = (void *)9528}; - SRpcMsg rpcRsp = {0}; - SEpSet epSet = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dmGetMnodeEpSet(&pDnode->data, &epSet); - dmSendRecv(&epSet, &rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - terrno = rpcRsp.code; - dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); - } else { - SAuthRsp authRsp = {0}; - tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); - memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); - memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); - *spi = authRsp.spi; - *encrypt = authRsp.encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, - authRsp.encrypt); - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; @@ -416,7 +356,6 @@ int32_t dmInitServer(SDnode *pDnode) { rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = (RpcAfp)dmRetrieveUserAuthInfo; rpcInit.parent = pDnode; pTrans->serverRpc = rpcOpen(&rpcInit); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 5cf2a36731..7e99c5583d 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -197,8 +197,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto CONN_OVER; } if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { - mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->conn.user, connReq.passwd, - pUser->pass); + mError("user:%s, failed to auth while acquire user, input:%s", pReq->conn.user, connReq.passwd); code = TSDB_CODE_RPC_AUTH_FAILURE; goto CONN_OVER; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3dfba4eca7..5326376bda 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -343,19 +343,20 @@ void mndStop(SMnode *pMnode) { return mndCleanupTimer(pMnode); } int32_t mndProcessMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; void *ahandle = pMsg->info.ahandle; - mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(pMsg->msgType), ahandle); - if (IsReq(pMsg) && !mndIsMaster(pMnode)) { - terrno = TSDB_CODE_APP_NOT_READY; - mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - return -1; - } + if (IsReq(pMsg)) { + if (!mndIsMaster(pMnode)) { + terrno = TSDB_CODE_APP_NOT_READY; + mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + return -1; + } - if (IsReq(pMsg) && (pMsg->contLen == 0 || pMsg->pCont == NULL)) { - terrno = TSDB_CODE_INVALID_MSG_LEN; - mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - return -1; + if (pMsg->contLen == 0 || pMsg->pCont == NULL) { + terrno = TSDB_CODE_INVALID_MSG_LEN; + mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + return -1; + } } MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; From 0164158256daf9fad228cf24918e57d0463aeb52 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 18:37:39 +0800 Subject: [PATCH 069/103] refactor: adjust msgcb --- include/common/tmsgcb.h | 6 ++-- source/common/src/tmsgcb.c | 28 +++++++++---------- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 2 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 7ba6e5044c..9fa657a2a6 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -60,9 +60,9 @@ typedef struct { ReportStartup reportStartupFp; } SMsgCb; -void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); -int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg); -int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); +void tmsgSetDefault(const SMsgCb* msgcb); +int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg); +int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg); void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index f69fb65f04..126a4c023a 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -17,46 +17,46 @@ #include "tmsgcb.h" #include "taoserror.h" -static SMsgCb tsDefaultMsgCb; +static SMsgCb defaultMsgCb; -void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } +void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; } -int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg) { - PutToQueueFp fp = pMsgCb->queueFps[qtype]; - return (*fp)(pMsgCb->mgmt, pMsg); +int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) { + PutToQueueFp fp = msgcb->queueFps[qtype]; + return (*fp)(msgcb->mgmt, pMsg); } -int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { - GetQueueSizeFp fp = pMsgCb->qsizeFp; - return (*fp)(pMsgCb->mgmt, vgId, qtype); +int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) { + GetQueueSizeFp fp = msgcb->qsizeFp; + return (*fp)(msgcb->mgmt, vgId, qtype); } int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { - SendReqFp fp = tsDefaultMsgCb.sendReqFp; + SendReqFp fp = defaultMsgCb.sendReqFp; return (*fp)(epSet, pMsg); } void tmsgSendRsp(SRpcMsg* pMsg) { - SendRspFp fp = tsDefaultMsgCb.sendRspFp; + SendRspFp fp = defaultMsgCb.sendRspFp; return (*fp)(pMsg); } void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { - SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; + SendRedirectRspFp fp = defaultMsgCb.sendRedirectRspFp; (*fp)(pMsg, pNewEpSet); } void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { - RegisterBrokenLinkArgFp fp = tsDefaultMsgCb.registerBrokenLinkArgFp; + RegisterBrokenLinkArgFp fp = defaultMsgCb.registerBrokenLinkArgFp; (*fp)(pMsg); } void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { - ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; + ReleaseHandleFp fp = defaultMsgCb.releaseHandleFp; (*fp)(pHandle, type); } void tmsgReportStartup(const char* name, const char* desc) { - ReportStartup fp = tsDefaultMsgCb.reportStartupFp; + ReportStartup fp = defaultMsgCb.reportStartupFp; (*fp)(name, desc); } \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 3a26388d14..ecfa37725a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -78,7 +78,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); if (pWrapper->ntype == DNODE || InChildProc(pWrapper)) { - tmsgSetDefaultMsgCb(&input.msgCb); + tmsgSetDefault(&input.msgCb); } if (OnlyInSingleProc(pWrapper)) { diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 82acd7fddc..c4ed48fe60 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -56,7 +56,7 @@ class MndTestTrans2 : public ::testing::Test { msgCb.sendReqFp = sendReq; msgCb.sendRspFp = sendRsp; msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack - tmsgSetDefaultMsgCb(&msgCb); + tmsgSetDefault(&msgCb); SMnodeOpt opt = {0}; opt.deploy = 1; From 8daaea58de512252bc8091644d830fe632a55dd3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 May 2022 18:51:40 +0800 Subject: [PATCH 070/103] feat(query): fix ci --- source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 748fb60ef9..a20d0e4718 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -74,6 +74,7 @@ int32_t diffFunction(SqlFunctionCtx *pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); +int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4672105841..5358930df0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -875,7 +875,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, - .finalizeFunc = functionFinalize + .finalizeFunc = lastFinalize }, { .name = "diff", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ca10d3513e..d54cc96611 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1981,6 +1981,19 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + + char* in = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + + return pResInfo->numOfRes; +} + bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDiffInfo); return true; From 5ca0e81bba2663fd8289daeafbd03ed8e6604611 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 17 May 2022 17:38:21 +0800 Subject: [PATCH 071/103] stream final interval operator --- include/libs/nodes/nodes.h | 1 + source/libs/executor/inc/executorimpl.h | 27 ++- source/libs/executor/src/executorimpl.c | 35 ++-- source/libs/executor/src/scanoperator.c | 120 ++++++++++- source/libs/executor/src/timewindowoperator.c | 196 +++++++++++++++++- 5 files changed, 342 insertions(+), 37 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 27dae6d210..291e08fdbf 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -208,6 +208,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bf178612ba..e010363a1e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -378,6 +378,13 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_DATAREADER, } EStreamScanMode; +typedef struct SCatchSupporter { + SHashObj* pWindowHashTable; // quick locate the window object for each window + SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file + int32_t keySize; + int64_t* pKeyBuf; +} SCatchSupporter; + typedef struct SStreamBlockScanInfo { SArray* pBlockLists; // multiple SSDatablock. SSDataBlock* pRes; // result SSDataBlock @@ -400,6 +407,8 @@ typedef struct SStreamBlockScanInfo { EStreamScanMode scanMode; SOperatorInfo* pOperatorDumy; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. + SCatchSupporter childAggSup; + SArray* childIds; } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -460,6 +469,16 @@ typedef struct SIntervalAggOperatorInfo { bool invertible; } SIntervalAggOperatorInfo; +typedef struct SStreamFinalIntervalOperatorInfo { + SOptrBasicInfo binfo; // basic info + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + SAggSupporter aggSup; // aggregate supporter + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; +} SStreamFinalIntervalOperatorInfo; + typedef struct SAggOperatorInfo { SOptrBasicInfo binfo; SAggSupporter aggSup; @@ -696,6 +715,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, @@ -771,9 +793,8 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); - -void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, - uint64_t groupId, int32_t numOfOutput); +int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize, + const char* pKey, const char* pDir); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d0a1840d72..e42ced4669 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -342,28 +342,6 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return pResultRow; } -void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, - uint64_t groupId, int32_t numOfOutput) { - SAggSupporter* pSup = &pInfo->aggSup; - SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(bytes)); - SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); - SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; - for (int32_t i = 0; i < numOfOutput; ++i) { - pCtx[i].resultInfo = getResultCell(pResult, i, pInfo->binfo.rowCellInfoOffset); - struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; - if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) { - continue; - } - pResInfo->initialized = false; - if (pCtx[i].functionId != -1) { - pCtx[i].fpSet.init(&pCtx[i], pResInfo); - } - } -} - /** * the struct of key in hash table * +----------+---------------+ @@ -5321,3 +5299,16 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } + +int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize, + const char* pKey, const char* pDir) { + pCatchSup->keySize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY); + pCatchSup->pKeyBuf = taosMemoryCalloc(1, pCatchSup->keySize); + int32_t pageSize = rowSize * 32; + int32_t bufSize = pageSize * 4096; + createDiskbasedBuf(&pCatchSup->pDataBuf, pageSize, bufSize, pKey, pDir); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);; + return TSDB_CODE_SUCCESS; +} + \ No newline at end of file diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d042e463f0..2e3f8c8044 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -36,6 +36,11 @@ #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) +typedef struct SWindowPosition { + int32_t pageId; + int32_t rowId; +} SWindowPosition; + static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); @@ -675,6 +680,96 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti return NULL; } +void static setSupKeyBuf(SCatchSupporter* pSup, int64_t groupId, int64_t childId, TSKEY ts) { + int64_t* pKey = (int64_t*)pSup->pKeyBuf; + pKey[0] = groupId; + pKey[1] = childId; + pKey[2] = ts; +} + +static int32_t catchWidonwInfo(SSDataBlock* pDataBlock, SCatchSupporter* pSup, + int32_t pageId, int32_t tsIndex, int64_t childId) { + SColumnInfoData* pColDataInfo = taosArrayGet(pDataBlock->pDataBlock, tsIndex); + TSKEY* tsCols = (int64_t*)pColDataInfo->pData; + for (int32_t i = 0; i < pDataBlock->info.rows; i++) { + setSupKeyBuf(pSup, pDataBlock->info.groupId, childId, tsCols[i]); + SWindowPosition* p1 = (SWindowPosition*)taosHashGet(pSup->pWindowHashTable, + pSup->pKeyBuf, pSup->keySize); + if (p1 == NULL) { + SWindowPosition pos = {.pageId = pageId, .rowId = i}; + int32_t code = taosHashPut(pSup->pWindowHashTable, pSup->pKeyBuf, pSup->keySize, &pos, + sizeof(SWindowPosition)); + if (code != TSDB_CODE_SUCCESS ) { + return code; + } + } else { + p1->pageId = pageId; + p1->rowId = i; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t catchDatablock(SSDataBlock* pDataBlock, SCatchSupporter* pSup, + int32_t tsIndex, int64_t childId) { + int32_t start = 0; + int32_t stop = 0; + int32_t pageSize = getBufPageSize(pSup->pDataBuf); + while(start < pDataBlock->info.rows) { + blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pageSize); + SSDataBlock* pDB = blockDataExtractBlock(pDataBlock, start, stop - start + 1); + if (pDB == NULL) { + return terrno; + } + int32_t pageId = -1; + void* pPage = getNewBufPage(pSup->pDataBuf, pDataBlock->info.groupId, &pageId); + if (pPage == NULL) { + blockDataDestroy(pDB); + return terrno; + } + int32_t size = blockDataGetSize(pDB) + sizeof(int32_t) + pDB->info.numOfCols * sizeof(int32_t); + assert(size <= pageSize); + blockDataToBuf(pPage, pDB); + setBufPageDirty(pPage, true); + releaseBufPage(pSup->pDataBuf, pPage); + blockDataDestroy(pDB); + start = stop + 1; + int32_t code = catchWidonwInfo(pDB, pSup, pageId, tsIndex, childId); + if (code != TSDB_CODE_SUCCESS ) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* getDataFromCatch(SStreamBlockScanInfo* pInfo) { + SSDataBlock* pBlock = pInfo->pUpdateRes; + if (pInfo->updateResIndex < pBlock->info.rows) { + blockDataCleanup(pInfo->pRes); + SCatchSupporter* pCSup = &pInfo->childAggSup; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, 0); + TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; + int32_t size = taosArrayGetSize(pInfo->childIds); + for (int32_t i = 0; i < size; i++) { + int64_t id = *(int64_t *)taosArrayGet(pInfo->childIds, i); + setSupKeyBuf(pCSup, pBlock->info.groupId, id, + tsCols[pInfo->updateResIndex]); + SWindowPosition* pos = (SWindowPosition*)taosHashGet(pCSup->pWindowHashTable, + pCSup->pKeyBuf, pCSup->keySize); + void* buf = getBufPage(pCSup->pDataBuf, pos->pageId); + SSDataBlock* pDB = createOneDataBlock(pInfo->pRes, false); + blockDataFromBuf(pDB, buf); + SSDataBlock* pSub = blockDataExtractBlock(pDB, pos->rowId, 1); + blockDataMerge(pInfo->pRes, pSub, NULL); + blockDataDestroy(pDB); + blockDataDestroy(pSub); + } + pInfo->updateResIndex++; + return pInfo->pRes; + } + return NULL; +} + static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -688,6 +783,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { size_t total = taosArrayGetSize(pInfo->pBlockLists); if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { + SSDataBlock* pDB = getDataFromCatch(pInfo); + if (pDB != NULL) { + return pDB; + } else { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } + } + if (pInfo->validBlockIndex >= total) { doClearBufferedBlocks(pInfo); pOperator->status = OP_EXEC_DONE; @@ -695,7 +799,17 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } int32_t current = pInfo->validBlockIndex++; - return taosArrayGetP(pInfo->pBlockLists, current); + SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); + if (pBlock->info.type == STREAM_REPROCESS) { + pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; + } else { + int32_t code = catchDatablock(pBlock, &pInfo->childAggSup, pInfo->primaryTsIndex, 0); + if (code != TDB_CODE_SUCCESS) { + pTaskInfo->code = code; + longjmp(pTaskInfo->env, code); + } + } + return pBlock; } else { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { blockDataDestroy(pInfo->pUpdateRes); @@ -857,6 +971,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR pInfo->pOperatorDumy = pOperatorDumy; pInfo->interval = pSTInfo->interval; + size_t childKeyBufSize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY); + initCatchSupporter(&pInfo->childAggSup, 1024, childKeyBufSize, + "StreamFinalInterval", "/tmp/"); // TODO(liuyao) get row size from phy plan + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blocking = false; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 479ce394b1..62732aa1f9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -8,6 +8,8 @@ typedef enum SResultTsInterpType { RESULT_ROW_END_INTERP = 2, } SResultTsInterpType; +static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator); + /* * There are two cases to handle: * @@ -473,8 +475,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun } static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, int32_t prevPosition, SIntervalAggOperatorInfo* pInfo) { - int32_t order = pInfo->order; + TSKEY* primaryKeys, int32_t prevPosition, int32_t order) { bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; @@ -723,7 +724,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { break; } @@ -1031,18 +1032,41 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } } } -static void doClearWindows(SIntervalAggOperatorInfo* pInfo, int32_t numOfOutput, SSDataBlock* pBlock) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + +void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, + int16_t bytes, uint64_t groupId, int32_t numOfOutput) { + SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); + SResultRowPosition* p1 = + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, + GET_RES_WINDOW_KEY_LEN(bytes)); + SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); + SqlFunctionCtx* pCtx = pBinfo->pCtx; + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].resultInfo = getResultCell(pResult, i, pBinfo->rowCellInfoOffset); + struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; + if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) { + continue; + } + pResInfo->initialized = false; + if (pCtx[i].functionId != -1) { + pCtx[i].fpSet.init(&pCtx[i], pResInfo); + } + } +} + +static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, + SInterval* pIntrerval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], &pInfo->interval, - pInfo->interval.precision, NULL); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pIntrerval, + pIntrerval->precision, NULL); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - doClearWindow(pInfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); + doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); } } @@ -1084,7 +1108,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type); } if (pBlock->info.type == STREAM_REPROCESS) { - doClearWindows(pInfo, pOperator->numOfExprs, pBlock); + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, + pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock); continue; } pInfo->order = TSDB_ORDER_ASC; @@ -1097,8 +1122,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - // TODO: remove for stream - /*ASSERT(pInfo->binfo.pRes->info.rows > 0);*/ pOperator->status = OP_RES_TO_RETURN; return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; @@ -1116,6 +1139,12 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { cleanupAggSup(&pInfo->aggSup); } +void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo *)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + cleanupAggSup(&pInfo->aggSup); +} + bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { if (!fmIsInvertible(pFCtx[i].functionId)) { @@ -1185,6 +1214,63 @@ _error: return NULL; } +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, + keyBufSize, pTaskInfo->id.str); + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "StreamFinalIntervalOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, + destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, + NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, @@ -1548,3 +1634,91 @@ _error: pTaskInfo->code = code; return NULL; } + +static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, + int32_t tableGroupId) { + SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardStep = 0; + + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); + STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, + &pInfo->interval, pInfo->interval.precision, NULL); + while (1) { + int32_t code = + setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + pInfo->order, false); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); + if (startPos < 0) { + break; + } + } + return pUpdated; +} + +static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { + SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SArray* pUpdated = NULL; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } else if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + } + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); + if (pBlock->info.type == STREAM_REPROCESS) { + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, + pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock); + continue; + } + pUpdated = doHashInterval(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + } + + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + pOperator->status = OP_RES_TO_RETURN; + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; +} From 8b6aa46d483526ab7594a339cc44ef27220fe35e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 11:54:52 +0000 Subject: [PATCH 072/103] fix alter table coredump --- source/dnode/vnode/src/meta/metaTable.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 3d1854927c..619458af24 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -442,7 +442,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl } entry.version = version; - int tlen; + int tlen; + SSchema *pNewSchema = NULL; switch (pAlterTbReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: if (pColumn) { @@ -451,8 +452,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl } pSchema->sver++; pSchema->nCols++; - pSchema->pSchema = - taosMemoryRealloc(entry.ntbEntry.schema.pSchema, sizeof(SSchema) * entry.ntbEntry.schema.nCols); + pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); + memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); + pSchema->pSchema = pNewSchema; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags; @@ -511,6 +513,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl metaULock(pMeta); + if (pNewSchema) taosMemoryFree(pNewSchema); tDecoderClear(&dc); tdbTbcClose(pTbDbc); tdbTbcClose(pUidIdxc); From bad77f8e8489189fefa0e3cf435b2fae9a885611 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 12:05:39 +0000 Subject: [PATCH 073/103] refact TDB --- source/libs/tdb/inc/tdb.h | 4 ++-- source/libs/tdb/src/db/tdbDb.c | 10 +++++----- source/libs/tdb/src/db/tdbTable.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 521b61e3c8..37d5e98c64 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -31,13 +31,13 @@ typedef struct STBC TBC; typedef struct STxn TXN; // TDB -int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb); +int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb); int tdbClose(TDB *pDb); int tdbBegin(TDB *pDb, TXN *pTxn); int tdbCommit(TDB *pDb, TXN *pTxn); // TTB -int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); int tdbTbClose(TTB *pTb); int tdbTbDrop(TTB *pTb); int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 912c1ecaaf..44c945c21b 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { +int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb) { TDB *pDb; int dsize; int zsize; @@ -25,7 +25,7 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { *ppDb = NULL; - dsize = strlen(rootDir); + dsize = strlen(dbname); zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; pPtr = (uint8_t *)tdbOsCalloc(1, zsize); @@ -37,12 +37,12 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { pPtr += sizeof(*pDb); // pDb->rootDir pDb->rootDir = pPtr; - memcpy(pDb->rootDir, rootDir, dsize); + memcpy(pDb->rootDir, dbname, dsize); pDb->rootDir[dsize] = '\0'; pPtr = pPtr + dsize + 1; // pDb->jfname pDb->jfname = pPtr; - memcpy(pDb->jfname, rootDir, dsize); + memcpy(pDb->jfname, dbname, dsize); pDb->jfname[dsize] = '/'; memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; @@ -62,7 +62,7 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { } memset(pDb->pgrHash, 0, tsize); - mkdir(rootDir, 0755); + mkdir(dbname, 0755); *ppDb = pDb; return 0; diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index ce7da0b471..9a8f7a2f55 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -24,7 +24,7 @@ struct STBC { SBTC btc; }; -int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { +int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { TTB *pTb; SPager *pPager; int ret; @@ -42,9 +42,9 @@ int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn // pTb->pEnv pTb->pEnv = pEnv; - pPager = tdbEnvGetPager(pEnv, fname); + pPager = tdbEnvGetPager(pEnv, tbname); if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, tbname); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { return -1; From cda6fa50a91dc1bba09d9ab63ab88ca6b38167e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 18 May 2022 12:19:32 +0000 Subject: [PATCH 074/103] refact: tdb --- source/libs/tdb/inc/tdb.h | 56 +++++++++++++++---------------- source/libs/tdb/src/db/tdbDb.c | 18 +++++----- source/libs/tdb/src/db/tdbTable.c | 2 +- source/libs/tdb/src/inc/tdbInt.h | 4 +-- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 37d5e98c64..5912fd800c 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,7 +22,7 @@ extern "C" { #endif -typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int32_t kLen1, const void *pKey2, int32_t kLen2); // exposed types typedef struct STDB TDB; @@ -31,42 +31,42 @@ typedef struct STBC TBC; typedef struct STxn TXN; // TDB -int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb); -int tdbClose(TDB *pDb); -int tdbBegin(TDB *pDb, TXN *pTxn); -int tdbCommit(TDB *pDb, TXN *pTxn); +int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb); +int32_t tdbClose(TDB *pDb); +int32_t tdbBegin(TDB *pDb, TXN *pTxn); +int32_t tdbCommit(TDB *pDb, TXN *pTxn); // TTB -int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); -int tdbTbClose(TTB *pTb); -int tdbTbDrop(TTB *pTb); -int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); -int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int32_t tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int32_t tdbTbClose(TTB *pTb); +int32_t tdbTbDrop(TTB *pTb); +int32_t tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int32_t tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); +int32_t tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int32_t tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); +int32_t tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // TBC -int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); -int tdbTbcClose(TBC *pTbc); -int tdbTbcIsValid(TBC *pTbc); -int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); -int tdbTbcMoveToFirst(TBC *pTbc); -int tdbTbcMoveToLast(TBC *pTbc); -int tdbTbcMoveToNext(TBC *pTbc); -int tdbTbcMoveToPrev(TBC *pTbc); -int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); -int tdbTbcDelete(TBC *pTbc); -int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); +int32_t tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); +int32_t tdbTbcClose(TBC *pTbc); +int32_t tdbTbcIsValid(TBC *pTbc); +int32_t tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); +int32_t tdbTbcMoveToFirst(TBC *pTbc); +int32_t tdbTbcMoveToLast(TBC *pTbc); +int32_t tdbTbcMoveToNext(TBC *pTbc); +int32_t tdbTbcMoveToPrev(TBC *pTbc); +int32_t tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); +int32_t tdbTbcDelete(TBC *pTbc); +int32_t tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int32_t tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 #define TDB_TXN_READ_UNCOMMITTED 0x2 -int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, - int flags); -int tdbTxnClose(TXN *pTxn); +int32_t tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), + void *xArg, int flags); +int32_t tdbTxnClose(TXN *pTxn); // other void tdbFree(void *); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 44c945c21b..c06f7305ab 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb) { +int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb) { TDB *pDb; int dsize; int zsize; @@ -36,16 +36,16 @@ int tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb) { pDb = (TDB *)pPtr; pPtr += sizeof(*pDb); // pDb->rootDir - pDb->rootDir = pPtr; - memcpy(pDb->rootDir, dbname, dsize); - pDb->rootDir[dsize] = '\0'; + pDb->dbName = pPtr; + memcpy(pDb->dbName, dbname, dsize); + pDb->dbName[dsize] = '\0'; pPtr = pPtr + dsize + 1; // pDb->jfname - pDb->jfname = pPtr; - memcpy(pDb->jfname, dbname, dsize); - pDb->jfname[dsize] = '/'; - memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); - pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; + pDb->jnName = pPtr; + memcpy(pDb->jnName, dbname, dsize); + pDb->jnName[dsize] = '/'; + memcpy(pDb->jnName + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pDb->jnName[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; pDb->jfd = -1; diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 9a8f7a2f55..7211fe4926 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -44,7 +44,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF pPager = tdbEnvGetPager(pEnv, tbname); if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, tbname); + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { return -1; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index bb59a7fc61..2c79f39bc0 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -335,8 +335,8 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { } struct STDB { - char *rootDir; - char *jfname; + char *dbName; + char *jnName; int jfd; SPCache *pCache; SPager *pgrList; From 6f4c0a2e8fd2b0eb51eee7cb2e85974cb5d52846 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 18 May 2022 20:28:14 +0800 Subject: [PATCH 075/103] fix: some problems of parser and planner --- include/libs/nodes/querynodes.h | 1 + include/util/taoserror.h | 1 + source/libs/parser/inc/sql.y | 1 + source/libs/parser/src/parAstCreater.c | 9 +- source/libs/parser/src/parTranslater.c | 20 +- source/libs/parser/src/parUtil.c | 2 + source/libs/parser/src/sql.c | 1766 ++++++++++---------- source/libs/parser/test/parSelectTest.cpp | 6 + source/libs/planner/inc/planInt.h | 13 +- source/libs/planner/src/planLogicCreater.c | 12 +- source/libs/planner/src/planOptimizer.c | 72 +- source/libs/planner/src/planSpliter.c | 108 +- source/libs/planner/test/planJoinTest.cpp | 8 + source/libs/planner/test/planSetOpTest.cpp | 26 +- 14 files changed, 1083 insertions(+), 962 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 8c1482894a..ee986d5aab 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -248,6 +248,7 @@ typedef struct SSetOperator { SNode* pRight; SNodeList* pOrderByList; // SOrderByExprNode SNode* pLimit; + char stmtName[TSDB_TABLE_NAME_LEN]; } SSetOperator; typedef enum ESqlClause { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index aaf5a44514..be6005d46b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -646,6 +646,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_ALTER_TABLE TAOS_DEF_ERROR_CODE(0, 0x2649) #define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x264A) #define TSDB_CODE_PAR_INVALID_MODIFY_COL TAOS_DEF_ERROR_CODE(0, 0x264B) +#define TSDB_CODE_PAR_INVALID_TBNAME TAOS_DEF_ERROR_CODE(0, 0x264C) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 836e97c4db..10bb47d3ff 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -621,6 +621,7 @@ column_reference(A) ::= table_name(B) NK_DOT column_name(C). pseudo_column(A) ::= ROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= TBNAME(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= table_name(B) NK_DOT TBNAME(C). { A = createRawExprNodeExt(pCxt, &B, &C, createFunctionNode(pCxt, &C, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)))); } pseudo_column(A) ::= QSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= QENDTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5de6968c51..a438a1e843 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -155,9 +155,9 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t return TSDB_CODE_SUCCESS == pCxt->errCode; } -static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) { +static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb) { if (NULL == pDbName) { - if (query && NULL == pCxt->pQueryCxt->db) { + if (demandDb && NULL == pCxt->pQueryCxt->db) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED); } } else { @@ -457,6 +457,8 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const STok } if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias); + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pSubquery)) { + strcpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias); } return (SNode*)tempTable; } @@ -637,6 +639,7 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* setOp->opType = type; setOp->pLeft = pLeft; setOp->pRight = pRight; + sprintf(setOp->stmtName, "%p", setOp); return (SNode*)setOp; } @@ -1140,7 +1143,7 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions) { - if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { + if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName) || !checkDbName(pCxt, NULL, true)) { return NULL; } SCreateIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_INDEX_STMT); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 07a0d357ba..18e0843e92 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -766,6 +766,20 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) pCxt->pCurrStmt->hasRepeatScanFuncs = true; } } + if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsScanPseudoColumnFunc(pFunc->funcId)) { + if (0 == LIST_LENGTH(pFunc->pParameterList)) { + if (QUERY_NODE_REAL_TABLE != nodeType(pCxt->pCurrStmt->pFromTable)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TBNAME); + } + } else { + SValueNode* pVal = nodesListGetNode(pFunc->pParameterList, 0); + STableNode* pTable = NULL; + pCxt->errCode = findTable(pCxt, pVal->literal, &pTable); + if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TBNAME); + } + } + } return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } @@ -1758,12 +1772,13 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } -static SNode* createSetOperProject(SNode* pNode) { +static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { return NULL; } pCol->node.resType = ((SExprNode*)pNode)->resType; + strcpy(pCol->tableAlias, pTableAlias); strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); strcpy(pCol->node.aliasName, pCol->colName); return (SNode*)pCol; @@ -1817,7 +1832,8 @@ static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* p } strcpy(pRightExpr->aliasName, pLeftExpr->aliasName); pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0'; - if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pLeft))) { + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, + createSetOperProject(pSetOperator->stmtName, pLeft))) { return TSDB_CODE_OUT_OF_MEMORY; } } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index cb2d1b7c07..1ae066a8c2 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -158,6 +158,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Primary timestamp column cannot be dropped"; case TSDB_CODE_PAR_INVALID_MODIFY_COL: return "Only binary/nchar column length could be modified"; + case TSDB_CODE_PAR_INVALID_TBNAME: + return "Invalid tbname pseudo column"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index ed812405e0..795b5ca641 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -134,17 +134,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 603 -#define YYNRULE 451 +#define YYNSTATE 605 +#define YYNRULE 452 #define YYNTOKEN 238 -#define YY_MAX_SHIFT 602 -#define YY_MIN_SHIFTREDUCE 890 -#define YY_MAX_SHIFTREDUCE 1340 -#define YY_ERROR_ACTION 1341 -#define YY_ACCEPT_ACTION 1342 -#define YY_NO_ACTION 1343 -#define YY_MIN_REDUCE 1344 -#define YY_MAX_REDUCE 1794 +#define YY_MAX_SHIFT 604 +#define YY_MIN_SHIFTREDUCE 893 +#define YY_MAX_SHIFTREDUCE 1344 +#define YY_ERROR_ACTION 1345 +#define YY_ACCEPT_ACTION 1346 +#define YY_NO_ACTION 1347 +#define YY_MIN_REDUCE 1348 +#define YY_MAX_REDUCE 1799 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -213,223 +213,223 @@ typedef union { *********** Begin parsing tables **********************************************/ #define YY_ACTTAB_COUNT (2167) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 382, 76, 383, 1376, 390, 517, 383, 1376, 1627, 28, - /* 10 */ 221, 1463, 35, 33, 113, 1459, 342, 346, 1642, 1772, - /* 20 */ 300, 1466, 1156, 1623, 1631, 1629, 36, 34, 32, 31, - /* 30 */ 30, 1724, 1771, 1474, 94, 520, 1769, 93, 92, 91, - /* 40 */ 90, 89, 88, 87, 86, 85, 1658, 1154, 1519, 291, - /* 50 */ 32, 31, 30, 501, 290, 1721, 1772, 477, 14, 1517, - /* 60 */ 35, 33, 1281, 500, 1162, 516, 1627, 1613, 300, 144, - /* 70 */ 1156, 348, 273, 1769, 397, 36, 34, 32, 31, 30, - /* 80 */ 1, 1623, 1630, 1629, 1670, 112, 24, 132, 1643, 503, - /* 90 */ 1645, 1646, 499, 520, 520, 1154, 36, 34, 32, 31, - /* 100 */ 30, 61, 599, 1243, 270, 481, 14, 517, 35, 33, - /* 110 */ 595, 594, 1162, 1155, 108, 516, 300, 304, 1156, 104, - /* 120 */ 1178, 273, 1469, 110, 936, 128, 418, 130, 2, 1356, - /* 130 */ 54, 482, 1786, 1476, 1772, 1474, 517, 69, 210, 1717, - /* 140 */ 476, 1344, 475, 1154, 1193, 1772, 1772, 145, 104, 416, - /* 150 */ 599, 1769, 1243, 1244, 14, 423, 1157, 1467, 146, 1770, - /* 160 */ 1162, 1155, 1769, 1769, 1474, 103, 102, 101, 100, 99, - /* 170 */ 98, 97, 96, 95, 1249, 38, 2, 552, 1160, 1161, - /* 180 */ 54, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, - /* 190 */ 1220, 1221, 1222, 1223, 1224, 1225, 551, 550, 599, 549, - /* 200 */ 548, 547, 1244, 493, 1157, 341, 1305, 340, 147, 1155, - /* 210 */ 27, 298, 1238, 1239, 1240, 1241, 1242, 1246, 1247, 1248, - /* 220 */ 131, 1180, 504, 1249, 1431, 303, 1160, 1161, 1564, 1206, - /* 230 */ 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, - /* 240 */ 1222, 1223, 1224, 1225, 1403, 464, 1303, 1304, 1306, 1307, - /* 250 */ 55, 54, 1157, 36, 34, 32, 31, 30, 147, 27, - /* 260 */ 298, 1238, 1239, 1240, 1241, 1242, 1246, 1247, 1248, 63, - /* 270 */ 288, 477, 554, 187, 1160, 1161, 1342, 1206, 1207, 1208, - /* 280 */ 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, 1222, 1223, - /* 290 */ 1224, 1225, 35, 33, 36, 34, 32, 31, 30, 112, - /* 300 */ 300, 1452, 1156, 147, 575, 574, 573, 315, 147, 572, - /* 310 */ 571, 570, 114, 565, 564, 563, 562, 561, 560, 559, - /* 320 */ 558, 121, 1295, 1235, 1519, 312, 54, 1154, 988, 1658, - /* 330 */ 305, 1642, 1555, 1557, 316, 1517, 501, 110, 138, 310, - /* 340 */ 35, 33, 1226, 471, 1162, 990, 920, 128, 300, 1513, - /* 350 */ 1156, 479, 140, 1717, 1718, 1476, 1722, 26, 397, 1658, - /* 360 */ 8, 36, 34, 32, 31, 30, 501, 36, 34, 32, - /* 370 */ 31, 30, 470, 467, 1772, 1154, 500, 1406, 313, 147, - /* 380 */ 1613, 53, 599, 461, 924, 925, 128, 144, 35, 33, - /* 390 */ 333, 1769, 1162, 1155, 1476, 516, 300, 1670, 1156, 1182, - /* 400 */ 267, 1643, 503, 1645, 1646, 499, 381, 520, 9, 385, - /* 410 */ 335, 331, 1026, 543, 542, 541, 1030, 540, 1032, 1033, - /* 420 */ 539, 1035, 536, 1154, 1041, 533, 1043, 1044, 530, 527, - /* 430 */ 599, 36, 34, 32, 31, 30, 1157, 432, 431, 128, - /* 440 */ 1162, 1155, 430, 472, 468, 109, 427, 1477, 517, 426, - /* 450 */ 425, 424, 1183, 153, 147, 39, 9, 1724, 1160, 1161, - /* 460 */ 347, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, - /* 470 */ 1220, 1221, 1222, 1223, 1224, 1225, 1474, 59, 599, 387, - /* 480 */ 58, 1720, 432, 431, 1157, 1178, 1465, 430, 147, 1155, - /* 490 */ 109, 427, 1519, 446, 426, 425, 424, 1337, 311, 546, - /* 500 */ 1193, 1156, 1367, 1517, 372, 454, 1160, 1161, 308, 1206, - /* 510 */ 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, - /* 520 */ 1222, 1223, 1224, 1225, 212, 1627, 1154, 36, 34, 32, - /* 530 */ 31, 30, 1157, 1772, 389, 1613, 1519, 385, 504, 1366, - /* 540 */ 1623, 1630, 1629, 1162, 1565, 1365, 144, 1518, 157, 156, - /* 550 */ 1769, 1613, 520, 455, 1160, 1161, 1642, 1206, 1207, 1208, - /* 560 */ 1209, 1210, 1211, 1212, 496, 518, 1220, 1221, 1222, 1223, - /* 570 */ 1224, 1225, 35, 33, 269, 517, 1178, 198, 1336, 61, - /* 580 */ 300, 599, 1156, 365, 1658, 517, 377, 357, 1613, 429, - /* 590 */ 428, 501, 1155, 1772, 1613, 1364, 173, 514, 1363, 1362, - /* 600 */ 1470, 500, 483, 1474, 378, 1613, 144, 1154, 137, 517, - /* 610 */ 1769, 481, 1361, 1474, 414, 410, 406, 402, 172, 282, - /* 620 */ 1181, 358, 1670, 1280, 1162, 258, 1643, 503, 1645, 1646, - /* 630 */ 499, 517, 520, 1117, 937, 1157, 936, 1474, 477, 1450, - /* 640 */ 2, 1119, 62, 396, 1613, 170, 1360, 1613, 1613, 569, - /* 650 */ 567, 1772, 1179, 11, 10, 1556, 1557, 1160, 1161, 1474, - /* 660 */ 127, 1613, 599, 938, 146, 1359, 112, 283, 1769, 281, - /* 670 */ 280, 7, 420, 1155, 376, 1358, 422, 371, 370, 369, - /* 680 */ 368, 367, 364, 363, 362, 361, 360, 356, 355, 354, - /* 690 */ 353, 352, 351, 350, 349, 1613, 554, 517, 517, 421, - /* 700 */ 517, 1118, 485, 169, 110, 164, 1451, 166, 477, 1471, - /* 710 */ 1593, 1355, 515, 1245, 1613, 1354, 1157, 1353, 1352, 141, - /* 720 */ 1717, 1718, 444, 1722, 1613, 1474, 1474, 162, 1474, 1351, - /* 730 */ 1642, 1350, 1349, 1257, 1250, 442, 112, 1348, 1160, 1161, - /* 740 */ 422, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 496, 518, - /* 750 */ 1220, 1221, 1222, 1223, 1224, 1225, 517, 568, 1658, 517, - /* 760 */ 1613, 1347, 1552, 421, 1613, 501, 1613, 1613, 235, 155, - /* 770 */ 25, 314, 924, 925, 110, 500, 1729, 1276, 1613, 1613, - /* 780 */ 1613, 1613, 1288, 1724, 1474, 481, 1613, 1474, 1180, 142, - /* 790 */ 1717, 1718, 245, 1722, 1602, 1504, 1670, 336, 552, 79, - /* 800 */ 1643, 503, 1645, 1646, 499, 1642, 520, 1719, 178, 1710, - /* 810 */ 1613, 176, 180, 272, 1706, 179, 182, 551, 550, 181, - /* 820 */ 549, 548, 547, 1393, 557, 1772, 1446, 184, 1279, 1231, - /* 830 */ 183, 1388, 118, 1658, 1386, 1180, 1141, 1142, 146, 323, - /* 840 */ 501, 1276, 1769, 45, 129, 433, 201, 46, 271, 251, - /* 850 */ 500, 11, 10, 435, 1613, 1633, 438, 1339, 1340, 37, - /* 860 */ 481, 249, 52, 488, 556, 51, 1642, 1461, 37, 1457, - /* 870 */ 37, 1670, 456, 190, 79, 1643, 503, 1645, 1646, 499, - /* 880 */ 437, 520, 158, 1302, 1710, 224, 203, 486, 272, 1706, - /* 890 */ 116, 1635, 75, 117, 1658, 445, 495, 545, 1357, 1251, - /* 900 */ 1772, 480, 71, 453, 118, 1165, 54, 1164, 1213, 186, - /* 910 */ 1112, 500, 45, 144, 216, 1613, 1449, 1769, 1432, 1642, - /* 920 */ 525, 440, 117, 465, 447, 226, 434, 118, 207, 1377, - /* 930 */ 509, 185, 1670, 232, 415, 80, 1643, 503, 1645, 1646, - /* 940 */ 499, 119, 520, 78, 1019, 1710, 117, 1658, 1659, 293, - /* 950 */ 1706, 139, 244, 1514, 501, 50, 1740, 478, 49, 215, - /* 960 */ 1047, 961, 1051, 213, 500, 218, 220, 1058, 1613, 460, - /* 970 */ 1737, 1168, 1642, 1167, 57, 56, 345, 3, 962, 152, - /* 980 */ 1178, 1056, 318, 322, 339, 1670, 120, 278, 262, 1643, - /* 990 */ 503, 1645, 1646, 499, 988, 520, 268, 279, 359, 329, - /* 1000 */ 1658, 325, 321, 149, 240, 1125, 1554, 480, 552, 154, - /* 1010 */ 366, 374, 379, 1184, 380, 373, 388, 500, 375, 1187, - /* 1020 */ 391, 1613, 161, 392, 1186, 163, 473, 551, 550, 393, - /* 1030 */ 549, 548, 547, 165, 147, 394, 1185, 168, 1670, 1345, - /* 1040 */ 395, 80, 1643, 503, 1645, 1646, 499, 1642, 520, 60, - /* 1050 */ 489, 1710, 398, 417, 171, 293, 1706, 139, 419, 1464, - /* 1060 */ 94, 175, 1460, 93, 92, 91, 90, 89, 88, 87, - /* 1070 */ 86, 85, 177, 84, 1162, 1658, 1738, 122, 123, 287, - /* 1080 */ 1642, 188, 501, 1462, 241, 1458, 124, 1597, 125, 448, - /* 1090 */ 1183, 191, 500, 452, 193, 449, 1613, 457, 196, 466, - /* 1100 */ 1751, 1750, 507, 6, 1731, 206, 474, 242, 1658, 462, - /* 1110 */ 463, 458, 5, 1670, 135, 501, 80, 1643, 503, 1645, - /* 1120 */ 1646, 499, 292, 520, 199, 500, 1710, 208, 202, 1613, - /* 1130 */ 293, 1706, 1785, 111, 1276, 469, 1182, 1741, 40, 490, - /* 1140 */ 487, 1744, 1642, 1725, 1788, 18, 1670, 294, 510, 80, - /* 1150 */ 1643, 503, 1645, 1646, 499, 1563, 520, 505, 506, 1710, - /* 1160 */ 1562, 511, 512, 293, 1706, 1785, 228, 302, 209, 230, - /* 1170 */ 1658, 243, 68, 1475, 1767, 70, 523, 501, 1691, 1447, - /* 1180 */ 246, 237, 598, 47, 134, 252, 289, 500, 259, 248, - /* 1190 */ 1768, 1613, 253, 214, 1607, 250, 1642, 1606, 484, 217, - /* 1200 */ 317, 1603, 319, 320, 219, 1642, 491, 1150, 1670, 1151, - /* 1210 */ 150, 80, 1643, 503, 1645, 1646, 499, 324, 520, 1601, - /* 1220 */ 326, 1710, 327, 328, 1658, 293, 1706, 1785, 1600, 330, - /* 1230 */ 1599, 501, 332, 1658, 1598, 334, 1728, 1583, 151, 337, - /* 1240 */ 501, 500, 338, 1128, 1127, 1613, 343, 344, 1575, 1574, - /* 1250 */ 500, 1577, 1576, 1095, 1613, 1547, 1546, 1545, 1544, 1642, - /* 1260 */ 481, 1543, 1670, 1542, 1541, 132, 1643, 503, 1645, 1646, - /* 1270 */ 499, 1670, 520, 1540, 258, 1643, 503, 1645, 1646, 499, - /* 1280 */ 1539, 520, 1538, 1537, 1536, 1535, 1534, 1658, 1533, 1532, - /* 1290 */ 1531, 1530, 1529, 115, 501, 1528, 1527, 1526, 1525, 1642, - /* 1300 */ 1772, 1524, 1523, 1097, 500, 1522, 1521, 1520, 1613, 159, - /* 1310 */ 1787, 1405, 1373, 144, 106, 136, 384, 1769, 1642, 386, - /* 1320 */ 1372, 1591, 927, 1585, 926, 1670, 1569, 1658, 81, 1643, - /* 1330 */ 503, 1645, 1646, 499, 501, 520, 1560, 160, 1710, 400, - /* 1340 */ 107, 167, 1709, 1706, 500, 1453, 1658, 1404, 1613, 1402, - /* 1350 */ 1400, 401, 955, 498, 405, 1398, 1396, 1385, 1384, 409, - /* 1360 */ 399, 404, 403, 500, 413, 1670, 407, 1613, 81, 1643, - /* 1370 */ 503, 1645, 1646, 499, 1642, 520, 408, 411, 1710, 412, - /* 1380 */ 1371, 1455, 492, 1706, 1670, 1062, 1454, 266, 1643, 503, - /* 1390 */ 1645, 1646, 499, 497, 520, 494, 1682, 1061, 987, 986, - /* 1400 */ 985, 984, 1658, 602, 566, 1394, 568, 83, 981, 501, - /* 1410 */ 284, 174, 1389, 980, 979, 285, 436, 239, 1387, 500, - /* 1420 */ 286, 439, 1370, 1613, 441, 1369, 443, 82, 1590, 105, - /* 1430 */ 1135, 1584, 48, 1642, 1568, 591, 587, 583, 579, 238, - /* 1440 */ 1670, 450, 192, 81, 1643, 503, 1645, 1646, 499, 1567, - /* 1450 */ 520, 1559, 64, 1710, 195, 4, 37, 15, 1707, 200, - /* 1460 */ 1301, 1658, 43, 77, 205, 133, 233, 204, 501, 197, - /* 1470 */ 22, 1633, 1294, 451, 65, 10, 23, 16, 500, 42, - /* 1480 */ 1273, 1272, 1613, 211, 41, 297, 126, 1642, 143, 1330, - /* 1490 */ 1319, 307, 306, 17, 1325, 1324, 19, 295, 148, 1670, - /* 1500 */ 513, 1170, 267, 1643, 503, 1645, 1646, 499, 1329, 520, - /* 1510 */ 1328, 296, 29, 1236, 1215, 1658, 222, 1642, 1214, 1201, - /* 1520 */ 12, 20, 498, 502, 223, 459, 1163, 1299, 194, 1558, - /* 1530 */ 229, 1632, 500, 21, 234, 508, 1613, 225, 227, 66, - /* 1540 */ 67, 231, 1673, 1162, 13, 1658, 1133, 1217, 189, 519, - /* 1550 */ 1642, 44, 501, 1670, 1172, 71, 266, 1643, 503, 1645, - /* 1560 */ 1646, 499, 500, 520, 524, 1683, 1613, 522, 309, 299, - /* 1570 */ 1048, 526, 528, 1045, 529, 531, 1042, 1036, 1658, 532, - /* 1580 */ 1642, 521, 534, 1670, 535, 501, 267, 1643, 503, 1645, - /* 1590 */ 1646, 499, 1166, 520, 1034, 500, 537, 1025, 538, 1613, - /* 1600 */ 544, 1057, 301, 72, 1040, 1054, 1039, 1038, 1658, 1037, - /* 1610 */ 1642, 73, 477, 74, 1053, 501, 1670, 994, 953, 267, - /* 1620 */ 1643, 503, 1645, 1646, 499, 500, 520, 1055, 553, 1613, - /* 1630 */ 555, 236, 975, 974, 973, 1171, 972, 970, 1658, 1401, - /* 1640 */ 112, 971, 969, 968, 991, 501, 1670, 989, 965, 254, - /* 1650 */ 1643, 503, 1645, 1646, 499, 500, 520, 1174, 964, 1613, - /* 1660 */ 481, 1642, 578, 963, 960, 959, 958, 576, 518, 1220, - /* 1670 */ 1221, 1399, 577, 582, 581, 580, 1670, 1642, 110, 261, - /* 1680 */ 1643, 503, 1645, 1646, 499, 1397, 520, 584, 585, 1658, - /* 1690 */ 586, 1395, 588, 210, 1717, 476, 501, 475, 590, 589, - /* 1700 */ 1772, 1383, 592, 593, 1382, 1658, 500, 1368, 596, 597, - /* 1710 */ 1613, 1158, 501, 144, 247, 600, 601, 1769, 1343, 1343, - /* 1720 */ 1343, 1343, 500, 1343, 1642, 1343, 1613, 1670, 1343, 1343, - /* 1730 */ 263, 1643, 503, 1645, 1646, 499, 1343, 520, 1343, 1343, - /* 1740 */ 1343, 1343, 1642, 1670, 1343, 1343, 255, 1643, 503, 1645, - /* 1750 */ 1646, 499, 1658, 520, 1343, 1343, 1343, 1642, 1343, 501, - /* 1760 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 500, - /* 1770 */ 1658, 1343, 1343, 1613, 1343, 1343, 1343, 501, 1343, 1343, - /* 1780 */ 1343, 1343, 1343, 1343, 1343, 1658, 1343, 500, 1343, 1343, - /* 1790 */ 1670, 1613, 501, 264, 1643, 503, 1645, 1646, 499, 1343, - /* 1800 */ 520, 1343, 500, 1343, 1642, 1343, 1613, 1343, 1670, 1343, - /* 1810 */ 1343, 256, 1643, 503, 1645, 1646, 499, 1343, 520, 1343, - /* 1820 */ 1642, 1343, 1343, 1670, 1343, 1343, 265, 1643, 503, 1645, - /* 1830 */ 1646, 499, 1658, 520, 1343, 1343, 1343, 1343, 1343, 501, - /* 1840 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1658, 500, - /* 1850 */ 1343, 1343, 1343, 1613, 1343, 501, 1343, 1343, 1343, 1343, - /* 1860 */ 1343, 1343, 1343, 1343, 1343, 500, 1343, 1642, 1343, 1613, - /* 1870 */ 1670, 1343, 1343, 257, 1643, 503, 1645, 1646, 499, 1343, - /* 1880 */ 520, 1343, 1343, 1343, 1343, 1343, 1670, 1343, 1343, 1654, - /* 1890 */ 1643, 503, 1645, 1646, 499, 1658, 520, 1642, 1343, 1343, - /* 1900 */ 1343, 1343, 501, 1343, 1343, 1343, 1343, 1343, 1343, 1343, - /* 1910 */ 1343, 1343, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, - /* 1920 */ 1343, 1343, 1343, 1343, 1343, 1658, 1343, 1343, 1343, 1343, - /* 1930 */ 1343, 1343, 501, 1670, 1343, 1343, 1653, 1643, 503, 1645, - /* 1940 */ 1646, 499, 500, 520, 1343, 1343, 1613, 1642, 1343, 1343, - /* 1950 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, - /* 1960 */ 1343, 1343, 1343, 1670, 1343, 1343, 1652, 1643, 503, 1645, - /* 1970 */ 1646, 499, 1343, 520, 1343, 1658, 1343, 1642, 1343, 1343, - /* 1980 */ 1343, 1343, 501, 1343, 1343, 1343, 1343, 1343, 1343, 1343, - /* 1990 */ 1343, 1343, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, - /* 2000 */ 1343, 1343, 1343, 1343, 1343, 1658, 1343, 1343, 1343, 1343, - /* 2010 */ 1343, 1343, 501, 1670, 1343, 1343, 276, 1643, 503, 1645, - /* 2020 */ 1646, 499, 500, 520, 1343, 1343, 1613, 1642, 1343, 1343, - /* 2030 */ 1343, 1343, 1343, 1343, 1343, 1343, 1642, 1343, 1343, 1343, - /* 2040 */ 1343, 1343, 1343, 1670, 1343, 1343, 275, 1643, 503, 1645, - /* 2050 */ 1646, 499, 1343, 520, 1343, 1658, 1343, 1343, 1343, 1343, - /* 2060 */ 1343, 1343, 501, 1343, 1658, 1343, 1343, 1343, 1343, 1343, - /* 2070 */ 1343, 501, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, - /* 2080 */ 1343, 500, 1343, 1343, 1343, 1613, 1343, 1343, 1343, 1343, - /* 2090 */ 1642, 1343, 1343, 1670, 1343, 1343, 277, 1643, 503, 1645, - /* 2100 */ 1646, 499, 1670, 520, 1343, 274, 1643, 503, 1645, 1646, - /* 2110 */ 499, 1343, 520, 1343, 1343, 1343, 1343, 1343, 1658, 1343, - /* 2120 */ 1343, 1343, 1343, 1343, 1343, 501, 1343, 1343, 1343, 1343, - /* 2130 */ 1343, 1343, 1343, 1343, 1343, 500, 1343, 1343, 1343, 1613, - /* 2140 */ 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, 1343, - /* 2150 */ 1343, 1343, 1343, 1343, 1343, 1343, 1670, 1343, 1343, 260, - /* 2160 */ 1643, 503, 1645, 1646, 499, 1343, 520, + /* 0 */ 383, 76, 384, 1380, 391, 519, 384, 1380, 1631, 28, + /* 10 */ 223, 1467, 35, 33, 113, 1463, 343, 347, 1646, 1777, + /* 20 */ 301, 1470, 1159, 1627, 1635, 1633, 36, 34, 32, 31, + /* 30 */ 30, 1729, 1776, 1478, 94, 522, 1774, 93, 92, 91, + /* 40 */ 90, 89, 88, 87, 86, 85, 1662, 1157, 1523, 292, + /* 50 */ 32, 31, 30, 503, 291, 1726, 1777, 479, 14, 1521, + /* 60 */ 35, 33, 1285, 502, 1165, 518, 1631, 1617, 301, 145, + /* 70 */ 1159, 349, 274, 1774, 398, 36, 34, 32, 31, 30, + /* 80 */ 1, 1627, 1634, 1633, 1675, 112, 24, 132, 1647, 505, + /* 90 */ 1649, 1650, 501, 522, 522, 1157, 36, 34, 32, 31, + /* 100 */ 30, 61, 601, 1247, 271, 483, 14, 519, 35, 33, + /* 110 */ 597, 596, 1165, 1158, 108, 518, 301, 305, 1159, 104, + /* 120 */ 1181, 274, 1473, 110, 939, 128, 419, 130, 2, 1360, + /* 130 */ 54, 484, 1791, 1480, 1777, 1478, 519, 69, 212, 1722, + /* 140 */ 478, 1348, 477, 1157, 1196, 1777, 1777, 146, 104, 417, + /* 150 */ 601, 1774, 1247, 1248, 14, 424, 1160, 1471, 147, 1775, + /* 160 */ 1165, 1158, 1774, 1774, 1478, 103, 102, 101, 100, 99, + /* 170 */ 98, 97, 96, 95, 1253, 38, 2, 554, 1163, 1164, + /* 180 */ 54, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, + /* 190 */ 1224, 1225, 1226, 1227, 1228, 1229, 553, 552, 601, 551, + /* 200 */ 550, 549, 1248, 495, 1160, 342, 1309, 341, 148, 1158, + /* 210 */ 27, 299, 1242, 1243, 1244, 1245, 1246, 1250, 1251, 1252, + /* 220 */ 131, 1183, 506, 1253, 1435, 304, 1163, 1164, 1568, 1209, + /* 230 */ 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, + /* 240 */ 1226, 1227, 1228, 1229, 1407, 465, 1307, 1308, 1310, 1311, + /* 250 */ 55, 54, 1160, 36, 34, 32, 31, 30, 148, 27, + /* 260 */ 299, 1242, 1243, 1244, 1245, 1246, 1250, 1251, 1252, 63, + /* 270 */ 289, 479, 1284, 188, 1163, 1164, 1346, 1209, 1210, 1212, + /* 280 */ 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, 1226, 1227, + /* 290 */ 1228, 1229, 35, 33, 36, 34, 32, 31, 30, 112, + /* 300 */ 301, 1456, 1159, 148, 577, 576, 575, 316, 148, 574, + /* 310 */ 573, 572, 114, 567, 566, 565, 564, 563, 562, 561, + /* 320 */ 560, 121, 1299, 1239, 1523, 313, 54, 1157, 991, 1662, + /* 330 */ 306, 1646, 1559, 1561, 317, 1521, 472, 110, 140, 311, + /* 340 */ 35, 33, 1230, 473, 1165, 993, 923, 128, 301, 1517, + /* 350 */ 1159, 481, 142, 1722, 1723, 1480, 1727, 26, 398, 1662, + /* 360 */ 8, 36, 34, 32, 31, 30, 503, 36, 34, 32, + /* 370 */ 31, 30, 471, 468, 1777, 1157, 502, 1410, 314, 148, + /* 380 */ 1617, 556, 601, 462, 927, 928, 128, 145, 35, 33, + /* 390 */ 334, 1774, 1165, 1158, 1480, 518, 301, 1675, 1159, 53, + /* 400 */ 268, 1647, 505, 1649, 1650, 501, 382, 522, 9, 386, + /* 410 */ 336, 332, 1029, 545, 544, 543, 1033, 542, 1035, 1036, + /* 420 */ 541, 1038, 538, 1157, 1044, 535, 1046, 1047, 532, 529, + /* 430 */ 601, 36, 34, 32, 31, 30, 1160, 433, 432, 548, + /* 440 */ 1165, 1158, 431, 474, 469, 109, 428, 1454, 519, 427, + /* 450 */ 426, 425, 1185, 1469, 148, 39, 9, 1729, 1163, 1164, + /* 460 */ 348, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, + /* 470 */ 1224, 1225, 1226, 1227, 1228, 1229, 1478, 1283, 601, 1292, + /* 480 */ 1235, 1725, 433, 432, 1160, 1183, 1183, 431, 148, 1158, + /* 490 */ 109, 428, 1523, 447, 427, 426, 425, 1341, 312, 1637, + /* 500 */ 1196, 1159, 1617, 1521, 556, 455, 1163, 1164, 309, 1209, + /* 510 */ 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, + /* 520 */ 1226, 1227, 1228, 1229, 154, 1631, 1157, 36, 34, 32, + /* 530 */ 31, 30, 1160, 1777, 390, 1639, 1523, 386, 388, 1371, + /* 540 */ 1627, 1634, 1633, 1165, 1181, 373, 145, 1522, 59, 1370, + /* 550 */ 1774, 58, 522, 456, 1163, 1164, 1646, 1209, 1210, 1212, + /* 560 */ 1213, 1214, 1215, 1216, 498, 520, 1224, 1225, 1226, 1227, + /* 570 */ 1228, 1229, 35, 33, 270, 519, 1181, 214, 1340, 61, + /* 580 */ 301, 601, 1159, 366, 1662, 519, 378, 358, 1617, 158, + /* 590 */ 157, 503, 1158, 1777, 128, 1369, 174, 516, 1617, 1729, + /* 600 */ 1474, 502, 1481, 1478, 379, 1617, 145, 1157, 139, 519, + /* 610 */ 1774, 483, 1368, 1478, 415, 411, 407, 403, 173, 283, + /* 620 */ 485, 359, 1675, 1724, 1165, 259, 1647, 505, 1649, 1650, + /* 630 */ 501, 519, 522, 1120, 940, 1160, 939, 1478, 479, 445, + /* 640 */ 2, 1122, 62, 397, 1617, 171, 1367, 1211, 1211, 11, + /* 650 */ 10, 1777, 443, 1366, 430, 429, 7, 1163, 1164, 1478, + /* 660 */ 127, 1617, 601, 941, 147, 1365, 112, 284, 1774, 282, + /* 670 */ 281, 1184, 421, 1158, 377, 1364, 423, 372, 371, 370, + /* 680 */ 369, 368, 365, 364, 363, 362, 361, 357, 356, 355, + /* 690 */ 354, 353, 352, 351, 350, 1617, 1183, 519, 519, 422, + /* 700 */ 519, 1121, 1617, 170, 110, 165, 1455, 167, 479, 1475, + /* 710 */ 1597, 1186, 517, 1249, 1617, 1363, 1160, 1362, 1359, 143, + /* 720 */ 1722, 1723, 487, 1727, 1617, 1478, 1478, 163, 1478, 1261, + /* 730 */ 1646, 1358, 1357, 1356, 1254, 490, 112, 1355, 1163, 1164, + /* 740 */ 1182, 1209, 1210, 1212, 1213, 1214, 1215, 1216, 498, 520, + /* 750 */ 1224, 1225, 1226, 1227, 1228, 1229, 519, 246, 1662, 519, + /* 760 */ 1508, 1354, 1353, 1352, 1617, 503, 1617, 1617, 236, 1556, + /* 770 */ 25, 315, 571, 569, 110, 502, 156, 927, 928, 1617, + /* 780 */ 1617, 1617, 1617, 1351, 1478, 483, 1617, 1478, 506, 144, + /* 790 */ 1722, 1723, 1606, 1727, 1569, 559, 1675, 1450, 554, 79, + /* 800 */ 1647, 505, 1649, 1650, 501, 1646, 522, 423, 570, 1715, + /* 810 */ 1617, 1617, 1617, 273, 1711, 1560, 1561, 553, 552, 1397, + /* 820 */ 551, 550, 549, 1734, 1280, 1777, 1280, 558, 179, 1392, + /* 830 */ 422, 177, 1617, 1662, 1144, 1145, 199, 324, 147, 1465, + /* 840 */ 503, 434, 1774, 181, 129, 1390, 180, 1461, 337, 252, + /* 850 */ 502, 436, 183, 185, 1617, 182, 184, 118, 1211, 45, + /* 860 */ 483, 250, 52, 46, 272, 51, 1646, 439, 202, 11, + /* 870 */ 10, 1675, 1343, 1344, 79, 1647, 505, 1649, 1650, 501, + /* 880 */ 438, 522, 159, 37, 1715, 37, 191, 37, 273, 1711, + /* 890 */ 225, 497, 75, 116, 1662, 446, 547, 457, 454, 1306, + /* 900 */ 1777, 482, 71, 1361, 117, 1168, 54, 488, 204, 187, + /* 910 */ 1436, 502, 118, 145, 1167, 1617, 1453, 1774, 218, 1646, + /* 920 */ 45, 441, 491, 1255, 209, 1217, 435, 1115, 466, 1663, + /* 930 */ 227, 186, 1675, 511, 448, 80, 1647, 505, 1649, 1650, + /* 940 */ 501, 527, 522, 78, 233, 1715, 117, 1662, 1381, 294, + /* 950 */ 1711, 141, 1022, 416, 503, 50, 1518, 480, 49, 1745, + /* 960 */ 245, 217, 220, 215, 502, 118, 3, 1181, 1617, 461, + /* 970 */ 1742, 1171, 1646, 119, 57, 56, 346, 222, 117, 153, + /* 980 */ 1170, 1050, 319, 323, 340, 1675, 1054, 964, 263, 1647, + /* 990 */ 505, 1649, 1650, 501, 279, 522, 269, 991, 280, 330, + /* 1000 */ 1662, 326, 322, 150, 965, 1061, 241, 482, 554, 1128, + /* 1010 */ 360, 155, 1558, 1059, 367, 374, 375, 502, 120, 376, + /* 1020 */ 380, 1617, 1187, 381, 389, 1190, 475, 553, 552, 392, + /* 1030 */ 551, 550, 549, 393, 148, 162, 164, 1189, 1675, 1349, + /* 1040 */ 394, 80, 1647, 505, 1649, 1650, 501, 1646, 522, 166, + /* 1050 */ 395, 1715, 1188, 396, 169, 294, 1711, 141, 60, 399, + /* 1060 */ 94, 172, 420, 93, 92, 91, 90, 89, 88, 87, + /* 1070 */ 86, 85, 1468, 418, 288, 1662, 1743, 1165, 176, 1464, + /* 1080 */ 1646, 84, 503, 178, 122, 1601, 123, 242, 189, 1466, + /* 1090 */ 1462, 124, 502, 125, 449, 1186, 1617, 450, 192, 453, + /* 1100 */ 1746, 243, 194, 467, 458, 197, 1756, 509, 1662, 6, + /* 1110 */ 459, 1755, 476, 1675, 1736, 503, 80, 1647, 505, 1649, + /* 1120 */ 1650, 501, 464, 522, 293, 502, 1715, 200, 470, 1617, + /* 1130 */ 294, 1711, 1790, 463, 203, 5, 208, 1280, 135, 210, + /* 1140 */ 111, 1749, 1646, 1185, 40, 492, 1675, 295, 1730, 80, + /* 1150 */ 1647, 505, 1649, 1650, 501, 216, 522, 1567, 489, 1715, + /* 1160 */ 18, 1566, 211, 294, 1711, 1790, 512, 229, 507, 244, + /* 1170 */ 1662, 508, 303, 513, 1772, 514, 70, 503, 231, 1696, + /* 1180 */ 68, 525, 1793, 1479, 247, 1451, 238, 502, 600, 1611, + /* 1190 */ 290, 1617, 47, 1773, 249, 134, 1646, 253, 251, 1610, + /* 1200 */ 486, 219, 260, 318, 221, 1646, 254, 1607, 1675, 493, + /* 1210 */ 320, 80, 1647, 505, 1649, 1650, 501, 321, 522, 1153, + /* 1220 */ 1154, 1715, 151, 325, 1662, 294, 1711, 1790, 1605, 327, + /* 1230 */ 328, 503, 1604, 1662, 329, 331, 1733, 1603, 333, 1602, + /* 1240 */ 503, 502, 335, 1587, 152, 1617, 338, 339, 1131, 1130, + /* 1250 */ 502, 1581, 1580, 344, 1617, 1579, 345, 1578, 1098, 1646, + /* 1260 */ 483, 1551, 1675, 1550, 1549, 132, 1647, 505, 1649, 1650, + /* 1270 */ 501, 1675, 522, 1548, 259, 1647, 505, 1649, 1650, 501, + /* 1280 */ 1547, 522, 1546, 1545, 1544, 1543, 1542, 1662, 1541, 1540, + /* 1290 */ 1539, 1538, 1537, 1536, 503, 1535, 1534, 1533, 115, 1646, + /* 1300 */ 1777, 1532, 1531, 1530, 502, 1529, 1528, 1527, 1617, 1100, + /* 1310 */ 1792, 1526, 1525, 145, 1524, 1409, 1377, 1774, 1646, 160, + /* 1320 */ 1376, 385, 138, 401, 930, 1675, 1595, 1662, 81, 1647, + /* 1330 */ 505, 1649, 1650, 501, 503, 522, 106, 929, 1715, 405, + /* 1340 */ 1402, 107, 1714, 1711, 502, 1589, 1662, 1573, 1617, 1564, + /* 1350 */ 1457, 161, 387, 500, 168, 1408, 1406, 958, 402, 1404, + /* 1360 */ 1400, 406, 410, 502, 414, 1675, 400, 1617, 81, 1647, + /* 1370 */ 505, 1649, 1650, 501, 1646, 522, 404, 408, 1715, 409, + /* 1380 */ 412, 413, 494, 1711, 1675, 1389, 1388, 267, 1647, 505, + /* 1390 */ 1649, 1650, 501, 499, 522, 496, 1687, 1375, 1459, 1065, + /* 1400 */ 1064, 1458, 1662, 604, 990, 989, 568, 83, 988, 503, + /* 1410 */ 987, 570, 1398, 984, 983, 175, 285, 240, 982, 502, + /* 1420 */ 437, 286, 1391, 1617, 1393, 287, 440, 1374, 442, 105, + /* 1430 */ 1373, 444, 1594, 1646, 82, 593, 589, 585, 581, 239, + /* 1440 */ 1675, 1588, 1138, 81, 1647, 505, 1649, 1650, 501, 1572, + /* 1450 */ 522, 451, 1571, 1715, 1563, 64, 196, 4, 1712, 37, + /* 1460 */ 15, 1662, 201, 77, 43, 207, 234, 48, 503, 1305, + /* 1470 */ 133, 205, 22, 41, 1298, 206, 452, 193, 502, 1637, + /* 1480 */ 65, 198, 1617, 23, 16, 298, 1277, 1646, 1276, 213, + /* 1490 */ 136, 308, 307, 126, 42, 1329, 1334, 17, 1323, 1675, + /* 1500 */ 515, 1173, 268, 1647, 505, 1649, 1650, 501, 1328, 522, + /* 1510 */ 296, 1333, 13, 1332, 297, 1662, 10, 1646, 19, 1219, + /* 1520 */ 1240, 1218, 500, 29, 137, 460, 1166, 12, 195, 149, + /* 1530 */ 20, 1204, 502, 504, 1562, 21, 1617, 230, 224, 1303, + /* 1540 */ 226, 71, 228, 1165, 1175, 1662, 1136, 510, 190, 66, + /* 1550 */ 1646, 232, 503, 1675, 67, 1636, 267, 1647, 505, 1649, + /* 1560 */ 1650, 501, 502, 522, 235, 1688, 1617, 1678, 521, 300, + /* 1570 */ 1221, 44, 524, 1051, 526, 310, 528, 530, 1662, 1048, + /* 1580 */ 1646, 523, 531, 1675, 1045, 503, 268, 1647, 505, 1649, + /* 1590 */ 1650, 501, 1169, 522, 533, 502, 1039, 534, 536, 1617, + /* 1600 */ 1037, 537, 302, 539, 540, 1028, 1060, 1058, 1662, 1043, + /* 1610 */ 1646, 1042, 479, 72, 1041, 503, 1675, 1040, 546, 268, + /* 1620 */ 1647, 505, 1649, 1650, 501, 502, 522, 1057, 73, 1617, + /* 1630 */ 74, 1056, 956, 555, 997, 1174, 557, 237, 1662, 978, + /* 1640 */ 112, 977, 976, 973, 975, 503, 1675, 974, 972, 255, + /* 1650 */ 1647, 505, 1649, 1650, 501, 502, 522, 1177, 971, 1617, + /* 1660 */ 483, 1646, 994, 992, 968, 967, 966, 963, 520, 1224, + /* 1670 */ 1225, 962, 961, 1405, 578, 579, 1675, 1646, 110, 262, + /* 1680 */ 1647, 505, 1649, 1650, 501, 580, 522, 1403, 582, 1662, + /* 1690 */ 583, 584, 1401, 212, 1722, 478, 503, 477, 586, 588, + /* 1700 */ 1777, 1399, 587, 591, 592, 1662, 502, 590, 1387, 595, + /* 1710 */ 1617, 594, 503, 145, 1386, 1372, 598, 1774, 599, 1347, + /* 1720 */ 1161, 248, 502, 602, 1646, 603, 1617, 1675, 1347, 1347, + /* 1730 */ 264, 1647, 505, 1649, 1650, 501, 1347, 522, 1347, 1347, + /* 1740 */ 1347, 1347, 1646, 1675, 1347, 1347, 256, 1647, 505, 1649, + /* 1750 */ 1650, 501, 1662, 522, 1347, 1347, 1347, 1646, 1347, 503, + /* 1760 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 502, + /* 1770 */ 1662, 1347, 1347, 1617, 1347, 1347, 1347, 503, 1347, 1347, + /* 1780 */ 1347, 1347, 1347, 1347, 1347, 1662, 1347, 502, 1347, 1347, + /* 1790 */ 1675, 1617, 503, 265, 1647, 505, 1649, 1650, 501, 1347, + /* 1800 */ 522, 1347, 502, 1347, 1646, 1347, 1617, 1347, 1675, 1347, + /* 1810 */ 1347, 257, 1647, 505, 1649, 1650, 501, 1347, 522, 1347, + /* 1820 */ 1646, 1347, 1347, 1675, 1347, 1347, 266, 1647, 505, 1649, + /* 1830 */ 1650, 501, 1662, 522, 1347, 1347, 1347, 1347, 1347, 503, + /* 1840 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1662, 502, + /* 1850 */ 1347, 1347, 1347, 1617, 1347, 503, 1347, 1347, 1347, 1347, + /* 1860 */ 1347, 1347, 1347, 1347, 1347, 502, 1347, 1646, 1347, 1617, + /* 1870 */ 1675, 1347, 1347, 258, 1647, 505, 1649, 1650, 501, 1347, + /* 1880 */ 522, 1347, 1347, 1347, 1347, 1347, 1675, 1347, 1347, 1658, + /* 1890 */ 1647, 505, 1649, 1650, 501, 1662, 522, 1646, 1347, 1347, + /* 1900 */ 1347, 1347, 503, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 1910 */ 1347, 1347, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, + /* 1920 */ 1347, 1347, 1347, 1347, 1347, 1662, 1347, 1347, 1347, 1347, + /* 1930 */ 1347, 1347, 503, 1675, 1347, 1347, 1657, 1647, 505, 1649, + /* 1940 */ 1650, 501, 502, 522, 1347, 1347, 1617, 1646, 1347, 1347, + /* 1950 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 1960 */ 1347, 1347, 1347, 1675, 1347, 1347, 1656, 1647, 505, 1649, + /* 1970 */ 1650, 501, 1347, 522, 1347, 1662, 1347, 1646, 1347, 1347, + /* 1980 */ 1347, 1347, 503, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 1990 */ 1347, 1347, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, + /* 2000 */ 1347, 1347, 1347, 1347, 1347, 1662, 1347, 1347, 1347, 1347, + /* 2010 */ 1347, 1347, 503, 1675, 1347, 1347, 277, 1647, 505, 1649, + /* 2020 */ 1650, 501, 502, 522, 1347, 1347, 1617, 1646, 1347, 1347, + /* 2030 */ 1347, 1347, 1347, 1347, 1347, 1347, 1646, 1347, 1347, 1347, + /* 2040 */ 1347, 1347, 1347, 1675, 1347, 1347, 276, 1647, 505, 1649, + /* 2050 */ 1650, 501, 1347, 522, 1347, 1662, 1347, 1347, 1347, 1347, + /* 2060 */ 1347, 1347, 503, 1347, 1662, 1347, 1347, 1347, 1347, 1347, + /* 2070 */ 1347, 503, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, + /* 2080 */ 1347, 502, 1347, 1347, 1347, 1617, 1347, 1347, 1347, 1347, + /* 2090 */ 1646, 1347, 1347, 1675, 1347, 1347, 278, 1647, 505, 1649, + /* 2100 */ 1650, 501, 1675, 522, 1347, 275, 1647, 505, 1649, 1650, + /* 2110 */ 501, 1347, 522, 1347, 1347, 1347, 1347, 1347, 1662, 1347, + /* 2120 */ 1347, 1347, 1347, 1347, 1347, 503, 1347, 1347, 1347, 1347, + /* 2130 */ 1347, 1347, 1347, 1347, 1347, 502, 1347, 1347, 1347, 1617, + /* 2140 */ 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, 1347, + /* 2150 */ 1347, 1347, 1347, 1347, 1347, 1347, 1675, 1347, 1347, 261, + /* 2160 */ 1647, 505, 1649, 1650, 501, 1347, 522, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 244, 251, 246, 247, 244, 248, 246, 247, 290, 321, @@ -459,7 +459,7 @@ static const YYCODETYPE yy_lookahead[] = { /* 240 */ 192, 193, 194, 195, 0, 217, 218, 219, 220, 221, /* 250 */ 4, 80, 156, 12, 13, 14, 15, 16, 208, 196, /* 260 */ 197, 198, 199, 200, 201, 202, 203, 204, 205, 165, - /* 270 */ 166, 248, 57, 169, 178, 179, 238, 181, 182, 183, + /* 270 */ 166, 248, 4, 169, 178, 179, 238, 181, 182, 183, /* 280 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, /* 290 */ 194, 195, 12, 13, 12, 13, 14, 15, 16, 276, /* 300 */ 20, 0, 22, 208, 60, 61, 62, 63, 208, 65, @@ -470,141 +470,141 @@ static const YYCODETYPE yy_lookahead[] = { /* 350 */ 22, 328, 329, 330, 331, 277, 333, 2, 57, 269, /* 360 */ 80, 12, 13, 14, 15, 16, 276, 12, 13, 14, /* 370 */ 15, 16, 312, 143, 336, 47, 286, 0, 261, 208, - /* 380 */ 290, 3, 102, 293, 42, 43, 269, 349, 12, 13, - /* 390 */ 151, 353, 64, 113, 277, 20, 20, 307, 22, 20, + /* 380 */ 290, 57, 102, 293, 42, 43, 269, 349, 12, 13, + /* 390 */ 151, 353, 64, 113, 277, 20, 20, 307, 22, 3, /* 400 */ 310, 311, 312, 313, 314, 315, 245, 317, 80, 248, /* 410 */ 171, 172, 93, 94, 95, 96, 97, 98, 99, 100, /* 420 */ 101, 102, 103, 47, 105, 106, 107, 108, 109, 110, - /* 430 */ 102, 12, 13, 14, 15, 16, 156, 60, 61, 269, - /* 440 */ 64, 113, 65, 213, 214, 68, 69, 277, 248, 72, - /* 450 */ 73, 74, 20, 55, 208, 80, 80, 308, 178, 179, + /* 430 */ 102, 12, 13, 14, 15, 16, 156, 60, 61, 91, + /* 440 */ 64, 113, 65, 213, 214, 68, 69, 0, 248, 72, + /* 450 */ 73, 74, 20, 241, 208, 80, 80, 308, 178, 179, /* 460 */ 260, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 470 */ 190, 191, 192, 193, 194, 195, 276, 79, 102, 14, - /* 480 */ 82, 332, 60, 61, 156, 20, 241, 65, 208, 113, - /* 490 */ 68, 69, 269, 296, 72, 73, 74, 148, 275, 91, - /* 500 */ 81, 22, 241, 280, 75, 248, 178, 179, 273, 181, + /* 470 */ 190, 191, 192, 193, 194, 195, 276, 209, 102, 14, + /* 480 */ 14, 332, 60, 61, 156, 20, 20, 65, 208, 113, + /* 490 */ 68, 69, 269, 296, 72, 73, 74, 148, 275, 44, + /* 500 */ 81, 22, 290, 280, 57, 248, 178, 179, 273, 181, /* 510 */ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - /* 520 */ 192, 193, 194, 195, 145, 290, 47, 12, 13, 14, - /* 530 */ 15, 16, 156, 336, 245, 290, 269, 248, 286, 241, - /* 540 */ 305, 306, 307, 64, 292, 241, 349, 280, 119, 120, - /* 550 */ 353, 290, 317, 296, 178, 179, 241, 181, 182, 183, + /* 520 */ 192, 193, 194, 195, 55, 290, 47, 12, 13, 14, + /* 530 */ 15, 16, 156, 336, 245, 80, 269, 248, 14, 241, + /* 540 */ 305, 306, 307, 64, 20, 75, 349, 280, 79, 241, + /* 550 */ 353, 82, 317, 296, 178, 179, 241, 181, 182, 183, /* 560 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, /* 570 */ 194, 195, 12, 13, 18, 248, 20, 145, 229, 253, - /* 580 */ 20, 102, 22, 27, 269, 248, 30, 260, 290, 255, - /* 590 */ 256, 276, 113, 336, 290, 241, 33, 260, 241, 241, - /* 600 */ 274, 286, 224, 276, 48, 290, 349, 47, 45, 248, + /* 580 */ 20, 102, 22, 27, 269, 248, 30, 260, 290, 119, + /* 590 */ 120, 276, 113, 336, 269, 241, 33, 260, 290, 308, + /* 600 */ 274, 286, 277, 276, 48, 290, 349, 47, 45, 248, /* 610 */ 353, 296, 241, 276, 51, 52, 53, 54, 55, 35, - /* 620 */ 20, 260, 307, 4, 64, 310, 311, 312, 313, 314, - /* 630 */ 315, 248, 317, 79, 20, 156, 22, 276, 248, 0, - /* 640 */ 80, 87, 79, 260, 290, 82, 241, 290, 290, 255, - /* 650 */ 256, 336, 20, 1, 2, 285, 286, 178, 179, 276, + /* 620 */ 224, 260, 307, 332, 64, 310, 311, 312, 313, 314, + /* 630 */ 315, 248, 317, 79, 20, 156, 22, 276, 248, 21, + /* 640 */ 80, 87, 79, 260, 290, 82, 241, 182, 182, 1, + /* 650 */ 2, 336, 34, 241, 255, 256, 37, 178, 179, 276, /* 660 */ 145, 290, 102, 49, 349, 241, 276, 83, 353, 85, - /* 670 */ 86, 37, 88, 113, 118, 241, 92, 121, 122, 123, + /* 670 */ 86, 20, 88, 113, 118, 241, 92, 121, 122, 123, /* 680 */ 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - /* 690 */ 134, 135, 136, 137, 138, 290, 57, 248, 248, 115, - /* 700 */ 248, 147, 41, 140, 314, 142, 0, 144, 248, 260, - /* 710 */ 260, 241, 260, 139, 290, 241, 156, 241, 241, 329, - /* 720 */ 330, 331, 21, 333, 290, 276, 276, 164, 276, 241, - /* 730 */ 241, 241, 241, 81, 160, 34, 276, 241, 178, 179, - /* 740 */ 92, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 750 */ 190, 191, 192, 193, 194, 195, 248, 41, 269, 248, - /* 760 */ 290, 241, 276, 115, 290, 276, 290, 290, 260, 283, - /* 770 */ 196, 260, 42, 43, 314, 286, 206, 207, 290, 290, - /* 780 */ 290, 290, 14, 308, 276, 296, 290, 276, 20, 329, - /* 790 */ 330, 331, 262, 333, 0, 265, 307, 81, 92, 310, - /* 800 */ 311, 312, 313, 314, 315, 241, 317, 332, 84, 320, - /* 810 */ 290, 87, 84, 324, 325, 87, 84, 111, 112, 87, - /* 820 */ 114, 115, 116, 0, 257, 336, 259, 84, 209, 14, - /* 830 */ 87, 0, 41, 269, 0, 20, 167, 168, 349, 45, - /* 840 */ 276, 207, 353, 41, 18, 22, 41, 145, 146, 23, - /* 850 */ 286, 1, 2, 22, 290, 44, 22, 193, 194, 41, - /* 860 */ 296, 35, 36, 41, 64, 39, 241, 270, 41, 270, - /* 870 */ 41, 307, 81, 270, 310, 311, 312, 313, 314, 315, - /* 880 */ 4, 317, 56, 81, 320, 41, 81, 226, 324, 325, - /* 890 */ 41, 80, 80, 41, 269, 19, 270, 270, 242, 81, - /* 900 */ 336, 276, 90, 299, 41, 47, 80, 47, 81, 33, - /* 910 */ 81, 286, 41, 349, 356, 290, 0, 353, 258, 241, - /* 920 */ 41, 45, 41, 347, 303, 81, 50, 41, 341, 247, - /* 930 */ 81, 55, 307, 81, 249, 310, 311, 312, 313, 314, - /* 940 */ 315, 41, 317, 117, 81, 320, 41, 269, 269, 324, - /* 950 */ 325, 326, 81, 279, 276, 79, 309, 334, 82, 350, - /* 960 */ 81, 47, 81, 338, 286, 350, 350, 81, 290, 344, - /* 970 */ 345, 113, 241, 113, 148, 149, 150, 337, 64, 153, - /* 980 */ 20, 81, 248, 45, 158, 307, 81, 304, 310, 311, - /* 990 */ 312, 313, 314, 315, 47, 317, 170, 255, 248, 173, - /* 1000 */ 269, 175, 176, 177, 297, 154, 248, 276, 92, 40, - /* 1010 */ 284, 139, 248, 20, 243, 282, 243, 286, 282, 20, - /* 1020 */ 301, 290, 253, 286, 20, 253, 348, 111, 112, 294, - /* 1030 */ 114, 115, 116, 253, 208, 276, 20, 253, 307, 0, - /* 1040 */ 287, 310, 311, 312, 313, 314, 315, 241, 317, 253, - /* 1050 */ 228, 320, 248, 243, 253, 324, 325, 326, 269, 269, - /* 1060 */ 21, 269, 269, 24, 25, 26, 27, 28, 29, 30, - /* 1070 */ 31, 32, 269, 248, 64, 269, 345, 269, 269, 243, - /* 1080 */ 241, 251, 276, 269, 301, 269, 269, 290, 269, 163, - /* 1090 */ 20, 251, 286, 286, 251, 300, 290, 276, 251, 216, - /* 1100 */ 346, 346, 215, 223, 343, 342, 222, 294, 269, 211, - /* 1110 */ 290, 287, 210, 307, 340, 276, 310, 311, 312, 313, - /* 1120 */ 314, 315, 290, 317, 291, 286, 320, 339, 291, 290, - /* 1130 */ 324, 325, 326, 276, 207, 290, 20, 309, 40, 227, - /* 1140 */ 225, 335, 241, 308, 357, 80, 307, 230, 142, 310, - /* 1150 */ 311, 312, 313, 314, 315, 291, 317, 290, 290, 320, - /* 1160 */ 291, 288, 287, 324, 325, 326, 276, 290, 327, 251, - /* 1170 */ 269, 265, 251, 276, 335, 80, 272, 276, 323, 259, - /* 1180 */ 248, 251, 243, 298, 302, 263, 295, 286, 263, 252, - /* 1190 */ 352, 290, 263, 351, 0, 239, 241, 0, 352, 351, - /* 1200 */ 72, 0, 47, 174, 351, 241, 352, 47, 307, 47, - /* 1210 */ 47, 310, 311, 312, 313, 314, 315, 174, 317, 0, + /* 690 */ 134, 135, 136, 137, 138, 290, 20, 248, 248, 115, + /* 700 */ 248, 147, 290, 140, 314, 142, 0, 144, 248, 260, + /* 710 */ 260, 20, 260, 139, 290, 241, 156, 241, 241, 329, + /* 720 */ 330, 331, 41, 333, 290, 276, 276, 164, 276, 81, + /* 730 */ 241, 241, 241, 241, 160, 41, 276, 241, 178, 179, + /* 740 */ 20, 181, 182, 183, 184, 185, 186, 187, 188, 189, + /* 750 */ 190, 191, 192, 193, 194, 195, 248, 262, 269, 248, + /* 760 */ 265, 241, 241, 241, 290, 276, 290, 290, 260, 276, + /* 770 */ 196, 260, 255, 256, 314, 286, 283, 42, 43, 290, + /* 780 */ 290, 290, 290, 241, 276, 296, 290, 276, 286, 329, + /* 790 */ 330, 331, 0, 333, 292, 257, 307, 259, 92, 310, + /* 800 */ 311, 312, 313, 314, 315, 241, 317, 92, 41, 320, + /* 810 */ 290, 290, 290, 324, 325, 285, 286, 111, 112, 0, + /* 820 */ 114, 115, 116, 206, 207, 336, 207, 64, 84, 0, + /* 830 */ 115, 87, 290, 269, 167, 168, 145, 45, 349, 270, + /* 840 */ 276, 22, 353, 84, 18, 0, 87, 270, 81, 23, + /* 850 */ 286, 22, 84, 84, 290, 87, 87, 41, 182, 41, + /* 860 */ 296, 35, 36, 145, 146, 39, 241, 22, 41, 1, + /* 870 */ 2, 307, 193, 194, 310, 311, 312, 313, 314, 315, + /* 880 */ 4, 317, 56, 41, 320, 41, 270, 41, 324, 325, + /* 890 */ 41, 270, 80, 41, 269, 19, 270, 81, 299, 81, + /* 900 */ 336, 276, 90, 242, 41, 47, 80, 226, 81, 33, + /* 910 */ 258, 286, 41, 349, 47, 290, 0, 353, 356, 241, + /* 920 */ 41, 45, 228, 81, 341, 81, 50, 81, 347, 269, + /* 930 */ 81, 55, 307, 81, 303, 310, 311, 312, 313, 314, + /* 940 */ 315, 41, 317, 117, 81, 320, 41, 269, 247, 324, + /* 950 */ 325, 326, 81, 249, 276, 79, 279, 334, 82, 309, + /* 960 */ 81, 350, 350, 338, 286, 41, 337, 20, 290, 344, + /* 970 */ 345, 113, 241, 41, 148, 149, 150, 350, 41, 153, + /* 980 */ 113, 81, 248, 45, 158, 307, 81, 47, 310, 311, + /* 990 */ 312, 313, 314, 315, 304, 317, 170, 47, 255, 173, + /* 1000 */ 269, 175, 176, 177, 64, 81, 297, 276, 92, 154, + /* 1010 */ 248, 40, 248, 81, 284, 282, 139, 286, 81, 282, + /* 1020 */ 248, 290, 20, 243, 243, 20, 348, 111, 112, 301, + /* 1030 */ 114, 115, 116, 286, 208, 253, 253, 20, 307, 0, + /* 1040 */ 294, 310, 311, 312, 313, 314, 315, 241, 317, 253, + /* 1050 */ 276, 320, 20, 287, 253, 324, 325, 326, 253, 248, + /* 1060 */ 21, 253, 269, 24, 25, 26, 27, 28, 29, 30, + /* 1070 */ 31, 32, 269, 243, 243, 269, 345, 64, 269, 269, + /* 1080 */ 241, 248, 276, 269, 269, 290, 269, 301, 251, 269, + /* 1090 */ 269, 269, 286, 269, 163, 20, 290, 300, 251, 286, + /* 1100 */ 309, 294, 251, 216, 276, 251, 346, 215, 269, 223, + /* 1110 */ 287, 346, 222, 307, 343, 276, 310, 311, 312, 313, + /* 1120 */ 314, 315, 290, 317, 290, 286, 320, 291, 290, 290, + /* 1130 */ 324, 325, 326, 211, 291, 210, 342, 207, 340, 339, + /* 1140 */ 276, 335, 241, 20, 40, 227, 307, 230, 308, 310, + /* 1150 */ 311, 312, 313, 314, 315, 351, 317, 291, 225, 320, + /* 1160 */ 80, 291, 327, 324, 325, 326, 142, 276, 290, 265, + /* 1170 */ 269, 290, 290, 288, 335, 287, 80, 276, 251, 323, + /* 1180 */ 251, 272, 357, 276, 248, 259, 251, 286, 243, 0, + /* 1190 */ 295, 290, 298, 352, 252, 302, 241, 263, 239, 0, + /* 1200 */ 352, 351, 263, 72, 351, 241, 263, 0, 307, 352, + /* 1210 */ 47, 310, 311, 312, 313, 314, 315, 174, 317, 47, /* 1220 */ 47, 320, 47, 174, 269, 324, 325, 326, 0, 47, - /* 1230 */ 0, 276, 47, 269, 0, 47, 335, 0, 80, 160, - /* 1240 */ 276, 286, 159, 113, 156, 290, 152, 151, 0, 0, - /* 1250 */ 286, 0, 0, 44, 290, 0, 0, 0, 0, 241, + /* 1230 */ 47, 276, 0, 269, 174, 47, 335, 0, 47, 0, + /* 1240 */ 276, 286, 47, 0, 80, 290, 160, 159, 113, 156, + /* 1250 */ 286, 0, 0, 152, 290, 0, 151, 0, 44, 241, /* 1260 */ 296, 0, 307, 0, 0, 310, 311, 312, 313, 314, /* 1270 */ 315, 307, 317, 0, 310, 311, 312, 313, 314, 315, /* 1280 */ 0, 317, 0, 0, 0, 0, 0, 269, 0, 0, - /* 1290 */ 0, 0, 0, 40, 276, 0, 0, 0, 0, 241, - /* 1300 */ 336, 0, 0, 22, 286, 0, 0, 0, 290, 40, - /* 1310 */ 355, 0, 0, 349, 37, 41, 44, 353, 241, 44, - /* 1320 */ 0, 0, 14, 0, 14, 307, 0, 269, 310, 311, - /* 1330 */ 312, 313, 314, 315, 276, 317, 0, 38, 320, 45, - /* 1340 */ 37, 37, 324, 325, 286, 0, 269, 0, 290, 0, - /* 1350 */ 0, 37, 59, 276, 37, 0, 0, 0, 0, 37, - /* 1360 */ 47, 45, 47, 286, 37, 307, 47, 290, 310, 311, - /* 1370 */ 312, 313, 314, 315, 241, 317, 45, 47, 320, 45, - /* 1380 */ 0, 0, 324, 325, 307, 47, 0, 310, 311, 312, - /* 1390 */ 313, 314, 315, 316, 317, 318, 319, 22, 47, 47, - /* 1400 */ 47, 47, 269, 19, 41, 0, 41, 89, 47, 276, - /* 1410 */ 22, 87, 0, 47, 47, 22, 48, 33, 0, 286, - /* 1420 */ 22, 47, 0, 290, 22, 0, 22, 20, 0, 45, - /* 1430 */ 47, 0, 145, 241, 0, 51, 52, 53, 54, 55, - /* 1440 */ 307, 22, 142, 310, 311, 312, 313, 314, 315, 0, - /* 1450 */ 317, 0, 80, 320, 37, 41, 41, 212, 325, 81, - /* 1460 */ 81, 269, 41, 79, 41, 80, 82, 80, 276, 140, - /* 1470 */ 80, 44, 81, 145, 80, 2, 41, 212, 286, 41, - /* 1480 */ 81, 81, 290, 44, 206, 293, 161, 241, 44, 81, - /* 1490 */ 81, 12, 13, 41, 47, 47, 41, 47, 44, 307, + /* 1290 */ 0, 0, 0, 0, 276, 0, 0, 0, 40, 241, + /* 1300 */ 336, 0, 0, 0, 286, 0, 0, 0, 290, 22, + /* 1310 */ 355, 0, 0, 349, 0, 0, 0, 353, 241, 40, + /* 1320 */ 0, 44, 41, 45, 14, 307, 0, 269, 310, 311, + /* 1330 */ 312, 313, 314, 315, 276, 317, 37, 14, 320, 45, + /* 1340 */ 0, 37, 324, 325, 286, 0, 269, 0, 290, 0, + /* 1350 */ 0, 38, 44, 276, 37, 0, 0, 59, 37, 0, + /* 1360 */ 0, 37, 37, 286, 37, 307, 47, 290, 310, 311, + /* 1370 */ 312, 313, 314, 315, 241, 317, 47, 47, 320, 45, + /* 1380 */ 47, 45, 324, 325, 307, 0, 0, 310, 311, 312, + /* 1390 */ 313, 314, 315, 316, 317, 318, 319, 0, 0, 47, + /* 1400 */ 22, 0, 269, 19, 47, 47, 41, 89, 47, 276, + /* 1410 */ 47, 41, 0, 47, 47, 87, 22, 33, 47, 286, + /* 1420 */ 48, 22, 0, 290, 0, 22, 47, 0, 22, 45, + /* 1430 */ 0, 22, 0, 241, 20, 51, 52, 53, 54, 55, + /* 1440 */ 307, 0, 47, 310, 311, 312, 313, 314, 315, 0, + /* 1450 */ 317, 22, 0, 320, 0, 80, 37, 41, 325, 41, + /* 1460 */ 212, 269, 81, 79, 41, 44, 82, 145, 276, 81, + /* 1470 */ 80, 80, 80, 206, 81, 41, 145, 142, 286, 44, + /* 1480 */ 80, 140, 290, 41, 212, 293, 81, 241, 81, 44, + /* 1490 */ 44, 12, 13, 161, 41, 47, 81, 41, 81, 307, /* 1500 */ 116, 22, 310, 311, 312, 313, 314, 315, 47, 317, - /* 1510 */ 47, 47, 80, 178, 81, 269, 44, 241, 81, 22, - /* 1520 */ 80, 80, 276, 180, 81, 141, 47, 81, 144, 0, - /* 1530 */ 37, 44, 286, 80, 44, 143, 290, 80, 80, 80, - /* 1540 */ 80, 140, 80, 64, 212, 269, 162, 81, 164, 80, - /* 1550 */ 241, 80, 276, 307, 22, 90, 310, 311, 312, 313, - /* 1560 */ 314, 315, 286, 317, 47, 319, 290, 91, 47, 293, - /* 1570 */ 81, 80, 47, 81, 80, 47, 81, 81, 269, 80, - /* 1580 */ 241, 102, 47, 307, 80, 276, 310, 311, 312, 313, - /* 1590 */ 314, 315, 113, 317, 81, 286, 47, 22, 80, 290, - /* 1600 */ 92, 47, 293, 80, 104, 47, 104, 104, 269, 104, - /* 1610 */ 241, 80, 248, 80, 22, 276, 307, 64, 59, 310, - /* 1620 */ 311, 312, 313, 314, 315, 286, 317, 113, 58, 290, - /* 1630 */ 78, 41, 47, 47, 47, 156, 47, 22, 269, 0, - /* 1640 */ 276, 47, 47, 47, 64, 276, 307, 47, 47, 310, + /* 1510 */ 47, 47, 212, 47, 47, 269, 2, 241, 41, 81, + /* 1520 */ 178, 81, 276, 80, 44, 141, 47, 80, 144, 44, + /* 1530 */ 80, 22, 286, 180, 0, 80, 290, 37, 81, 81, + /* 1540 */ 80, 90, 80, 64, 22, 269, 162, 143, 164, 80, + /* 1550 */ 241, 140, 276, 307, 80, 44, 310, 311, 312, 313, + /* 1560 */ 314, 315, 286, 317, 44, 319, 290, 80, 80, 293, + /* 1570 */ 81, 80, 91, 81, 47, 47, 80, 47, 269, 81, + /* 1580 */ 241, 102, 80, 307, 81, 276, 310, 311, 312, 313, + /* 1590 */ 314, 315, 113, 317, 47, 286, 81, 80, 47, 290, + /* 1600 */ 81, 80, 293, 47, 80, 22, 47, 113, 269, 104, + /* 1610 */ 241, 104, 248, 80, 104, 276, 307, 104, 92, 310, + /* 1620 */ 311, 312, 313, 314, 315, 286, 317, 47, 80, 290, + /* 1630 */ 80, 22, 59, 58, 64, 156, 78, 41, 269, 47, + /* 1640 */ 276, 47, 47, 22, 47, 276, 307, 47, 47, 310, /* 1650 */ 311, 312, 313, 314, 315, 286, 317, 178, 47, 290, - /* 1660 */ 296, 241, 37, 47, 47, 47, 47, 47, 189, 190, - /* 1670 */ 191, 0, 45, 37, 45, 47, 307, 241, 314, 310, - /* 1680 */ 311, 312, 313, 314, 315, 0, 317, 47, 45, 269, - /* 1690 */ 37, 0, 47, 329, 330, 331, 276, 333, 37, 45, - /* 1700 */ 336, 0, 47, 46, 0, 269, 286, 0, 22, 21, - /* 1710 */ 290, 22, 276, 349, 22, 21, 20, 353, 358, 358, - /* 1720 */ 358, 358, 286, 358, 241, 358, 290, 307, 358, 358, + /* 1660 */ 296, 241, 64, 47, 47, 47, 47, 47, 189, 190, + /* 1670 */ 191, 47, 47, 0, 47, 45, 307, 241, 314, 310, + /* 1680 */ 311, 312, 313, 314, 315, 37, 317, 0, 47, 269, + /* 1690 */ 45, 37, 0, 329, 330, 331, 276, 333, 47, 37, + /* 1700 */ 336, 0, 45, 45, 37, 269, 286, 47, 0, 46, + /* 1710 */ 290, 47, 276, 349, 0, 0, 22, 353, 21, 358, + /* 1720 */ 22, 22, 286, 21, 241, 20, 290, 307, 358, 358, /* 1730 */ 310, 311, 312, 313, 314, 315, 358, 317, 358, 358, /* 1740 */ 358, 358, 241, 307, 358, 358, 310, 311, 312, 313, /* 1750 */ 314, 315, 269, 317, 358, 358, 358, 241, 358, 276, @@ -650,9 +650,9 @@ static const YYCODETYPE yy_lookahead[] = { /* 2150 */ 358, 358, 358, 358, 358, 358, 307, 358, 358, 310, /* 2160 */ 311, 312, 313, 314, 315, 358, 317, }; -#define YY_SHIFT_COUNT (602) +#define YY_SHIFT_COUNT (604) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1707) +#define YY_SHIFT_MAX (1715) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 826, 0, 0, 48, 96, 96, 96, 96, 280, 280, /* 10 */ 96, 96, 328, 376, 560, 376, 376, 376, 376, 376, @@ -660,63 +660,63 @@ static const unsigned short int yy_shift_ofst[] = { /* 30 */ 376, 376, 376, 376, 376, 376, 376, 376, 95, 95, /* 40 */ 375, 375, 375, 1479, 1479, 1479, 100, 50, 171, 45, /* 50 */ 45, 342, 342, 246, 171, 171, 45, 45, 45, 45, - /* 60 */ 45, 45, 17, 45, 201, 323, 600, 201, 45, 45, - /* 70 */ 201, 45, 201, 201, 600, 201, 45, 215, 556, 63, + /* 60 */ 45, 45, 17, 45, 201, 323, 651, 201, 45, 45, + /* 70 */ 201, 45, 201, 201, 651, 201, 45, 324, 556, 63, /* 80 */ 14, 14, 13, 479, 422, 479, 479, 479, 479, 479, /* 90 */ 479, 479, 479, 479, 479, 479, 479, 479, 479, 479, - /* 100 */ 479, 479, 479, 479, 584, 614, 465, 465, 301, 281, - /* 110 */ 379, 379, 379, 639, 281, 632, 600, 201, 201, 600, - /* 120 */ 408, 800, 319, 319, 319, 319, 319, 319, 319, 1384, - /* 130 */ 1039, 377, 349, 28, 104, 230, 730, 102, 648, 432, - /* 140 */ 570, 634, 570, 768, 378, 378, 378, 619, 815, 960, - /* 150 */ 938, 947, 851, 960, 960, 969, 872, 872, 960, 993, - /* 160 */ 993, 999, 17, 600, 17, 1004, 17, 632, 1016, 17, - /* 170 */ 17, 960, 17, 993, 201, 201, 201, 201, 201, 201, - /* 180 */ 201, 201, 201, 201, 201, 960, 993, 1010, 999, 215, - /* 190 */ 926, 600, 215, 1004, 215, 632, 1016, 215, 1070, 883, - /* 200 */ 887, 1010, 883, 887, 1010, 1010, 880, 884, 898, 902, - /* 210 */ 927, 632, 1116, 1098, 912, 915, 917, 912, 915, 912, - /* 220 */ 915, 1065, 201, 887, 1010, 1010, 887, 1010, 1006, 632, - /* 230 */ 1016, 215, 408, 215, 632, 1095, 800, 960, 215, 993, - /* 240 */ 2167, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 244, 563, - /* 250 */ 141, 876, 706, 916, 241, 84, 355, 515, 419, 85, - /* 260 */ 282, 282, 282, 282, 282, 282, 282, 282, 239, 398, - /* 270 */ 429, 554, 652, 574, 36, 36, 36, 36, 794, 716, - /* 280 */ 724, 728, 732, 743, 823, 831, 834, 701, 669, 702, - /* 290 */ 791, 802, 805, 850, 664, 661, 822, 818, 145, 827, - /* 300 */ 811, 829, 844, 849, 852, 863, 858, 860, 871, 879, - /* 310 */ 881, 886, 900, 905, 812, 914, 1194, 1197, 1128, 1201, - /* 320 */ 1155, 1029, 1160, 1162, 1163, 1043, 1219, 1173, 1175, 1049, - /* 330 */ 1228, 1182, 1230, 1185, 1234, 1188, 1237, 1158, 1079, 1083, - /* 340 */ 1130, 1088, 1251, 1252, 1094, 1096, 1248, 1249, 1209, 1255, - /* 350 */ 1256, 1257, 1258, 1261, 1263, 1264, 1273, 1280, 1282, 1283, - /* 360 */ 1284, 1285, 1286, 1288, 1289, 1290, 1291, 1253, 1292, 1295, - /* 370 */ 1296, 1297, 1298, 1301, 1281, 1302, 1305, 1306, 1307, 1311, - /* 380 */ 1312, 1269, 1277, 1274, 1308, 1272, 1310, 1275, 1320, 1299, - /* 390 */ 1303, 1321, 1323, 1326, 1336, 1304, 1345, 1293, 1347, 1349, - /* 400 */ 1313, 1294, 1314, 1350, 1315, 1316, 1317, 1355, 1319, 1331, - /* 410 */ 1322, 1356, 1330, 1334, 1327, 1357, 1358, 1380, 1381, 1318, - /* 420 */ 1324, 1338, 1375, 1386, 1351, 1352, 1353, 1354, 1363, 1365, - /* 430 */ 1361, 1366, 1367, 1405, 1388, 1412, 1393, 1368, 1418, 1398, - /* 440 */ 1374, 1422, 1402, 1425, 1404, 1407, 1428, 1287, 1383, 1431, - /* 450 */ 1325, 1419, 1328, 1300, 1434, 1449, 1451, 1372, 1417, 1329, - /* 460 */ 1414, 1415, 1245, 1378, 1421, 1379, 1385, 1387, 1390, 1391, - /* 470 */ 1423, 1427, 1394, 1435, 1265, 1399, 1400, 1439, 1278, 1438, - /* 480 */ 1444, 1408, 1452, 1332, 1409, 1447, 1448, 1450, 1461, 1463, - /* 490 */ 1464, 1409, 1473, 1335, 1455, 1433, 1432, 1437, 1454, 1440, - /* 500 */ 1441, 1472, 1497, 1343, 1453, 1443, 1446, 1457, 1458, 1392, - /* 510 */ 1459, 1529, 1493, 1401, 1460, 1465, 1487, 1490, 1462, 1466, - /* 520 */ 1469, 1532, 1471, 1476, 1489, 1517, 1521, 1491, 1492, 1525, - /* 530 */ 1494, 1495, 1528, 1499, 1496, 1535, 1504, 1513, 1549, 1518, - /* 540 */ 1500, 1502, 1503, 1505, 1575, 1508, 1523, 1531, 1554, 1533, - /* 550 */ 1514, 1558, 1592, 1559, 1570, 1553, 1552, 1590, 1585, 1586, - /* 560 */ 1587, 1589, 1594, 1615, 1595, 1596, 1580, 1363, 1600, 1365, - /* 570 */ 1601, 1611, 1616, 1617, 1618, 1619, 1639, 1620, 1627, 1625, - /* 580 */ 1671, 1628, 1629, 1636, 1685, 1640, 1643, 1653, 1691, 1645, - /* 590 */ 1654, 1661, 1701, 1655, 1657, 1704, 1707, 1686, 1688, 1689, - /* 600 */ 1692, 1694, 1696, + /* 100 */ 479, 479, 479, 479, 584, 614, 524, 524, 301, 281, + /* 110 */ 432, 432, 432, 447, 281, 720, 651, 201, 201, 651, + /* 120 */ 348, 763, 319, 319, 319, 319, 319, 319, 319, 1384, + /* 130 */ 1039, 377, 349, 28, 104, 230, 465, 466, 735, 102, + /* 140 */ 715, 691, 617, 619, 617, 396, 396, 396, 268, 676, + /* 150 */ 947, 938, 950, 855, 947, 947, 971, 877, 877, 947, + /* 160 */ 1002, 1002, 1005, 17, 651, 17, 1017, 17, 720, 1032, + /* 170 */ 17, 17, 947, 17, 1002, 201, 201, 201, 201, 201, + /* 180 */ 201, 201, 201, 201, 201, 201, 947, 1002, 1013, 1005, + /* 190 */ 324, 931, 651, 324, 1017, 324, 720, 1032, 324, 1075, + /* 200 */ 887, 892, 1013, 887, 892, 1013, 1013, 201, 886, 890, + /* 210 */ 922, 925, 930, 720, 1123, 1104, 918, 933, 917, 918, + /* 220 */ 933, 918, 933, 1080, 892, 1013, 1013, 892, 1013, 1024, + /* 230 */ 720, 1032, 324, 348, 324, 720, 1096, 763, 947, 324, + /* 240 */ 1002, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 2167, 244, + /* 250 */ 563, 141, 876, 706, 916, 241, 84, 355, 515, 419, + /* 260 */ 85, 282, 282, 282, 282, 282, 282, 282, 282, 239, + /* 270 */ 469, 470, 554, 648, 574, 36, 36, 36, 36, 792, + /* 280 */ 767, 744, 759, 768, 769, 819, 829, 845, 618, 667, + /* 290 */ 718, 816, 818, 827, 868, 679, 681, 694, 842, 145, + /* 300 */ 844, 455, 846, 849, 852, 863, 871, 858, 867, 879, + /* 310 */ 900, 905, 924, 932, 937, 812, 940, 1189, 1199, 1131, + /* 320 */ 1207, 1163, 1043, 1172, 1173, 1175, 1049, 1228, 1182, 1183, + /* 330 */ 1060, 1232, 1188, 1237, 1191, 1239, 1195, 1243, 1164, 1086, + /* 340 */ 1088, 1135, 1093, 1251, 1252, 1101, 1105, 1255, 1257, 1214, + /* 350 */ 1261, 1263, 1264, 1273, 1280, 1282, 1283, 1284, 1285, 1286, + /* 360 */ 1288, 1289, 1290, 1291, 1292, 1293, 1295, 1296, 1258, 1297, + /* 370 */ 1301, 1302, 1303, 1305, 1306, 1287, 1307, 1311, 1312, 1314, + /* 380 */ 1315, 1316, 1279, 1299, 1281, 1310, 1277, 1323, 1308, 1320, + /* 390 */ 1313, 1304, 1326, 1345, 1347, 1349, 1317, 1350, 1298, 1355, + /* 400 */ 1356, 1319, 1278, 1321, 1359, 1329, 1294, 1324, 1340, 1330, + /* 410 */ 1334, 1325, 1360, 1333, 1336, 1327, 1385, 1386, 1397, 1398, + /* 420 */ 1318, 1328, 1352, 1378, 1401, 1357, 1358, 1361, 1363, 1365, + /* 430 */ 1370, 1366, 1367, 1371, 1412, 1394, 1424, 1399, 1372, 1422, + /* 440 */ 1403, 1379, 1427, 1406, 1430, 1409, 1414, 1432, 1322, 1395, + /* 450 */ 1441, 1332, 1429, 1331, 1335, 1449, 1452, 1454, 1375, 1419, + /* 460 */ 1341, 1416, 1418, 1248, 1381, 1423, 1388, 1390, 1391, 1392, + /* 470 */ 1393, 1434, 1421, 1435, 1400, 1442, 1272, 1405, 1407, 1445, + /* 480 */ 1267, 1453, 1446, 1415, 1456, 1300, 1417, 1448, 1461, 1463, + /* 490 */ 1464, 1466, 1467, 1417, 1514, 1342, 1477, 1438, 1443, 1440, + /* 500 */ 1480, 1447, 1450, 1485, 1509, 1353, 1455, 1457, 1458, 1460, + /* 510 */ 1462, 1404, 1469, 1534, 1500, 1411, 1474, 1451, 1511, 1520, + /* 520 */ 1487, 1489, 1488, 1522, 1491, 1481, 1492, 1527, 1528, 1496, + /* 530 */ 1498, 1530, 1502, 1503, 1547, 1517, 1515, 1551, 1521, 1519, + /* 540 */ 1556, 1524, 1505, 1507, 1510, 1513, 1583, 1526, 1533, 1548, + /* 550 */ 1559, 1550, 1494, 1580, 1609, 1573, 1575, 1570, 1558, 1596, + /* 560 */ 1592, 1594, 1595, 1597, 1600, 1621, 1601, 1611, 1598, 1365, + /* 570 */ 1616, 1370, 1617, 1618, 1619, 1620, 1624, 1625, 1673, 1627, + /* 580 */ 1630, 1648, 1687, 1641, 1645, 1654, 1692, 1651, 1657, 1662, + /* 590 */ 1701, 1660, 1658, 1667, 1708, 1664, 1663, 1714, 1715, 1694, + /* 600 */ 1697, 1698, 1699, 1702, 1705, }; -#define YY_REDUCE_COUNT (247) +#define YY_REDUCE_COUNT (248) #define YY_REDUCE_MIN (-317) #define YY_REDUCE_MAX (1849) static const short yy_reduce_ofst[] = { @@ -728,86 +728,86 @@ static const short yy_reduce_ofst[] = { /* 50 */ -112, -244, -240, -317, -202, -190, -243, 200, 327, 361, /* 60 */ 383, 449, -152, 450, -221, 60, -64, -144, 337, 452, /* 70 */ 55, 508, 78, 223, 47, 117, 511, -250, -177, -312, - /* 80 */ -312, -312, -113, 245, -34, 261, 298, 304, 354, 357, - /* 90 */ 358, 371, 405, 424, 434, 470, 474, 476, 477, 488, - /* 100 */ 490, 491, 496, 520, 70, -139, 161, 289, 326, 334, - /* 110 */ -277, 149, 475, -114, 394, 486, 252, 170, 267, 370, - /* 120 */ 530, 567, -259, -255, 597, 599, 603, 626, 627, 604, - /* 130 */ 656, 660, 558, 576, 621, 587, 682, 685, 674, 647, - /* 140 */ 623, 623, 623, 679, 609, 615, 616, 640, 679, 734, - /* 150 */ 683, 742, 707, 750, 758, 726, 733, 736, 764, 771, - /* 160 */ 773, 719, 769, 737, 772, 735, 780, 759, 753, 784, - /* 170 */ 796, 804, 801, 810, 789, 790, 792, 793, 803, 808, - /* 180 */ 809, 814, 816, 817, 819, 825, 836, 797, 783, 830, - /* 190 */ 795, 807, 840, 813, 843, 821, 824, 847, 828, 754, - /* 200 */ 833, 820, 755, 837, 832, 845, 761, 763, 774, 788, - /* 210 */ 623, 857, 835, 841, 838, 842, 787, 846, 848, 854, - /* 220 */ 853, 855, 679, 864, 867, 868, 869, 877, 873, 890, - /* 230 */ 875, 918, 906, 921, 897, 904, 920, 932, 930, 939, - /* 240 */ 885, 882, 891, 922, 925, 929, 937, 956, + /* 80 */ -312, -312, -113, 212, -34, 298, 308, 354, 371, 405, + /* 90 */ 412, 424, 434, 474, 476, 477, 490, 491, 492, 496, + /* 100 */ 520, 521, 522, 542, 70, -139, 161, 289, 326, 399, + /* 110 */ -277, 149, 291, -114, 517, 493, 502, 325, 267, 530, + /* 120 */ 495, 538, -259, -255, 569, 577, 616, 621, 626, 599, + /* 130 */ 661, 652, 562, 581, 631, 583, 660, 660, 701, 704, + /* 140 */ 677, 650, 623, 623, 623, 611, 612, 627, 629, 660, + /* 150 */ 734, 690, 743, 709, 762, 764, 730, 733, 737, 772, + /* 160 */ 780, 781, 728, 782, 747, 783, 746, 796, 774, 766, + /* 170 */ 801, 805, 811, 808, 830, 793, 803, 809, 810, 814, + /* 180 */ 815, 817, 820, 821, 822, 824, 833, 831, 795, 786, + /* 190 */ 837, 797, 813, 847, 807, 851, 828, 823, 854, 791, + /* 200 */ 760, 836, 832, 765, 843, 834, 838, 660, 771, 794, + /* 210 */ 798, 800, 623, 864, 840, 835, 841, 804, 825, 848, + /* 220 */ 850, 857, 853, 856, 866, 878, 881, 870, 882, 885, + /* 230 */ 891, 888, 927, 904, 929, 907, 909, 926, 936, 935, + /* 240 */ 945, 894, 893, 895, 934, 939, 943, 942, 959, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 10 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 20 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 30 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 40 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 50 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 60 */ 1341, 1341, 1410, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 70 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1408, 1548, 1341, - /* 80 */ 1712, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 90 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 100 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1410, 1341, - /* 110 */ 1723, 1723, 1723, 1408, 1341, 1341, 1341, 1341, 1341, 1341, - /* 120 */ 1503, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1586, - /* 130 */ 1341, 1341, 1789, 1341, 1592, 1747, 1341, 1341, 1456, 1739, - /* 140 */ 1715, 1729, 1716, 1341, 1774, 1774, 1774, 1732, 1341, 1341, - /* 150 */ 1341, 1341, 1578, 1341, 1341, 1553, 1550, 1550, 1341, 1341, - /* 160 */ 1341, 1341, 1410, 1341, 1410, 1341, 1410, 1341, 1341, 1410, - /* 170 */ 1410, 1341, 1410, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 180 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1408, - /* 190 */ 1588, 1341, 1408, 1341, 1408, 1341, 1341, 1408, 1341, 1754, - /* 200 */ 1752, 1341, 1754, 1752, 1341, 1341, 1766, 1762, 1745, 1743, - /* 210 */ 1729, 1341, 1341, 1341, 1780, 1776, 1792, 1780, 1776, 1780, - /* 220 */ 1776, 1341, 1341, 1752, 1341, 1341, 1752, 1341, 1561, 1341, - /* 230 */ 1341, 1408, 1341, 1408, 1341, 1472, 1341, 1341, 1408, 1341, - /* 240 */ 1580, 1594, 1570, 1506, 1506, 1506, 1411, 1346, 1341, 1341, - /* 250 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1468, - /* 260 */ 1657, 1765, 1764, 1688, 1687, 1686, 1684, 1656, 1341, 1341, - /* 270 */ 1341, 1341, 1341, 1341, 1650, 1651, 1649, 1648, 1341, 1341, - /* 280 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 290 */ 1341, 1341, 1341, 1713, 1341, 1777, 1781, 1341, 1341, 1341, - /* 300 */ 1634, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 310 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 320 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 330 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 340 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 350 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 360 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 370 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 380 */ 1341, 1341, 1341, 1375, 1341, 1341, 1341, 1341, 1341, 1341, - /* 390 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 400 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 410 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 420 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1437, 1436, - /* 430 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 440 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 450 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 460 */ 1736, 1746, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 470 */ 1341, 1634, 1341, 1763, 1341, 1722, 1718, 1341, 1341, 1714, - /* 480 */ 1341, 1341, 1775, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 490 */ 1341, 1341, 1708, 1341, 1681, 1341, 1341, 1341, 1341, 1341, - /* 500 */ 1341, 1341, 1341, 1644, 1341, 1341, 1341, 1341, 1341, 1341, - /* 510 */ 1341, 1341, 1341, 1341, 1341, 1341, 1633, 1341, 1672, 1341, - /* 520 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1500, 1341, 1341, - /* 530 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 540 */ 1485, 1483, 1482, 1481, 1341, 1478, 1341, 1341, 1341, 1341, - /* 550 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1430, 1341, 1341, - /* 560 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1421, 1341, 1420, - /* 570 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 580 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 590 */ 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, 1341, - /* 600 */ 1341, 1341, 1341, + /* 0 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 10 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 20 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 30 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 40 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 50 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 60 */ 1345, 1345, 1414, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 70 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1412, 1552, 1345, + /* 80 */ 1717, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 90 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 100 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1414, 1345, + /* 110 */ 1728, 1728, 1728, 1412, 1345, 1345, 1345, 1345, 1345, 1345, + /* 120 */ 1507, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1590, + /* 130 */ 1345, 1345, 1794, 1345, 1596, 1752, 1345, 1345, 1345, 1345, + /* 140 */ 1460, 1744, 1720, 1734, 1721, 1779, 1779, 1779, 1737, 1345, + /* 150 */ 1345, 1345, 1345, 1582, 1345, 1345, 1557, 1554, 1554, 1345, + /* 160 */ 1345, 1345, 1345, 1414, 1345, 1414, 1345, 1414, 1345, 1345, + /* 170 */ 1414, 1414, 1345, 1414, 1345, 1345, 1345, 1345, 1345, 1345, + /* 180 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 190 */ 1412, 1592, 1345, 1412, 1345, 1412, 1345, 1345, 1412, 1345, + /* 200 */ 1759, 1757, 1345, 1759, 1757, 1345, 1345, 1345, 1771, 1767, + /* 210 */ 1750, 1748, 1734, 1345, 1345, 1345, 1785, 1781, 1797, 1785, + /* 220 */ 1781, 1785, 1781, 1345, 1757, 1345, 1345, 1757, 1345, 1565, + /* 230 */ 1345, 1345, 1412, 1345, 1412, 1345, 1476, 1345, 1345, 1412, + /* 240 */ 1345, 1584, 1598, 1574, 1510, 1510, 1510, 1415, 1350, 1345, + /* 250 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 260 */ 1472, 1661, 1770, 1769, 1693, 1692, 1691, 1689, 1660, 1345, + /* 270 */ 1345, 1345, 1345, 1345, 1345, 1654, 1655, 1653, 1652, 1345, + /* 280 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 290 */ 1345, 1345, 1345, 1345, 1718, 1345, 1782, 1786, 1345, 1345, + /* 300 */ 1345, 1638, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 310 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 320 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 330 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 340 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 350 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 360 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 370 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 380 */ 1345, 1345, 1345, 1345, 1379, 1345, 1345, 1345, 1345, 1345, + /* 390 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 400 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 410 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 420 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1441, + /* 430 */ 1440, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 440 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 450 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 460 */ 1345, 1741, 1751, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 470 */ 1345, 1345, 1345, 1638, 1345, 1768, 1345, 1727, 1723, 1345, + /* 480 */ 1345, 1719, 1345, 1345, 1780, 1345, 1345, 1345, 1345, 1345, + /* 490 */ 1345, 1345, 1345, 1345, 1713, 1345, 1686, 1345, 1345, 1345, + /* 500 */ 1345, 1345, 1345, 1345, 1345, 1648, 1345, 1345, 1345, 1345, + /* 510 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1637, 1345, + /* 520 */ 1677, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1504, + /* 530 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 540 */ 1345, 1345, 1489, 1487, 1486, 1485, 1345, 1482, 1345, 1345, + /* 550 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1434, + /* 560 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1425, + /* 570 */ 1345, 1424, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 580 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 590 */ 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, 1345, + /* 600 */ 1345, 1345, 1345, 1345, 1345, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1835,139 +1835,140 @@ static const char *const yyRuleName[] = { /* 315 */ "column_reference ::= table_name NK_DOT column_name", /* 316 */ "pseudo_column ::= ROWTS", /* 317 */ "pseudo_column ::= TBNAME", - /* 318 */ "pseudo_column ::= QSTARTTS", - /* 319 */ "pseudo_column ::= QENDTS", - /* 320 */ "pseudo_column ::= WSTARTTS", - /* 321 */ "pseudo_column ::= WENDTS", - /* 322 */ "pseudo_column ::= WDURATION", - /* 323 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 324 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 325 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", - /* 326 */ "function_expression ::= literal_func", - /* 327 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 328 */ "literal_func ::= NOW", - /* 329 */ "noarg_func ::= NOW", - /* 330 */ "noarg_func ::= TODAY", - /* 331 */ "noarg_func ::= TIMEZONE", - /* 332 */ "star_func ::= COUNT", - /* 333 */ "star_func ::= FIRST", - /* 334 */ "star_func ::= LAST", - /* 335 */ "star_func ::= LAST_ROW", - /* 336 */ "star_func_para_list ::= NK_STAR", - /* 337 */ "star_func_para_list ::= other_para_list", - /* 338 */ "other_para_list ::= star_func_para", - /* 339 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 340 */ "star_func_para ::= expression", - /* 341 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 342 */ "predicate ::= expression compare_op expression", - /* 343 */ "predicate ::= expression BETWEEN expression AND expression", - /* 344 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 345 */ "predicate ::= expression IS NULL", - /* 346 */ "predicate ::= expression IS NOT NULL", - /* 347 */ "predicate ::= expression in_op in_predicate_value", - /* 348 */ "compare_op ::= NK_LT", - /* 349 */ "compare_op ::= NK_GT", - /* 350 */ "compare_op ::= NK_LE", - /* 351 */ "compare_op ::= NK_GE", - /* 352 */ "compare_op ::= NK_NE", - /* 353 */ "compare_op ::= NK_EQ", - /* 354 */ "compare_op ::= LIKE", - /* 355 */ "compare_op ::= NOT LIKE", - /* 356 */ "compare_op ::= MATCH", - /* 357 */ "compare_op ::= NMATCH", - /* 358 */ "compare_op ::= CONTAINS", - /* 359 */ "in_op ::= IN", - /* 360 */ "in_op ::= NOT IN", - /* 361 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 362 */ "boolean_value_expression ::= boolean_primary", - /* 363 */ "boolean_value_expression ::= NOT boolean_primary", - /* 364 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 365 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 366 */ "boolean_primary ::= predicate", - /* 367 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 368 */ "common_expression ::= expression", - /* 369 */ "common_expression ::= boolean_value_expression", - /* 370 */ "from_clause ::= FROM table_reference_list", - /* 371 */ "table_reference_list ::= table_reference", - /* 372 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 373 */ "table_reference ::= table_primary", - /* 374 */ "table_reference ::= joined_table", - /* 375 */ "table_primary ::= table_name alias_opt", - /* 376 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 377 */ "table_primary ::= subquery alias_opt", - /* 378 */ "table_primary ::= parenthesized_joined_table", - /* 379 */ "alias_opt ::=", - /* 380 */ "alias_opt ::= table_alias", - /* 381 */ "alias_opt ::= AS table_alias", - /* 382 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 383 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 384 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 385 */ "join_type ::=", - /* 386 */ "join_type ::= INNER", - /* 387 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 388 */ "set_quantifier_opt ::=", - /* 389 */ "set_quantifier_opt ::= DISTINCT", - /* 390 */ "set_quantifier_opt ::= ALL", - /* 391 */ "select_list ::= NK_STAR", - /* 392 */ "select_list ::= select_sublist", - /* 393 */ "select_sublist ::= select_item", - /* 394 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 395 */ "select_item ::= common_expression", - /* 396 */ "select_item ::= common_expression column_alias", - /* 397 */ "select_item ::= common_expression AS column_alias", - /* 398 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 399 */ "where_clause_opt ::=", - /* 400 */ "where_clause_opt ::= WHERE search_condition", - /* 401 */ "partition_by_clause_opt ::=", - /* 402 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 403 */ "twindow_clause_opt ::=", - /* 404 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 405 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", - /* 406 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 407 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 408 */ "sliding_opt ::=", - /* 409 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 410 */ "fill_opt ::=", - /* 411 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 412 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 413 */ "fill_mode ::= NONE", - /* 414 */ "fill_mode ::= PREV", - /* 415 */ "fill_mode ::= NULL", - /* 416 */ "fill_mode ::= LINEAR", - /* 417 */ "fill_mode ::= NEXT", - /* 418 */ "group_by_clause_opt ::=", - /* 419 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 420 */ "group_by_list ::= expression", - /* 421 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 422 */ "having_clause_opt ::=", - /* 423 */ "having_clause_opt ::= HAVING search_condition", - /* 424 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 425 */ "query_expression_body ::= query_primary", - /* 426 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 427 */ "query_expression_body ::= query_expression_body UNION query_expression_body", - /* 428 */ "query_primary ::= query_specification", - /* 429 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", - /* 430 */ "order_by_clause_opt ::=", - /* 431 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 432 */ "slimit_clause_opt ::=", - /* 433 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 434 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 435 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 436 */ "limit_clause_opt ::=", - /* 437 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 438 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 439 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 440 */ "subquery ::= NK_LP query_expression NK_RP", - /* 441 */ "search_condition ::= common_expression", - /* 442 */ "sort_specification_list ::= sort_specification", - /* 443 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 444 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 445 */ "ordering_specification_opt ::=", - /* 446 */ "ordering_specification_opt ::= ASC", - /* 447 */ "ordering_specification_opt ::= DESC", - /* 448 */ "null_ordering_opt ::=", - /* 449 */ "null_ordering_opt ::= NULLS FIRST", - /* 450 */ "null_ordering_opt ::= NULLS LAST", + /* 318 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 319 */ "pseudo_column ::= QSTARTTS", + /* 320 */ "pseudo_column ::= QENDTS", + /* 321 */ "pseudo_column ::= WSTARTTS", + /* 322 */ "pseudo_column ::= WENDTS", + /* 323 */ "pseudo_column ::= WDURATION", + /* 324 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 325 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 326 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", + /* 327 */ "function_expression ::= literal_func", + /* 328 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 329 */ "literal_func ::= NOW", + /* 330 */ "noarg_func ::= NOW", + /* 331 */ "noarg_func ::= TODAY", + /* 332 */ "noarg_func ::= TIMEZONE", + /* 333 */ "star_func ::= COUNT", + /* 334 */ "star_func ::= FIRST", + /* 335 */ "star_func ::= LAST", + /* 336 */ "star_func ::= LAST_ROW", + /* 337 */ "star_func_para_list ::= NK_STAR", + /* 338 */ "star_func_para_list ::= other_para_list", + /* 339 */ "other_para_list ::= star_func_para", + /* 340 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 341 */ "star_func_para ::= expression", + /* 342 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 343 */ "predicate ::= expression compare_op expression", + /* 344 */ "predicate ::= expression BETWEEN expression AND expression", + /* 345 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 346 */ "predicate ::= expression IS NULL", + /* 347 */ "predicate ::= expression IS NOT NULL", + /* 348 */ "predicate ::= expression in_op in_predicate_value", + /* 349 */ "compare_op ::= NK_LT", + /* 350 */ "compare_op ::= NK_GT", + /* 351 */ "compare_op ::= NK_LE", + /* 352 */ "compare_op ::= NK_GE", + /* 353 */ "compare_op ::= NK_NE", + /* 354 */ "compare_op ::= NK_EQ", + /* 355 */ "compare_op ::= LIKE", + /* 356 */ "compare_op ::= NOT LIKE", + /* 357 */ "compare_op ::= MATCH", + /* 358 */ "compare_op ::= NMATCH", + /* 359 */ "compare_op ::= CONTAINS", + /* 360 */ "in_op ::= IN", + /* 361 */ "in_op ::= NOT IN", + /* 362 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 363 */ "boolean_value_expression ::= boolean_primary", + /* 364 */ "boolean_value_expression ::= NOT boolean_primary", + /* 365 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 366 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 367 */ "boolean_primary ::= predicate", + /* 368 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 369 */ "common_expression ::= expression", + /* 370 */ "common_expression ::= boolean_value_expression", + /* 371 */ "from_clause ::= FROM table_reference_list", + /* 372 */ "table_reference_list ::= table_reference", + /* 373 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 374 */ "table_reference ::= table_primary", + /* 375 */ "table_reference ::= joined_table", + /* 376 */ "table_primary ::= table_name alias_opt", + /* 377 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 378 */ "table_primary ::= subquery alias_opt", + /* 379 */ "table_primary ::= parenthesized_joined_table", + /* 380 */ "alias_opt ::=", + /* 381 */ "alias_opt ::= table_alias", + /* 382 */ "alias_opt ::= AS table_alias", + /* 383 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 384 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 385 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 386 */ "join_type ::=", + /* 387 */ "join_type ::= INNER", + /* 388 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 389 */ "set_quantifier_opt ::=", + /* 390 */ "set_quantifier_opt ::= DISTINCT", + /* 391 */ "set_quantifier_opt ::= ALL", + /* 392 */ "select_list ::= NK_STAR", + /* 393 */ "select_list ::= select_sublist", + /* 394 */ "select_sublist ::= select_item", + /* 395 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 396 */ "select_item ::= common_expression", + /* 397 */ "select_item ::= common_expression column_alias", + /* 398 */ "select_item ::= common_expression AS column_alias", + /* 399 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 400 */ "where_clause_opt ::=", + /* 401 */ "where_clause_opt ::= WHERE search_condition", + /* 402 */ "partition_by_clause_opt ::=", + /* 403 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 404 */ "twindow_clause_opt ::=", + /* 405 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 406 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", + /* 407 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 408 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 409 */ "sliding_opt ::=", + /* 410 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 411 */ "fill_opt ::=", + /* 412 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 413 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 414 */ "fill_mode ::= NONE", + /* 415 */ "fill_mode ::= PREV", + /* 416 */ "fill_mode ::= NULL", + /* 417 */ "fill_mode ::= LINEAR", + /* 418 */ "fill_mode ::= NEXT", + /* 419 */ "group_by_clause_opt ::=", + /* 420 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 421 */ "group_by_list ::= expression", + /* 422 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 423 */ "having_clause_opt ::=", + /* 424 */ "having_clause_opt ::= HAVING search_condition", + /* 425 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 426 */ "query_expression_body ::= query_primary", + /* 427 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 428 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 429 */ "query_primary ::= query_specification", + /* 430 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", + /* 431 */ "order_by_clause_opt ::=", + /* 432 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 433 */ "slimit_clause_opt ::=", + /* 434 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 435 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 436 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 437 */ "limit_clause_opt ::=", + /* 438 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 439 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 440 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 441 */ "subquery ::= NK_LP query_expression NK_RP", + /* 442 */ "search_condition ::= common_expression", + /* 443 */ "sort_specification_list ::= sort_specification", + /* 444 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 445 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 446 */ "ordering_specification_opt ::=", + /* 447 */ "ordering_specification_opt ::= ASC", + /* 448 */ "ordering_specification_opt ::= DESC", + /* 449 */ "null_ordering_opt ::=", + /* 450 */ "null_ordering_opt ::= NULLS FIRST", + /* 451 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2878,139 +2879,140 @@ static const struct { { 312, -3 }, /* (315) column_reference ::= table_name NK_DOT column_name */ { 311, -1 }, /* (316) pseudo_column ::= ROWTS */ { 311, -1 }, /* (317) pseudo_column ::= TBNAME */ - { 311, -1 }, /* (318) pseudo_column ::= QSTARTTS */ - { 311, -1 }, /* (319) pseudo_column ::= QENDTS */ - { 311, -1 }, /* (320) pseudo_column ::= WSTARTTS */ - { 311, -1 }, /* (321) pseudo_column ::= WENDTS */ - { 311, -1 }, /* (322) pseudo_column ::= WDURATION */ - { 313, -4 }, /* (323) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 313, -4 }, /* (324) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 313, -6 }, /* (325) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ - { 313, -1 }, /* (326) function_expression ::= literal_func */ - { 307, -3 }, /* (327) literal_func ::= noarg_func NK_LP NK_RP */ - { 307, -1 }, /* (328) literal_func ::= NOW */ - { 317, -1 }, /* (329) noarg_func ::= NOW */ - { 317, -1 }, /* (330) noarg_func ::= TODAY */ - { 317, -1 }, /* (331) noarg_func ::= TIMEZONE */ - { 315, -1 }, /* (332) star_func ::= COUNT */ - { 315, -1 }, /* (333) star_func ::= FIRST */ - { 315, -1 }, /* (334) star_func ::= LAST */ - { 315, -1 }, /* (335) star_func ::= LAST_ROW */ - { 316, -1 }, /* (336) star_func_para_list ::= NK_STAR */ - { 316, -1 }, /* (337) star_func_para_list ::= other_para_list */ - { 318, -1 }, /* (338) other_para_list ::= star_func_para */ - { 318, -3 }, /* (339) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 319, -1 }, /* (340) star_func_para ::= expression */ - { 319, -3 }, /* (341) star_func_para ::= table_name NK_DOT NK_STAR */ - { 320, -3 }, /* (342) predicate ::= expression compare_op expression */ - { 320, -5 }, /* (343) predicate ::= expression BETWEEN expression AND expression */ - { 320, -6 }, /* (344) predicate ::= expression NOT BETWEEN expression AND expression */ - { 320, -3 }, /* (345) predicate ::= expression IS NULL */ - { 320, -4 }, /* (346) predicate ::= expression IS NOT NULL */ - { 320, -3 }, /* (347) predicate ::= expression in_op in_predicate_value */ - { 321, -1 }, /* (348) compare_op ::= NK_LT */ - { 321, -1 }, /* (349) compare_op ::= NK_GT */ - { 321, -1 }, /* (350) compare_op ::= NK_LE */ - { 321, -1 }, /* (351) compare_op ::= NK_GE */ - { 321, -1 }, /* (352) compare_op ::= NK_NE */ - { 321, -1 }, /* (353) compare_op ::= NK_EQ */ - { 321, -1 }, /* (354) compare_op ::= LIKE */ - { 321, -2 }, /* (355) compare_op ::= NOT LIKE */ - { 321, -1 }, /* (356) compare_op ::= MATCH */ - { 321, -1 }, /* (357) compare_op ::= NMATCH */ - { 321, -1 }, /* (358) compare_op ::= CONTAINS */ - { 322, -1 }, /* (359) in_op ::= IN */ - { 322, -2 }, /* (360) in_op ::= NOT IN */ - { 323, -3 }, /* (361) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 324, -1 }, /* (362) boolean_value_expression ::= boolean_primary */ - { 324, -2 }, /* (363) boolean_value_expression ::= NOT boolean_primary */ - { 324, -3 }, /* (364) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 324, -3 }, /* (365) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 325, -1 }, /* (366) boolean_primary ::= predicate */ - { 325, -3 }, /* (367) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 326, -1 }, /* (368) common_expression ::= expression */ - { 326, -1 }, /* (369) common_expression ::= boolean_value_expression */ - { 327, -2 }, /* (370) from_clause ::= FROM table_reference_list */ - { 328, -1 }, /* (371) table_reference_list ::= table_reference */ - { 328, -3 }, /* (372) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 329, -1 }, /* (373) table_reference ::= table_primary */ - { 329, -1 }, /* (374) table_reference ::= joined_table */ - { 330, -2 }, /* (375) table_primary ::= table_name alias_opt */ - { 330, -4 }, /* (376) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 330, -2 }, /* (377) table_primary ::= subquery alias_opt */ - { 330, -1 }, /* (378) table_primary ::= parenthesized_joined_table */ - { 332, 0 }, /* (379) alias_opt ::= */ - { 332, -1 }, /* (380) alias_opt ::= table_alias */ - { 332, -2 }, /* (381) alias_opt ::= AS table_alias */ - { 333, -3 }, /* (382) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 333, -3 }, /* (383) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 331, -6 }, /* (384) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 334, 0 }, /* (385) join_type ::= */ - { 334, -1 }, /* (386) join_type ::= INNER */ - { 336, -9 }, /* (387) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 337, 0 }, /* (388) set_quantifier_opt ::= */ - { 337, -1 }, /* (389) set_quantifier_opt ::= DISTINCT */ - { 337, -1 }, /* (390) set_quantifier_opt ::= ALL */ - { 338, -1 }, /* (391) select_list ::= NK_STAR */ - { 338, -1 }, /* (392) select_list ::= select_sublist */ - { 344, -1 }, /* (393) select_sublist ::= select_item */ - { 344, -3 }, /* (394) select_sublist ::= select_sublist NK_COMMA select_item */ - { 345, -1 }, /* (395) select_item ::= common_expression */ - { 345, -2 }, /* (396) select_item ::= common_expression column_alias */ - { 345, -3 }, /* (397) select_item ::= common_expression AS column_alias */ - { 345, -3 }, /* (398) select_item ::= table_name NK_DOT NK_STAR */ - { 339, 0 }, /* (399) where_clause_opt ::= */ - { 339, -2 }, /* (400) where_clause_opt ::= WHERE search_condition */ - { 340, 0 }, /* (401) partition_by_clause_opt ::= */ - { 340, -3 }, /* (402) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 341, 0 }, /* (403) twindow_clause_opt ::= */ - { 341, -6 }, /* (404) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 341, -4 }, /* (405) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ - { 341, -6 }, /* (406) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 341, -8 }, /* (407) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 291, 0 }, /* (408) sliding_opt ::= */ - { 291, -4 }, /* (409) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 346, 0 }, /* (410) fill_opt ::= */ - { 346, -4 }, /* (411) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 346, -6 }, /* (412) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 347, -1 }, /* (413) fill_mode ::= NONE */ - { 347, -1 }, /* (414) fill_mode ::= PREV */ - { 347, -1 }, /* (415) fill_mode ::= NULL */ - { 347, -1 }, /* (416) fill_mode ::= LINEAR */ - { 347, -1 }, /* (417) fill_mode ::= NEXT */ - { 342, 0 }, /* (418) group_by_clause_opt ::= */ - { 342, -3 }, /* (419) group_by_clause_opt ::= GROUP BY group_by_list */ - { 348, -1 }, /* (420) group_by_list ::= expression */ - { 348, -3 }, /* (421) group_by_list ::= group_by_list NK_COMMA expression */ - { 343, 0 }, /* (422) having_clause_opt ::= */ - { 343, -2 }, /* (423) having_clause_opt ::= HAVING search_condition */ - { 296, -4 }, /* (424) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 349, -1 }, /* (425) query_expression_body ::= query_primary */ - { 349, -4 }, /* (426) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 349, -3 }, /* (427) query_expression_body ::= query_expression_body UNION query_expression_body */ - { 353, -1 }, /* (428) query_primary ::= query_specification */ - { 353, -6 }, /* (429) query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ - { 350, 0 }, /* (430) order_by_clause_opt ::= */ - { 350, -3 }, /* (431) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 351, 0 }, /* (432) slimit_clause_opt ::= */ - { 351, -2 }, /* (433) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 351, -4 }, /* (434) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 351, -4 }, /* (435) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 352, 0 }, /* (436) limit_clause_opt ::= */ - { 352, -2 }, /* (437) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 352, -4 }, /* (438) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 352, -4 }, /* (439) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 314, -3 }, /* (440) subquery ::= NK_LP query_expression NK_RP */ - { 335, -1 }, /* (441) search_condition ::= common_expression */ - { 354, -1 }, /* (442) sort_specification_list ::= sort_specification */ - { 354, -3 }, /* (443) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 355, -3 }, /* (444) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 356, 0 }, /* (445) ordering_specification_opt ::= */ - { 356, -1 }, /* (446) ordering_specification_opt ::= ASC */ - { 356, -1 }, /* (447) ordering_specification_opt ::= DESC */ - { 357, 0 }, /* (448) null_ordering_opt ::= */ - { 357, -2 }, /* (449) null_ordering_opt ::= NULLS FIRST */ - { 357, -2 }, /* (450) null_ordering_opt ::= NULLS LAST */ + { 311, -3 }, /* (318) pseudo_column ::= table_name NK_DOT TBNAME */ + { 311, -1 }, /* (319) pseudo_column ::= QSTARTTS */ + { 311, -1 }, /* (320) pseudo_column ::= QENDTS */ + { 311, -1 }, /* (321) pseudo_column ::= WSTARTTS */ + { 311, -1 }, /* (322) pseudo_column ::= WENDTS */ + { 311, -1 }, /* (323) pseudo_column ::= WDURATION */ + { 313, -4 }, /* (324) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 313, -4 }, /* (325) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 313, -6 }, /* (326) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ + { 313, -1 }, /* (327) function_expression ::= literal_func */ + { 307, -3 }, /* (328) literal_func ::= noarg_func NK_LP NK_RP */ + { 307, -1 }, /* (329) literal_func ::= NOW */ + { 317, -1 }, /* (330) noarg_func ::= NOW */ + { 317, -1 }, /* (331) noarg_func ::= TODAY */ + { 317, -1 }, /* (332) noarg_func ::= TIMEZONE */ + { 315, -1 }, /* (333) star_func ::= COUNT */ + { 315, -1 }, /* (334) star_func ::= FIRST */ + { 315, -1 }, /* (335) star_func ::= LAST */ + { 315, -1 }, /* (336) star_func ::= LAST_ROW */ + { 316, -1 }, /* (337) star_func_para_list ::= NK_STAR */ + { 316, -1 }, /* (338) star_func_para_list ::= other_para_list */ + { 318, -1 }, /* (339) other_para_list ::= star_func_para */ + { 318, -3 }, /* (340) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 319, -1 }, /* (341) star_func_para ::= expression */ + { 319, -3 }, /* (342) star_func_para ::= table_name NK_DOT NK_STAR */ + { 320, -3 }, /* (343) predicate ::= expression compare_op expression */ + { 320, -5 }, /* (344) predicate ::= expression BETWEEN expression AND expression */ + { 320, -6 }, /* (345) predicate ::= expression NOT BETWEEN expression AND expression */ + { 320, -3 }, /* (346) predicate ::= expression IS NULL */ + { 320, -4 }, /* (347) predicate ::= expression IS NOT NULL */ + { 320, -3 }, /* (348) predicate ::= expression in_op in_predicate_value */ + { 321, -1 }, /* (349) compare_op ::= NK_LT */ + { 321, -1 }, /* (350) compare_op ::= NK_GT */ + { 321, -1 }, /* (351) compare_op ::= NK_LE */ + { 321, -1 }, /* (352) compare_op ::= NK_GE */ + { 321, -1 }, /* (353) compare_op ::= NK_NE */ + { 321, -1 }, /* (354) compare_op ::= NK_EQ */ + { 321, -1 }, /* (355) compare_op ::= LIKE */ + { 321, -2 }, /* (356) compare_op ::= NOT LIKE */ + { 321, -1 }, /* (357) compare_op ::= MATCH */ + { 321, -1 }, /* (358) compare_op ::= NMATCH */ + { 321, -1 }, /* (359) compare_op ::= CONTAINS */ + { 322, -1 }, /* (360) in_op ::= IN */ + { 322, -2 }, /* (361) in_op ::= NOT IN */ + { 323, -3 }, /* (362) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 324, -1 }, /* (363) boolean_value_expression ::= boolean_primary */ + { 324, -2 }, /* (364) boolean_value_expression ::= NOT boolean_primary */ + { 324, -3 }, /* (365) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 324, -3 }, /* (366) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 325, -1 }, /* (367) boolean_primary ::= predicate */ + { 325, -3 }, /* (368) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 326, -1 }, /* (369) common_expression ::= expression */ + { 326, -1 }, /* (370) common_expression ::= boolean_value_expression */ + { 327, -2 }, /* (371) from_clause ::= FROM table_reference_list */ + { 328, -1 }, /* (372) table_reference_list ::= table_reference */ + { 328, -3 }, /* (373) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 329, -1 }, /* (374) table_reference ::= table_primary */ + { 329, -1 }, /* (375) table_reference ::= joined_table */ + { 330, -2 }, /* (376) table_primary ::= table_name alias_opt */ + { 330, -4 }, /* (377) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 330, -2 }, /* (378) table_primary ::= subquery alias_opt */ + { 330, -1 }, /* (379) table_primary ::= parenthesized_joined_table */ + { 332, 0 }, /* (380) alias_opt ::= */ + { 332, -1 }, /* (381) alias_opt ::= table_alias */ + { 332, -2 }, /* (382) alias_opt ::= AS table_alias */ + { 333, -3 }, /* (383) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 333, -3 }, /* (384) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 331, -6 }, /* (385) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 334, 0 }, /* (386) join_type ::= */ + { 334, -1 }, /* (387) join_type ::= INNER */ + { 336, -9 }, /* (388) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 337, 0 }, /* (389) set_quantifier_opt ::= */ + { 337, -1 }, /* (390) set_quantifier_opt ::= DISTINCT */ + { 337, -1 }, /* (391) set_quantifier_opt ::= ALL */ + { 338, -1 }, /* (392) select_list ::= NK_STAR */ + { 338, -1 }, /* (393) select_list ::= select_sublist */ + { 344, -1 }, /* (394) select_sublist ::= select_item */ + { 344, -3 }, /* (395) select_sublist ::= select_sublist NK_COMMA select_item */ + { 345, -1 }, /* (396) select_item ::= common_expression */ + { 345, -2 }, /* (397) select_item ::= common_expression column_alias */ + { 345, -3 }, /* (398) select_item ::= common_expression AS column_alias */ + { 345, -3 }, /* (399) select_item ::= table_name NK_DOT NK_STAR */ + { 339, 0 }, /* (400) where_clause_opt ::= */ + { 339, -2 }, /* (401) where_clause_opt ::= WHERE search_condition */ + { 340, 0 }, /* (402) partition_by_clause_opt ::= */ + { 340, -3 }, /* (403) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 341, 0 }, /* (404) twindow_clause_opt ::= */ + { 341, -6 }, /* (405) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 341, -4 }, /* (406) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ + { 341, -6 }, /* (407) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 341, -8 }, /* (408) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 291, 0 }, /* (409) sliding_opt ::= */ + { 291, -4 }, /* (410) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 346, 0 }, /* (411) fill_opt ::= */ + { 346, -4 }, /* (412) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 346, -6 }, /* (413) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 347, -1 }, /* (414) fill_mode ::= NONE */ + { 347, -1 }, /* (415) fill_mode ::= PREV */ + { 347, -1 }, /* (416) fill_mode ::= NULL */ + { 347, -1 }, /* (417) fill_mode ::= LINEAR */ + { 347, -1 }, /* (418) fill_mode ::= NEXT */ + { 342, 0 }, /* (419) group_by_clause_opt ::= */ + { 342, -3 }, /* (420) group_by_clause_opt ::= GROUP BY group_by_list */ + { 348, -1 }, /* (421) group_by_list ::= expression */ + { 348, -3 }, /* (422) group_by_list ::= group_by_list NK_COMMA expression */ + { 343, 0 }, /* (423) having_clause_opt ::= */ + { 343, -2 }, /* (424) having_clause_opt ::= HAVING search_condition */ + { 296, -4 }, /* (425) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 349, -1 }, /* (426) query_expression_body ::= query_primary */ + { 349, -4 }, /* (427) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 349, -3 }, /* (428) query_expression_body ::= query_expression_body UNION query_expression_body */ + { 353, -1 }, /* (429) query_primary ::= query_specification */ + { 353, -6 }, /* (430) query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ + { 350, 0 }, /* (431) order_by_clause_opt ::= */ + { 350, -3 }, /* (432) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 351, 0 }, /* (433) slimit_clause_opt ::= */ + { 351, -2 }, /* (434) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 351, -4 }, /* (435) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 351, -4 }, /* (436) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 352, 0 }, /* (437) limit_clause_opt ::= */ + { 352, -2 }, /* (438) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 352, -4 }, /* (439) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 352, -4 }, /* (440) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 314, -3 }, /* (441) subquery ::= NK_LP query_expression NK_RP */ + { 335, -1 }, /* (442) search_condition ::= common_expression */ + { 354, -1 }, /* (443) sort_specification_list ::= sort_specification */ + { 354, -3 }, /* (444) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 355, -3 }, /* (445) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 356, 0 }, /* (446) ordering_specification_opt ::= */ + { 356, -1 }, /* (447) ordering_specification_opt ::= ASC */ + { 356, -1 }, /* (448) ordering_specification_opt ::= DESC */ + { 357, 0 }, /* (449) null_ordering_opt ::= */ + { 357, -2 }, /* (450) null_ordering_opt ::= NULLS FIRST */ + { 357, -2 }, /* (451) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3227,13 +3229,13 @@ static YYACTIONTYPE yy_reduce( case 295: /* index_name ::= NK_ID */ yytestcase(yyruleno==295); case 296: /* topic_name ::= NK_ID */ yytestcase(yyruleno==296); case 297: /* stream_name ::= NK_ID */ yytestcase(yyruleno==297); - case 329: /* noarg_func ::= NOW */ yytestcase(yyruleno==329); - case 330: /* noarg_func ::= TODAY */ yytestcase(yyruleno==330); - case 331: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==331); - case 332: /* star_func ::= COUNT */ yytestcase(yyruleno==332); - case 333: /* star_func ::= FIRST */ yytestcase(yyruleno==333); - case 334: /* star_func ::= LAST */ yytestcase(yyruleno==334); - case 335: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==335); + case 330: /* noarg_func ::= NOW */ yytestcase(yyruleno==330); + case 331: /* noarg_func ::= TODAY */ yytestcase(yyruleno==331); + case 332: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==332); + case 333: /* star_func ::= COUNT */ yytestcase(yyruleno==333); + case 334: /* star_func ::= FIRST */ yytestcase(yyruleno==334); + case 335: /* star_func ::= LAST */ yytestcase(yyruleno==335); + case 336: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==336); { yylhsminor.yy105 = yymsp[0].minor.yy0; } yymsp[0].minor.yy105 = yylhsminor.yy105; break; @@ -3286,7 +3288,7 @@ static YYACTIONTYPE yy_reduce( case 66: /* exists_opt ::= */ yytestcase(yyruleno==66); case 234: /* analyze_opt ::= */ yytestcase(yyruleno==234); case 242: /* agg_func_opt ::= */ yytestcase(yyruleno==242); - case 388: /* set_quantifier_opt ::= */ yytestcase(yyruleno==388); + case 389: /* set_quantifier_opt ::= */ yytestcase(yyruleno==389); { yymsp[1].minor.yy617 = false; } break; case 65: /* exists_opt ::= IF EXISTS */ @@ -3423,9 +3425,9 @@ static YYACTIONTYPE yy_reduce( case 211: /* func_name_list ::= func_name */ yytestcase(yyruleno==211); case 220: /* func_list ::= func */ yytestcase(yyruleno==220); case 286: /* literal_list ::= signed_literal */ yytestcase(yyruleno==286); - case 338: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==338); - case 393: /* select_sublist ::= select_item */ yytestcase(yyruleno==393); - case 442: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==442); + case 339: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==339); + case 394: /* select_sublist ::= select_item */ yytestcase(yyruleno==394); + case 443: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==443); { yylhsminor.yy60 = createNodeList(pCxt, yymsp[0].minor.yy172); } yymsp[0].minor.yy60 = yylhsminor.yy60; break; @@ -3435,9 +3437,9 @@ static YYACTIONTYPE yy_reduce( case 212: /* func_name_list ::= func_name_list NK_COMMA func_name */ yytestcase(yyruleno==212); case 221: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==221); case 287: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==287); - case 339: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==339); - case 394: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==394); - case 443: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==443); + case 340: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==340); + case 395: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==395); + case 444: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==444); { yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, yymsp[0].minor.yy172); } yymsp[-2].minor.yy60 = yylhsminor.yy60; break; @@ -3518,9 +3520,9 @@ static YYACTIONTYPE yy_reduce( break; case 128: /* specific_tags_opt ::= */ case 159: /* tags_def_opt ::= */ yytestcase(yyruleno==159); - case 401: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==401); - case 418: /* group_by_clause_opt ::= */ yytestcase(yyruleno==418); - case 430: /* order_by_clause_opt ::= */ yytestcase(yyruleno==430); + case 402: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==402); + case 419: /* group_by_clause_opt ::= */ yytestcase(yyruleno==419); + case 431: /* order_by_clause_opt ::= */ yytestcase(yyruleno==431); { yymsp[1].minor.yy60 = NULL; } break; case 129: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ @@ -3610,8 +3612,8 @@ static YYACTIONTYPE yy_reduce( { yymsp[-5].minor.yy248 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 160: /* tags_def_opt ::= tags_def */ - case 337: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==337); - case 392: /* select_list ::= select_sublist */ yytestcase(yyruleno==392); + case 338: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==338); + case 393: /* select_list ::= select_sublist */ yytestcase(yyruleno==393); { yylhsminor.yy60 = yymsp[0].minor.yy60; } yymsp[0].minor.yy60 = yylhsminor.yy60; break; @@ -3756,13 +3758,13 @@ static YYACTIONTYPE yy_reduce( case 206: /* like_pattern_opt ::= */ case 217: /* index_options ::= */ yytestcase(yyruleno==217); case 248: /* into_opt ::= */ yytestcase(yyruleno==248); - case 399: /* where_clause_opt ::= */ yytestcase(yyruleno==399); - case 403: /* twindow_clause_opt ::= */ yytestcase(yyruleno==403); - case 408: /* sliding_opt ::= */ yytestcase(yyruleno==408); - case 410: /* fill_opt ::= */ yytestcase(yyruleno==410); - case 422: /* having_clause_opt ::= */ yytestcase(yyruleno==422); - case 432: /* slimit_clause_opt ::= */ yytestcase(yyruleno==432); - case 436: /* limit_clause_opt ::= */ yytestcase(yyruleno==436); + case 400: /* where_clause_opt ::= */ yytestcase(yyruleno==400); + case 404: /* twindow_clause_opt ::= */ yytestcase(yyruleno==404); + case 409: /* sliding_opt ::= */ yytestcase(yyruleno==409); + case 411: /* fill_opt ::= */ yytestcase(yyruleno==411); + case 423: /* having_clause_opt ::= */ yytestcase(yyruleno==423); + case 433: /* slimit_clause_opt ::= */ yytestcase(yyruleno==433); + case 437: /* limit_clause_opt ::= */ yytestcase(yyruleno==437); { yymsp[1].minor.yy172 = NULL; } break; case 207: /* like_pattern_opt ::= LIKE NK_STRING */ @@ -3834,7 +3836,7 @@ static YYACTIONTYPE yy_reduce( break; case 235: /* analyze_opt ::= ANALYZE */ case 243: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==243); - case 389: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==389); + case 390: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==390); { yymsp[0].minor.yy617 = true; } break; case 236: /* explain_options ::= */ @@ -3870,9 +3872,9 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy105); } break; case 249: /* into_opt ::= INTO full_table_name */ - case 370: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==370); - case 400: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==400); - case 423: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==423); + case 371: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==371); + case 401: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==401); + case 424: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==424); { yymsp[-1].minor.yy172 = yymsp[0].minor.yy172; } break; case 250: /* stream_options ::= */ @@ -3941,17 +3943,17 @@ static YYACTIONTYPE yy_reduce( case 300: /* expression ::= column_reference */ yytestcase(yyruleno==300); case 301: /* expression ::= function_expression */ yytestcase(yyruleno==301); case 302: /* expression ::= subquery */ yytestcase(yyruleno==302); - case 326: /* function_expression ::= literal_func */ yytestcase(yyruleno==326); - case 362: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==362); - case 366: /* boolean_primary ::= predicate */ yytestcase(yyruleno==366); - case 368: /* common_expression ::= expression */ yytestcase(yyruleno==368); - case 369: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==369); - case 371: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==371); - case 373: /* table_reference ::= table_primary */ yytestcase(yyruleno==373); - case 374: /* table_reference ::= joined_table */ yytestcase(yyruleno==374); - case 378: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==378); - case 425: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==425); - case 428: /* query_primary ::= query_specification */ yytestcase(yyruleno==428); + case 327: /* function_expression ::= literal_func */ yytestcase(yyruleno==327); + case 363: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==363); + case 367: /* boolean_primary ::= predicate */ yytestcase(yyruleno==367); + case 369: /* common_expression ::= expression */ yytestcase(yyruleno==369); + case 370: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==370); + case 372: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==372); + case 374: /* table_reference ::= table_primary */ yytestcase(yyruleno==374); + case 375: /* table_reference ::= joined_table */ yytestcase(yyruleno==375); + case 379: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==379); + case 426: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==426); + case 429: /* query_primary ::= query_specification */ yytestcase(yyruleno==429); { yylhsminor.yy172 = yymsp[0].minor.yy172; } yymsp[0].minor.yy172 = yylhsminor.yy172; break; @@ -4010,9 +4012,9 @@ static YYACTIONTYPE yy_reduce( break; case 283: /* signed_literal ::= duration_literal */ case 285: /* signed_literal ::= literal_func */ yytestcase(yyruleno==285); - case 340: /* star_func_para ::= expression */ yytestcase(yyruleno==340); - case 395: /* select_item ::= common_expression */ yytestcase(yyruleno==395); - case 441: /* search_condition ::= common_expression */ yytestcase(yyruleno==441); + case 341: /* star_func_para ::= expression */ yytestcase(yyruleno==341); + case 396: /* select_item ::= common_expression */ yytestcase(yyruleno==396); + case 442: /* search_condition ::= common_expression */ yytestcase(yyruleno==442); { yylhsminor.yy172 = releaseRawExprNode(pCxt, yymsp[0].minor.yy172); } yymsp[0].minor.yy172 = yylhsminor.yy172; break; @@ -4021,7 +4023,7 @@ static YYACTIONTYPE yy_reduce( yymsp[0].minor.yy172 = yylhsminor.yy172; break; case 303: /* expression ::= NK_LP expression NK_RP */ - case 367: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==367); + case 368: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==368); { yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172)); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; @@ -4104,39 +4106,43 @@ static YYACTIONTYPE yy_reduce( break; case 316: /* pseudo_column ::= ROWTS */ case 317: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==317); - case 318: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==318); - case 319: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==319); - case 320: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==320); - case 321: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==321); - case 322: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==322); - case 328: /* literal_func ::= NOW */ yytestcase(yyruleno==328); + case 319: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==319); + case 320: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==320); + case 321: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==321); + case 322: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==322); + case 323: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==323); + case 329: /* literal_func ::= NOW */ yytestcase(yyruleno==329); { yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } yymsp[0].minor.yy172 = yylhsminor.yy172; break; - case 323: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 324: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==324); + case 318: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy105)))); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; + break; + case 324: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 325: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==325); { yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy105, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy105, yymsp[-1].minor.yy60)); } yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 325: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ + case 326: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ { yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), yymsp[-1].minor.yy248)); } yymsp[-5].minor.yy172 = yylhsminor.yy172; break; - case 327: /* literal_func ::= noarg_func NK_LP NK_RP */ + case 328: /* literal_func ::= noarg_func NK_LP NK_RP */ { yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy105, NULL)); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 336: /* star_func_para_list ::= NK_STAR */ + case 337: /* star_func_para_list ::= NK_STAR */ { yylhsminor.yy60 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy60 = yylhsminor.yy60; break; - case 341: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 398: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==398); + case 342: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 399: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==399); { yylhsminor.yy172 = createColumnNode(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 342: /* predicate ::= expression compare_op expression */ - case 347: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==347); + case 343: /* predicate ::= expression compare_op expression */ + case 348: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==348); { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); @@ -4144,7 +4150,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 343: /* predicate ::= expression BETWEEN expression AND expression */ + case 344: /* predicate ::= expression BETWEEN expression AND expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy172); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); @@ -4152,7 +4158,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy172 = yylhsminor.yy172; break; - case 344: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 345: /* predicate ::= expression NOT BETWEEN expression AND expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy172); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); @@ -4160,71 +4166,71 @@ static YYACTIONTYPE yy_reduce( } yymsp[-5].minor.yy172 = yylhsminor.yy172; break; - case 345: /* predicate ::= expression IS NULL */ + case 346: /* predicate ::= expression IS NULL */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), NULL)); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 346: /* predicate ::= expression IS NOT NULL */ + case 347: /* predicate ::= expression IS NOT NULL */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy172); yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), NULL)); } yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 348: /* compare_op ::= NK_LT */ + case 349: /* compare_op ::= NK_LT */ { yymsp[0].minor.yy572 = OP_TYPE_LOWER_THAN; } break; - case 349: /* compare_op ::= NK_GT */ + case 350: /* compare_op ::= NK_GT */ { yymsp[0].minor.yy572 = OP_TYPE_GREATER_THAN; } break; - case 350: /* compare_op ::= NK_LE */ + case 351: /* compare_op ::= NK_LE */ { yymsp[0].minor.yy572 = OP_TYPE_LOWER_EQUAL; } break; - case 351: /* compare_op ::= NK_GE */ + case 352: /* compare_op ::= NK_GE */ { yymsp[0].minor.yy572 = OP_TYPE_GREATER_EQUAL; } break; - case 352: /* compare_op ::= NK_NE */ + case 353: /* compare_op ::= NK_NE */ { yymsp[0].minor.yy572 = OP_TYPE_NOT_EQUAL; } break; - case 353: /* compare_op ::= NK_EQ */ + case 354: /* compare_op ::= NK_EQ */ { yymsp[0].minor.yy572 = OP_TYPE_EQUAL; } break; - case 354: /* compare_op ::= LIKE */ + case 355: /* compare_op ::= LIKE */ { yymsp[0].minor.yy572 = OP_TYPE_LIKE; } break; - case 355: /* compare_op ::= NOT LIKE */ + case 356: /* compare_op ::= NOT LIKE */ { yymsp[-1].minor.yy572 = OP_TYPE_NOT_LIKE; } break; - case 356: /* compare_op ::= MATCH */ + case 357: /* compare_op ::= MATCH */ { yymsp[0].minor.yy572 = OP_TYPE_MATCH; } break; - case 357: /* compare_op ::= NMATCH */ + case 358: /* compare_op ::= NMATCH */ { yymsp[0].minor.yy572 = OP_TYPE_NMATCH; } break; - case 358: /* compare_op ::= CONTAINS */ + case 359: /* compare_op ::= CONTAINS */ { yymsp[0].minor.yy572 = OP_TYPE_JSON_CONTAINS; } break; - case 359: /* in_op ::= IN */ + case 360: /* in_op ::= IN */ { yymsp[0].minor.yy572 = OP_TYPE_IN; } break; - case 360: /* in_op ::= NOT IN */ + case 361: /* in_op ::= NOT IN */ { yymsp[-1].minor.yy572 = OP_TYPE_NOT_IN; } break; - case 361: /* in_predicate_value ::= NK_LP expression_list NK_RP */ + case 362: /* in_predicate_value ::= NK_LP expression_list NK_RP */ { yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy60)); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 363: /* boolean_value_expression ::= NOT boolean_primary */ + case 364: /* boolean_value_expression ::= NOT boolean_primary */ { SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy172), NULL)); } yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 364: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 365: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); @@ -4232,7 +4238,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 365: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 366: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); @@ -4240,47 +4246,47 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 372: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ + case 373: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ { yylhsminor.yy172 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy172, yymsp[0].minor.yy172, NULL); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 375: /* table_primary ::= table_name alias_opt */ + case 376: /* table_primary ::= table_name alias_opt */ { yylhsminor.yy172 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy105, &yymsp[0].minor.yy105); } yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 376: /* table_primary ::= db_name NK_DOT table_name alias_opt */ + case 377: /* table_primary ::= db_name NK_DOT table_name alias_opt */ { yylhsminor.yy172 = createRealTableNode(pCxt, &yymsp[-3].minor.yy105, &yymsp[-1].minor.yy105, &yymsp[0].minor.yy105); } yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 377: /* table_primary ::= subquery alias_opt */ + case 378: /* table_primary ::= subquery alias_opt */ { yylhsminor.yy172 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172), &yymsp[0].minor.yy105); } yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 379: /* alias_opt ::= */ + case 380: /* alias_opt ::= */ { yymsp[1].minor.yy105 = nil_token; } break; - case 380: /* alias_opt ::= table_alias */ + case 381: /* alias_opt ::= table_alias */ { yylhsminor.yy105 = yymsp[0].minor.yy105; } yymsp[0].minor.yy105 = yylhsminor.yy105; break; - case 381: /* alias_opt ::= AS table_alias */ + case 382: /* alias_opt ::= AS table_alias */ { yymsp[-1].minor.yy105 = yymsp[0].minor.yy105; } break; - case 382: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 383: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==383); + case 383: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 384: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==384); { yymsp[-2].minor.yy172 = yymsp[-1].minor.yy172; } break; - case 384: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + case 385: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ { yylhsminor.yy172 = createJoinTableNode(pCxt, yymsp[-4].minor.yy636, yymsp[-5].minor.yy172, yymsp[-2].minor.yy172, yymsp[0].minor.yy172); } yymsp[-5].minor.yy172 = yylhsminor.yy172; break; - case 385: /* join_type ::= */ + case 386: /* join_type ::= */ { yymsp[1].minor.yy636 = JOIN_TYPE_INNER; } break; - case 386: /* join_type ::= INNER */ + case 387: /* join_type ::= INNER */ { yymsp[0].minor.yy636 = JOIN_TYPE_INNER; } break; - case 387: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 388: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { yymsp[-8].minor.yy172 = createSelectStmt(pCxt, yymsp[-7].minor.yy617, yymsp[-6].minor.yy60, yymsp[-5].minor.yy172); yymsp[-8].minor.yy172 = addWhereClause(pCxt, yymsp[-8].minor.yy172, yymsp[-4].minor.yy172); @@ -4290,70 +4296,70 @@ static YYACTIONTYPE yy_reduce( yymsp[-8].minor.yy172 = addHavingClause(pCxt, yymsp[-8].minor.yy172, yymsp[0].minor.yy172); } break; - case 390: /* set_quantifier_opt ::= ALL */ + case 391: /* set_quantifier_opt ::= ALL */ { yymsp[0].minor.yy617 = false; } break; - case 391: /* select_list ::= NK_STAR */ + case 392: /* select_list ::= NK_STAR */ { yymsp[0].minor.yy60 = NULL; } break; - case 396: /* select_item ::= common_expression column_alias */ + case 397: /* select_item ::= common_expression column_alias */ { yylhsminor.yy172 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172), &yymsp[0].minor.yy105); } yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 397: /* select_item ::= common_expression AS column_alias */ + case 398: /* select_item ::= common_expression AS column_alias */ { yylhsminor.yy172 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), &yymsp[0].minor.yy105); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 402: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 419: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==419); - case 431: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==431); + case 403: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 420: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==420); + case 432: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==432); { yymsp[-2].minor.yy60 = yymsp[0].minor.yy60; } break; - case 404: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + case 405: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ { yymsp[-5].minor.yy172 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), releaseRawExprNode(pCxt, yymsp[-1].minor.yy172)); } break; - case 405: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ + case 406: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ { yymsp[-3].minor.yy172 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172)); } break; - case 406: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + case 407: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ { yymsp[-5].minor.yy172 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), NULL, yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } break; - case 407: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + case 408: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ { yymsp[-7].minor.yy172 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy172), releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } break; - case 409: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + case 410: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ { yymsp[-3].minor.yy172 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy172); } break; - case 411: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ + case 412: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ { yymsp[-3].minor.yy172 = createFillNode(pCxt, yymsp[-1].minor.yy202, NULL); } break; - case 412: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + case 413: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ { yymsp[-5].minor.yy172 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy60)); } break; - case 413: /* fill_mode ::= NONE */ + case 414: /* fill_mode ::= NONE */ { yymsp[0].minor.yy202 = FILL_MODE_NONE; } break; - case 414: /* fill_mode ::= PREV */ + case 415: /* fill_mode ::= PREV */ { yymsp[0].minor.yy202 = FILL_MODE_PREV; } break; - case 415: /* fill_mode ::= NULL */ + case 416: /* fill_mode ::= NULL */ { yymsp[0].minor.yy202 = FILL_MODE_NULL; } break; - case 416: /* fill_mode ::= LINEAR */ + case 417: /* fill_mode ::= LINEAR */ { yymsp[0].minor.yy202 = FILL_MODE_LINEAR; } break; - case 417: /* fill_mode ::= NEXT */ + case 418: /* fill_mode ::= NEXT */ { yymsp[0].minor.yy202 = FILL_MODE_NEXT; } break; - case 420: /* group_by_list ::= expression */ + case 421: /* group_by_list ::= expression */ { yylhsminor.yy60 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } yymsp[0].minor.yy60 = yylhsminor.yy60; break; - case 421: /* group_by_list ::= group_by_list NK_COMMA expression */ + case 422: /* group_by_list ::= group_by_list NK_COMMA expression */ { yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } yymsp[-2].minor.yy60 = yylhsminor.yy60; break; - case 424: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 425: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { yylhsminor.yy172 = addOrderByClause(pCxt, yymsp[-3].minor.yy172, yymsp[-2].minor.yy60); yylhsminor.yy172 = addSlimitClause(pCxt, yylhsminor.yy172, yymsp[-1].minor.yy172); @@ -4361,56 +4367,56 @@ static YYACTIONTYPE yy_reduce( } yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 426: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + case 427: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ { yylhsminor.yy172 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy172, yymsp[0].minor.yy172); } yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 427: /* query_expression_body ::= query_expression_body UNION query_expression_body */ + case 428: /* query_expression_body ::= query_expression_body UNION query_expression_body */ { yylhsminor.yy172 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy172, yymsp[0].minor.yy172); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 429: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ + case 430: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ { yymsp[-5].minor.yy172 = yymsp[-4].minor.yy172; } yy_destructor(yypParser,350,&yymsp[-3].minor); yy_destructor(yypParser,351,&yymsp[-2].minor); yy_destructor(yypParser,352,&yymsp[-1].minor); break; - case 433: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 437: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==437); + case 434: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 438: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==438); { yymsp[-1].minor.yy172 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 434: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 438: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==438); + case 435: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 439: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==439); { yymsp[-3].minor.yy172 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 435: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 439: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==439); + case 436: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 440: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==440); { yymsp[-3].minor.yy172 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 440: /* subquery ::= NK_LP query_expression NK_RP */ + case 441: /* subquery ::= NK_LP query_expression NK_RP */ { yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy172); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 444: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + case 445: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ { yylhsminor.yy172 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), yymsp[-1].minor.yy14, yymsp[0].minor.yy17); } yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 445: /* ordering_specification_opt ::= */ + case 446: /* ordering_specification_opt ::= */ { yymsp[1].minor.yy14 = ORDER_ASC; } break; - case 446: /* ordering_specification_opt ::= ASC */ + case 447: /* ordering_specification_opt ::= ASC */ { yymsp[0].minor.yy14 = ORDER_ASC; } break; - case 447: /* ordering_specification_opt ::= DESC */ + case 448: /* ordering_specification_opt ::= DESC */ { yymsp[0].minor.yy14 = ORDER_DESC; } break; - case 448: /* null_ordering_opt ::= */ + case 449: /* null_ordering_opt ::= */ { yymsp[1].minor.yy17 = NULL_ORDER_DEFAULT; } break; - case 449: /* null_ordering_opt ::= NULLS FIRST */ + case 450: /* null_ordering_opt ::= NULLS FIRST */ { yymsp[-1].minor.yy17 = NULL_ORDER_FIRST; } break; - case 450: /* null_ordering_opt ::= NULLS LAST */ + case 451: /* null_ordering_opt ::= NULLS LAST */ { yymsp[-1].minor.yy17 = NULL_ORDER_LAST; } break; default: diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 47424d3138..09322007ea 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -70,6 +70,12 @@ TEST_F(ParserSelectTest, pseudoColumn) { run("SELECT _WSTARTTS, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); } +TEST_F(ParserSelectTest, pseudoColumnSemanticCheck) { + useDb("root", "test"); + + run("SELECT TBNAME FROM (SELECT * FROM st1s1)"); +} + TEST_F(ParserSelectTest, multiResFunc) { useDb("root", "test"); diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 4640ed99bd..6a18a267e2 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -27,12 +27,13 @@ extern "C" { #define QUERY_POLICY_HYBRID 2 #define QUERY_POLICY_QNODE 3 -#define planFatal(param, ...) qFatal("PLAN: " param, __VA_ARGS__) -#define planError(param, ...) qError("PLAN: " param, __VA_ARGS__) -#define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__) -#define planInfo(param, ...) qInfo("PLAN: " param, __VA_ARGS__) -#define planDebug(param, ...) qDebug("PLAN: " param, __VA_ARGS__) -#define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) +#define planFatal(param, ...) qFatal("PLAN: " param, __VA_ARGS__) +#define planError(param, ...) qError("PLAN: " param, __VA_ARGS__) +#define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__) +#define planInfo(param, ...) qInfo("PLAN: " param, __VA_ARGS__) +#define planDebug(param, ...) qDebug("PLAN: " param, __VA_ARGS__) +#define planDebugL(param, ...) qDebugL("PLAN: " param, __VA_ARGS__) +#define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c2434e60f3..6c567fd4ab 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -310,12 +310,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, SLogicNode** pLogicNode) { - int32_t code = createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode); - if (TSDB_CODE_SUCCESS == code) { - SNode* pNode; - FOREACH(pNode, (*pLogicNode)->pTargets) { strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias); } - } - return code; + return createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode); } static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable, @@ -879,7 +874,8 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator } if (TSDB_CODE_SUCCESS == code) { - code = createColumnByProjections(pCxt, NULL, pSetOperator->pProjectionList, &pProject->node.pTargets); + code = createColumnByProjections(pCxt, pSetOperator->stmtName, pSetOperator->pProjectionList, + &pProject->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -933,7 +929,7 @@ static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetO code = createSetOpAggLogicNode(pCxt, pSetOperator, &pSetOp); break; default: - code = -1; + code = TSDB_CODE_FAILED; break; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e38c180ac6..8645225c04 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -598,39 +598,63 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { return false; } -static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { - if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { - return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); +static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond; + if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { + return false; + } + bool hasPrimaryKeyEqualCond = false; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (cpdContainPrimaryKeyEqualCond(pJoin, pCond)) { + hasPrimaryKeyEqualCond = true; + break; + } + } + return hasPrimaryKeyEqualCond; + } else { + return cpdIsPrimaryKeyEqualCond(pJoin, pCond); } - return TSDB_CODE_SUCCESS; } -static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) { - if (LOGIC_COND_TYPE_AND != pOnCond->condType) { - return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); - } - bool hasPrimaryKeyEqualCond = false; - SNode* pCond = NULL; - FOREACH(pCond, pOnCond->pParameterList) { - if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { - hasPrimaryKeyEqualCond = true; - } - } - if (!hasPrimaryKeyEqualCond) { - return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); - } - return TSDB_CODE_SUCCESS; -} +// static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { +// if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { +// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); +// } +// return TSDB_CODE_SUCCESS; +// } + +// static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) { +// if (LOGIC_COND_TYPE_AND != pOnCond->condType) { +// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); +// } +// bool hasPrimaryKeyEqualCond = false; +// SNode* pCond = NULL; +// FOREACH(pCond, pOnCond->pParameterList) { +// if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { +// hasPrimaryKeyEqualCond = true; +// } +// } +// if (!hasPrimaryKeyEqualCond) { +// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); +// } +// return TSDB_CODE_SUCCESS; +// } static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { - return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); - } else { - return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions); + if (!cpdContainPrimaryKeyEqualCond(pJoin, pJoin->pOnConditions)) { + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } + return TSDB_CODE_SUCCESS; + // if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { + // return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); + // } else { + // return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions); + // } } static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 54bc24e8bb..d0ac239bcb 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -204,6 +204,75 @@ static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { return code; } +static bool unionIsChildSubplan(SLogicNode* pLogicNode, int32_t groupId) { + if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pLogicNode)) { + return ((SExchangeLogicNode*)pLogicNode)->srcGroupId == groupId; + } + + SNode* pChild; + FOREACH(pChild, pLogicNode->pChildren) { + bool isChild = unionIsChildSubplan((SLogicNode*)pChild, groupId); + if (isChild) { + return isChild; + } + } + return false; +} + +static int32_t unionMountSubplan(SLogicSubplan* pParent, SNodeList* pChildren) { + SNode* pChild = NULL; + WHERE_EACH(pChild, pChildren) { + if (unionIsChildSubplan(pParent->pNode, ((SLogicSubplan*)pChild)->id.groupId)) { + int32_t code = nodesListMakeAppend(&pParent->pChildren, pChild); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(NULL); + ERASE_NODE(pChildren); + continue; + } else { + return code; + } + } + WHERE_NEXT; + } + return TSDB_CODE_SUCCESS; +} + +static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) { + SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return NULL; + } + pSubplan->id.groupId = pCxt->groupId; + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->pNode = pNode; + return pSubplan; +} + +static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubplan, SLogicNode* pSplitNode) { + SNodeList* pSubplanChildren = pUnionSubplan->pChildren; + pUnionSubplan->pChildren = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + + SNode* pChild = NULL; + FOREACH(pChild, pSplitNode->pChildren) { + SLogicSubplan* pNewSubplan = unionCreateSubplan(pCxt, (SLogicNode*)pChild); + code = nodesListMakeStrictAppend(&pUnionSubplan->pChildren, pNewSubplan); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(NULL); + code = unionMountSubplan(pNewSubplan, pSubplanChildren); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyList(pSubplanChildren); + DESTORY_LIST(pSplitNode->pChildren); + } + return code; +} + static SLogicNode* uaMatchByNode(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { return pNode; @@ -227,17 +296,6 @@ static bool uaFindSplitNode(SLogicSubplan* pSubplan, SUaInfo* pInfo) { return NULL != pSplitNode; } -static SLogicSubplan* uaCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) { - SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); - if (NULL == pSubplan) { - return NULL; - } - pSubplan->id.groupId = pCxt->groupId; - pSubplan->subplanType = SUBPLAN_TYPE_SCAN; - pSubplan->pNode = pNode; - return pSubplan; -} - static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { @@ -276,20 +334,8 @@ static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { return TSDB_CODE_SUCCESS; } - int32_t code = TSDB_CODE_SUCCESS; - - SNode* pChild = NULL; - FOREACH(pChild, info.pProject->node.pChildren) { - code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild)); - if (TSDB_CODE_SUCCESS == code) { - REPLACE_NODE(NULL); - } else { - break; - } - } + int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pProject); if (TSDB_CODE_SUCCESS == code) { - nodesClearList(info.pProject->node.pChildren); - info.pProject->node.pChildren = NULL; code = uaCreateExchangeNode(pCxt, info.pSubplan, info.pProject); } ++(pCxt->groupId); @@ -343,20 +389,8 @@ static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { return TSDB_CODE_SUCCESS; } - int32_t code = TSDB_CODE_SUCCESS; - - SNode* pChild = NULL; - FOREACH(pChild, info.pAgg->node.pChildren) { - code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild)); - if (TSDB_CODE_SUCCESS == code) { - REPLACE_NODE(NULL); - } else { - break; - } - } + int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pAgg); if (TSDB_CODE_SUCCESS == code) { - nodesClearList(info.pAgg->node.pChildren); - info.pAgg->node.pChildren = NULL; code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg); } ++(pCxt->groupId); diff --git a/source/libs/planner/test/planJoinTest.cpp b/source/libs/planner/test/planJoinTest.cpp index 714900c4e5..eaedbd1db0 100644 --- a/source/libs/planner/test/planJoinTest.cpp +++ b/source/libs/planner/test/planJoinTest.cpp @@ -30,6 +30,14 @@ TEST_F(PlanJoinTest, basic) { run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); } +TEST_F(PlanJoinTest, complex) { + useDb("root", "test"); + + run("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 " + "WHERE t1.ts = t2.ts AND t1.c1 BETWEEN -10 AND 10 AND t2.c1 BETWEEN -100 AND 100 AND " + "(t1.c2 LIKE 'nchar%' OR t1.c1 = 0 OR t2.c2 LIKE 'nchar%' OR t2.c1 = 0)"); +} + TEST_F(PlanJoinTest, withWhere) { useDb("root", "test"); diff --git a/source/libs/planner/test/planSetOpTest.cpp b/source/libs/planner/test/planSetOpTest.cpp index ba7fde3c77..3c7bf9e43a 100644 --- a/source/libs/planner/test/planSetOpTest.cpp +++ b/source/libs/planner/test/planSetOpTest.cpp @@ -23,11 +23,33 @@ class PlanSetOpTest : public PlannerTestBase {}; TEST_F(PlanSetOpTest, unionAll) { useDb("root", "test"); - run("select c1, c2 from t1 where c1 > 10 union all select c1, c2 from t1 where c1 > 20"); + run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20"); +} + +TEST_F(PlanSetOpTest, unionAllSubquery) { + useDb("root", "test"); + + run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION ALL SELECT c1, c2 FROM t1)"); } TEST_F(PlanSetOpTest, union) { useDb("root", "test"); - run("select c1, c2 from t1 where c1 > 10 union select c1, c2 from t1 where c1 > 20"); + run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION SELECT c1, c2 FROM t1 WHERE c1 > 20"); +} + +TEST_F(PlanSetOpTest, unionContainJoin) { + useDb("root", "test"); + + run("SELECT t1.c1 FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts " + "WHERE t1.c1 IS NOT NULL GROUP BY t1.c1 HAVING t1.c1 IS NOT NULL " + "UNION " + "SELECT t1.c1 FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts " + "WHERE t1.c1 IS NOT NULL GROUP BY t1.c1 HAVING t1.c1 IS NOT NULL"); +} + +TEST_F(PlanSetOpTest, unionSubquery) { + useDb("root", "test"); + + run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION SELECT c1, c2 FROM t1)"); } From 97a20ee86b93fb3e30e261cda1e43e3db16c11ad Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 18 May 2022 20:31:30 +0800 Subject: [PATCH 076/103] update meta based on sversion --- include/libs/catalog/catalog.h | 2 + source/client/src/clientImpl.c | 11 +++ source/common/src/tmsg.c | 16 ++-- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 + source/dnode/vnode/src/vnd/vnodeSvr.c | 3 + source/libs/catalog/src/catalog.c | 101 ++++++++++++++++++++- 6 files changed, 124 insertions(+), 11 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index a29dc3a90a..f9d8fc0de1 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -170,6 +170,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); */ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName); +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables); + /** * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5d3a139aeb..c3ccecb0d3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -361,6 +361,17 @@ _return: return code; } +void freeRequestRes(SRequestObj* pRequest, void* res) { + if (NULL == res) { + return; + } + + if (TDMT_VND_SUBMIT == pRequest->type) { + tFreeSSubmitRsp((SSubmitRsp*)res); + } else if (TDMT_VND_QUERY == pRequest->type) { + + } +} SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { void* pRes = NULL; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4e97ebbe47..278fc2c193 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4087,10 +4087,8 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1; if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; - if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; - } + if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; + if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1; @@ -4104,12 +4102,10 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - } + if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; + pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); + if (NULL == pBlock->tblFName) return -1; + if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 2fdbfdd206..d8426db127 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -320,6 +320,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } + strcat(pRsp->tblFName, mr.me.name); + if (mr.me.type == TSDB_NORMAL_TABLE) { sverNew = mr.me.ntbEntry.schema.sver; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 464331319e..371ade4f76 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -676,6 +676,9 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); tDecoderClear(&decoder); + } else { + submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN); + sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname); } if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a620b84f6d..23957d1a6b 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2883,8 +2883,107 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + +int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver) { + *sver = -1; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + int32_t tbType = 0; + uint64_t suid = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + if (tbMeta) { + tbType = tbMeta->tableType; + suid = tbMeta->suid; + if (tbType != TSDB_CHILD_TABLE) { + *sver = tbMeta->sversion; + } + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (tbType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; + } + + ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, suid); + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &suid, sizeof(suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("stb not in stbCache, suid:%"PRIx64, suid); + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, suid, (*stbMeta)->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *sver = (*stbMeta)->sversion; + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + SName name; + int32_t sver = 0; + int32_t tbNum = taosArrayGetSize(pTables); + for (int32_t i = 0; i < tbNum; ++i) { + STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + if (CTG_IS_SYS_DBNAME(name.dbname)) { + continue; + } + + ctgGetTbSverFromCache(pCtg, &name, &sver); + if (sver >= 0 && sver < pTb->sver) { + catalogRemoveTableMeta(pCtg, &name); //TODO REMOVE STB FROM CACHE + } + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); } From f94eaa6730c57dd061d841a2124c57ba80aceedf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 20:41:21 +0800 Subject: [PATCH 077/103] fix: only send message to one vnode in the vgroup --- source/dnode/mnode/impl/inc/mndVgroup.h | 2 +- source/dnode/mnode/impl/src/mndDb.c | 57 ++++++++----------------- source/dnode/mnode/impl/src/mndTrans.c | 3 +- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++-- 4 files changed, 23 insertions(+), 47 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 1e95859157..9bf7b6eb89 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -37,7 +37,7 @@ int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c3e5195ad2..0bf7b240b2 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,8 +261,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) { STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); @@ -279,48 +278,29 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; } return 0; } -static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { STransAction action = {0}; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_VNODE; - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; } return 0; @@ -487,7 +467,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid) != 0) { return -1; } } @@ -726,11 +706,8 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { - return -1; - } + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup) != 0) { + return -1; } } else { SVgObj newVgroup = {0}; @@ -744,9 +721,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; } else { mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); @@ -757,7 +734,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c9b68f27e7..f2b955a4d8 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -986,7 +986,8 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); if (tmsgSendReq(&pAction->epSet, &rpcMsg) == 0) { - mDebug("trans:%d, action:%d is sent", pTrans->id, action); + mDebug("trans:%d, action:%d is sent to %s:%u", pTrans->id, action, pAction->epSet.eps[pAction->epSet.inUse].fqdn, + pAction->epSet.eps[pAction->epSet.inUse].port); pAction->msgSent = 1; pAction->msgReceived = 0; pAction->errCode = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index f8c717edf6..62021c6a7e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -256,7 +256,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SAlterVnodeReq alterReq = {0}; alterReq.vgVersion = pVgroup->version; alterReq.buffer = pDb->cfg.buffer; @@ -285,16 +285,14 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgO pReplica->port = pVgidDnode->port; memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } } +#if 0 if (alterReq.selfIndex == -1) { terrno = TSDB_CODE_MND_APP_ERROR; return NULL; } +#endif int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); if (contLen < 0) { From 03d15c9e1f53f6d72377887a85a6531c61042b22 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 20:41:56 +0800 Subject: [PATCH 078/103] fix: variable initialization order --- source/dnode/mgmt/mgmt_bnode/src/bmWorker.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 2 +- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmProc.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 5 ++--- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index 14c6250b2a..e019924268 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -38,9 +38,9 @@ static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index be8ddf1430..599fa07e1c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -153,9 +153,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (code != 0 && terrno != 0) code = terrno; SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; rpcSendResponse(&rsp); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 98e234af3c..9b9960cf3a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -19,9 +19,9 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index c94cf527c7..fcfc4f4cee 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -19,9 +19,9 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 757f609731..d486707c5e 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -433,7 +433,7 @@ void dmCloseProcRpcHandles(SProc *proc) { SRpcHandleInfo *pInfo = taosHashIterate(proc->hash, NULL); while (pInfo != NULL) { dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, pInfo->handle); - SRpcMsg rpcMsg = {.info = *pInfo, .code = TSDB_CODE_NODE_OFFLINE}; + SRpcMsg rpcMsg = {.code = TSDB_CODE_NODE_OFFLINE, .info = *pInfo}; rpcSendResponse(&rpcMsg); pInfo = taosHashIterate(proc->hash, pInfo); } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f2b955a4d8..35ecaa748e 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -842,13 +842,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, - pTrans->rpcInfo.ahandle); + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code, pTrans->stage, pTrans->rpcInfo.ahandle); SRpcMsg rspMsg = { - .info = pTrans->rpcInfo, .code = code, .pCont = rpcCont, .contLen = pTrans->rpcRspLen, + .info = pTrans->rpcInfo, }; tmsgSendRsp(&rspMsg); pTrans->rpcInfo.handle = NULL; From 0189d667a68eb990edb4a7a0adbaefbe100ec3c6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 18 May 2022 20:47:01 +0800 Subject: [PATCH 079/103] fix: some problems of parser and planner --- source/libs/parser/test/parSelectTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 09322007ea..2da50babd6 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -73,7 +73,7 @@ TEST_F(ParserSelectTest, pseudoColumn) { TEST_F(ParserSelectTest, pseudoColumnSemanticCheck) { useDb("root", "test"); - run("SELECT TBNAME FROM (SELECT * FROM st1s1)"); + run("SELECT TBNAME FROM (SELECT * FROM st1s1)", TSDB_CODE_PAR_INVALID_TBNAME, PARSER_STAGE_TRANSLATE); } TEST_F(ParserSelectTest, multiResFunc) { From 8ede87094f772b519b149ae5a04484c49442191b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 21:02:26 +0800 Subject: [PATCH 080/103] fix: incorrect sync status judgment --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 ++-- source/dnode/mnode/impl/src/mnode.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7894eb5acb..bd026b9725 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -117,7 +117,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SMonMloadInfo minfo = {0}; dmGetMnodeLoads(pMgmt, &minfo); if (minfo.isMnode && minfo.load.syncState != TAOS_SYNC_STATE_LEADER && - minfo.load.syncState != TAOS_SYNC_STATE_CANDIDATE) { + minfo.load.syncState != TAOS_SYNC_STATE_FOLLOWER) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 04c0a93485..87e69f8360 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -181,7 +181,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { return -1; } - pObj->role = TAOS_SYNC_STATE_FOLLOWER; + pObj->role = TAOS_SYNC_STATE_CANDIDATE; return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 62021c6a7e..830551ad56 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -416,7 +416,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } else { - pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pVgid->role = TAOS_SYNC_STATE_CANDIDATE; } mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); @@ -514,7 +514,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; pVgid->dnodeId = pDnode->id; - pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pVgid->role = TAOS_SYNC_STATE_CANDIDATE; pDnode->numOfVnodes++; mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5326376bda..3b479d42a4 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -489,7 +489,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; } - if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) { + if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_FOLLOWER) { pClusterInfo->vnodes_alive++; } pClusterInfo->vnodes_total++; From a696ed583a8f4ea7a5dd4a270e5efba2dbcee900 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 18 May 2022 21:23:06 +0800 Subject: [PATCH 081/103] test(stream): _wstartts should be reverse quoted --- example/src/tmq.c | 4 +- source/dnode/mnode/impl/src/mndSubscribe.c | 12 +- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 3 + source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/executorMain.c | 85 ++++++------ source/libs/executor/src/executorimpl.c | 151 +++++++++++---------- source/libs/qworker/src/qworker.c | 104 +++++++------- tests/script/tsim/tstream/basic1.sim | 16 +-- 9 files changed, 195 insertions(+), 183 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index b79d21d051..00eb8462f4 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -106,8 +106,8 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column as abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index c853794e86..c82472eec0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -206,7 +206,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { int32_t totalVgNum = pOutput->pSub->vgNum; - mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); + mInfo("mq rebalance: subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -231,6 +231,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld", pVgEp->vgId, consumerId); } taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); // put into removed @@ -250,6 +251,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from unassigned", pVgEp->vgId); } } @@ -263,6 +265,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR minVgCnt = totalVgNum / afterRebConsumerNum; imbConsumerNum = totalVgNum % afterRebConsumerNum; } + mInfo("mq rebalance: %d consumer after rebalance, at least %d vg each, %d consumer has more vg", afterRebConsumerNum, + minVgCnt, imbConsumerNum); // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; @@ -290,6 +294,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; } @@ -303,6 +308,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); } } } @@ -319,6 +325,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); + mInfo("mq rebalance: add new consumer %ld", consumerId); } } @@ -343,6 +350,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); + mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } @@ -360,6 +368,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); + mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } else { // if all consumer is removed, put all vg into unassigned @@ -372,6 +381,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pRebOutput->newConsumerId == -1); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); + mInfo("mq rebalance: unassign vg %d (second scan)", pRebOutput->pVgEp->vgId); } } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 38dedee5a2..d0420e1b84 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -179,6 +179,7 @@ struct STQ { SHashObj* pStreamTasks; SVnode* pVnode; SWal* pWal; + // TDB* pTdb; }; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 80aa350e4d..c69f7d71d5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -32,6 +32,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; + /*if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) {*/ + /*ASSERT(0);*/ + /*}*/ #if 0 pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 66073b70eb..35b81b135f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -106,7 +106,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { pMsg->contentLen = pMsg->contentLen; #endif - qDebugL("stream task string %s", (const char*)msg); + /*qDebugL("stream task string %s", (const char*)msg);*/ struct SSubplan* plan = NULL; int32_t code = qStringToSubplan(msg, &plan); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 354f4d8752..e2af6cb73b 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -15,10 +15,10 @@ #include #include "dataSinkMgt.h" -#include "texception.h" #include "os.h" #include "tarray.h" #include "tcache.h" +#include "texception.h" #include "tglobal.h" #include "tmsg.h" #include "tudf.h" @@ -32,15 +32,15 @@ typedef struct STaskMgmt { TdThreadMutex lock; - SCacheObj *qinfoPool; // query handle pool - int32_t vgId; - bool closed; + SCacheObj *qinfoPool; // query handle pool + int32_t vgId; + bool closed; } STaskMgmt; -int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model) { +int32_t qCreateExecTask(SReadHandle *readHandle, int32_t vgId, uint64_t taskId, SSubplan *pSubplan, + qTaskInfo_t *pTaskInfo, DataSinkHandle *handle, EOPTR_EXEC_MODEL model) { assert(readHandle != NULL && pSubplan != NULL); - SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; + SExecTaskInfo **pTask = (SExecTaskInfo **)pTaskInfo; int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, model); if (code != TSDB_CODE_SUCCESS) { @@ -56,46 +56,46 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, code = dsCreateDataSinker(pSubplan->pDataSink, handle); } - _error: +_error: // if failed to add ref for all tables in this query, abort current query return code; } #ifdef TEST_IMPL // wait moment -int waitMoment(SQInfo* pQInfo){ - if(pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if(pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if(pos){ +int waitMoment(SQInfo *pQInfo) { + if (pQInfo->sql) { + int ms = 0; + char *pcnt = strstr(pQInfo->sql, " count(*)"); + if (pcnt) return 0; + + char *pos = strstr(pQInfo->sql, " t_"); + if (pos) { pos += 3; ms = atoi(pos); - while(*pos >= '0' && *pos <= '9'){ - pos ++; + while (*pos >= '0' && *pos <= '9') { + pos++; } char unit_char = *pos; - if(unit_char == 'h'){ - ms *= 3600*1000; - } else if(unit_char == 'm'){ - ms *= 60*1000; - } else if(unit_char == 's'){ + if (unit_char == 'h') { + ms *= 3600 * 1000; + } else if (unit_char == 'm') { + ms *= 60 * 1000; + } else if (unit_char == 's') { ms *= 1000; } } - if(ms == 0) return 0; + if (ms == 0) return 0; printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if(ms < 1000) { + + if (ms < 1000) { taosMsleep(ms); } else { int used_ms = 0; - while(used_ms < ms) { + while (used_ms < ms) { taosMsleep(1000); used_ms += 1000; - if(isTaskKilled(pQInfo)){ + if (isTaskKilled(pQInfo)) { printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); break; } @@ -106,15 +106,14 @@ int waitMoment(SQInfo* pQInfo){ } #endif -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock **pRes, uint64_t *useconds) { + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)tinfo; int64_t threadId = taosGetSelfPthreadId(); *pRes = NULL; int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { - qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, - (void*)curOwner); + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void *)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; return pTaskInfo->code; } @@ -133,12 +132,11 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; - qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), - tstrerror(pTaskInfo->code)); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); return pTaskInfo->code; } - qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + /*qDebug("%s execTask is launched", GET_TASKID(pTaskInfo));*/ publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); @@ -154,12 +152,12 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { *useconds = pTaskInfo->cost.elapsedTime; } - int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0; + int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; pTaskInfo->totalRows += current; cleanUpUdfs(); - qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0); + /*qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",*/ + /*GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0);*/ atomic_store_64(&pTaskInfo->owner, 0); return pTaskInfo->code; @@ -208,18 +206,17 @@ int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { } void qDestroyTask(qTaskInfo_t qTaskHandle) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) qTaskHandle; - qDebug("%s execTask completed, numOfRows:%"PRId64, GET_TASKID(pTaskInfo), pTaskInfo->totalRows); + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qTaskHandle; + qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->totalRows); - queryCostStatis(pTaskInfo); // print the query cost summary + queryCostStatis(pTaskInfo); // print the query cost summary doDestroyTask(pTaskInfo); } int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes) { SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)tinfo; - int32_t capacity = 0; + int32_t capacity = 0; - return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); + return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); } - diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d0a1840d72..b8f06ebc67 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -106,7 +106,7 @@ static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo); static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); -static void releaseQueryBuf(size_t numOfTables); +static void releaseQueryBuf(size_t numOfTables); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); @@ -154,8 +154,9 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs); +static int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, + SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t* rowCellOffset, + SqlFunctionCtx* pCtx, int32_t numOfExprs); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); @@ -342,14 +343,12 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return pResultRow; } -void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, - uint64_t groupId, int32_t numOfOutput) { +void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, uint64_t groupId, int32_t numOfOutput) { SAggSupporter* pSup = &pInfo->aggSup; SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(bytes)); - SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, pInfo->binfo.rowCellInfoOffset); @@ -599,8 +598,9 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, + SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, + int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { // keep it temporarily bool hasAgg = pCtx[k].input.colDataAggIsSet; @@ -683,8 +683,8 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, - bool createDummyCol) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + int32_t scanFlag, bool createDummyCol) { if (pBlock->pBlockAgg != NULL) { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } else { @@ -735,7 +735,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, - int32_t scanFlag, bool createDummyCol) { + int32_t scanFlag, bool createDummyCol) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pOperator->numOfExprs; ++i) { @@ -743,7 +743,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pCtx[i].input.numOfRows = pBlock->info.rows; pCtx[i].pSrcBlock = pBlock; - pCtx[i].scanFlag = scanFlag; + pCtx[i].scanFlag = scanFlag; SInputColumnInfoData* pInput = &pCtx[i].input; pInput->uid = pBlock->info.uid; @@ -1003,14 +1003,14 @@ static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return false; } -// if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { -// // return QUERY_IS_ASC_QUERY(pQueryAttr); -// } -// -// // denote the order type -// if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { -// // return pCtx->param[0].i == pQueryAttr->order.order; -// } + // if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { + // // return QUERY_IS_ASC_QUERY(pQueryAttr); + // } + // + // // denote the order type + // if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { + // // return pCtx->param[0].i == pQueryAttr->order.order; + // } // in the reverse table scan, only the following functions need to be executed // if (IS_REVERSE_SCAN(pRuntimeEnv) || @@ -1128,16 +1128,16 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu } else if (fmIsAggFunc(pCtx[i].functionId)) { p = &pCtx[i]; } -// if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { -// tagLen += pCtx[i].resDataInfo.bytes; -// pTagCtx[num++] = &pCtx[i]; -// } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { -// // tag function may be the group by tag column -// // ts may be the required primary timestamp column -// continue; -// } else { -// // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ -// } + // if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + // tagLen += pCtx[i].resDataInfo.bytes; + // pTagCtx[num++] = &pCtx[i]; + // } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { + // // tag function may be the group by tag column + // // ts may be the required primary timestamp column + // continue; + // } else { + // // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ + // } } if (p != NULL) { @@ -2015,7 +2015,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { if (pFilterNode == NULL) { return; } @@ -2133,8 +2133,9 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p * @param pQInfo * @param result */ -int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { +int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, int32_t* rowCellOffset, SqlFunctionCtx* pCtx, + int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t start = pGroupResInfo->index; @@ -2172,11 +2173,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn } else { // expand the result into multiple rows. E.g., _wstartts, top(k, 20) // the _wstartts needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); - for(int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); - } + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } } } @@ -2192,11 +2193,12 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn return 0; } -void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf) { +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); - SExprInfo* pExprInfo = pOperator->pExpr; - int32_t numOfExprs = pOperator->numOfExprs; + SExprInfo* pExprInfo = pOperator->pExpr; + int32_t numOfExprs = pOperator->numOfExprs; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; @@ -3215,7 +3217,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { return TSDB_CODE_SUCCESS; } -SOperatorInfo* createExchangeOperatorInfo(void *pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -3328,7 +3330,7 @@ static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int3 static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExpr, int32_t rowIndex) { for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index -// pCtx[j].startRow = rowIndex; + // pCtx[j].startRow = rowIndex; } for (int32_t j = 0; j < numOfExpr; ++j) { @@ -3379,7 +3381,7 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { -// pCtx[i].size = 1; + // pCtx[i].size = 1; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -3605,7 +3607,7 @@ _error: return NULL; } -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag) { +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { // todo add more information about exchange operation if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { *order = TSDB_ORDER_ASC; @@ -3635,7 +3637,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - SOperatorInfo* downstream = pOperator->pDownstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; @@ -3975,7 +3977,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, pProjectInfo->pPseudoColInfo); + code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs, + pProjectInfo->pPseudoColInfo); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -4225,7 +4228,7 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pGroupList); ++i) { SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i); for (int32_t j = 0; j < taosArrayGetSize(pa); ++j) { - STableKeyInfo* pk = taosArrayGet(pa, j); + STableKeyInfo* pk = taosArrayGet(pa, j); STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++]; pTQueryInfo->lastKey = pk->lastKey; } @@ -4361,9 +4364,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; @@ -4503,10 +4506,10 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa } pCol->slotId = slotId; - pCol->colId = colId; - pCol->bytes = pType->bytes; - pCol->type = pType->type; - pCol->scale = pType->scale; + pCol->colId = colId; + pCol->bytes = pType->bytes; + pCol->type = pType->type; + pCol->scale = pType->scale; pCol->precision = pType->precision; pCol->dataBlockId = blockId; @@ -4581,10 +4584,10 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* if (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0) { pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); - SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); - if (NULL == res) { // todo handle error + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { // todo handle error } else { - res->node.resType = (SDataType) {.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; nodesListAppend(pFuncNode->pParameterList, res); } } @@ -4677,7 +4680,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. + SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t numOfCols = 0; @@ -4686,27 +4689,27 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); } else { - doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); } if (pDataReader == NULL && terrno != 0) { - qDebug("pDataReader is NULL"); + /*qDebug("pDataReader is NULL");*/ // return NULL; } else { - qDebug("pDataReader is not NULL"); + /*qDebug("pDataReader is not NULL");*/ } SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pDescNode); SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, tableIdList, pTaskInfo, - pScanPhyNode->node.pConditions, pOperatorDumy); + SOperatorInfo* pOperator = + createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols, + tableIdList, pTaskInfo, pScanPhyNode->node.pConditions, pOperatorDumy); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4855,7 +4858,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; - SColumn col = extractColumnFromColumnNode(pColNode); + SColumn col = extractColumnFromColumnNode(pColNode); pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; @@ -4923,11 +4926,11 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.scale = pColNode->node.resType.scale; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; c.precision = pColNode->node.resType.precision; return c; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index f48095ecb8..db63c71d11 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -9,11 +9,11 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; SQWorkerMgmt gQwMgmt = { - .lock = 0, - .qwRef = -1, - .qwNum = 0, + .lock = 0, + .qwRef = -1, + .qwNum = 0, }; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { @@ -110,9 +110,9 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) { QW_LOCK(QW_READ, &mgmt->schLock); - QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash)); + /*QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash));*/ - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; SQWSchStatus *sch = NULL; @@ -127,7 +127,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) { QW_UNLOCK(QW_READ, &mgmt->schLock); - QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash)); + /*QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash));*/ } char *qwPhaseStr(int32_t phase) { @@ -462,7 +462,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { } int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -499,7 +499,7 @@ _return: } int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -550,11 +550,11 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t code = 0; bool qcontinue = true; - SSDataBlock * pRes = NULL; + SSDataBlock *pRes = NULL; uint64_t useconds = 0; int32_t i = 0; int32_t execNum = 0; - qTaskInfo_t * taskHandle = &ctx->taskHandle; + qTaskInfo_t *taskHandle = &ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; while (true) { @@ -632,7 +632,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) return TSDB_CODE_QRY_OUT_OF_MEMORY; } - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; STaskStatus status = {0}; @@ -719,8 +719,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void } int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { - int32_t code = 0; - SQWTaskCtx * ctx = NULL; + int32_t code = 0; + SQWTaskCtx *ctx = NULL; SRpcHandleInfo *dropConnection = NULL; SRpcHandleInfo *cancelConnection = NULL; @@ -925,13 +925,13 @@ _return: } int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { - int32_t code = 0; - bool queryRsped = false; - SSubplan* plan = NULL; - SQWPhaseInput input = {0}; - qTaskInfo_t pTaskInfo = NULL; - DataSinkHandle sinkHandle = NULL; - SQWTaskCtx * ctx = NULL; + int32_t code = 0; + bool queryRsped = false; + SSubplan *plan = NULL; + SQWPhaseInput input = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + SQWTaskCtx *ctx = NULL; QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); @@ -944,7 +944,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex ctx->ctrlConnInfo = qwMsg->connInfo; - QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); + /*QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);*/ code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { @@ -1055,10 +1055,10 @@ _return: } int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - SQWTaskCtx * ctx = NULL; + SQWTaskCtx *ctx = NULL; int32_t code = 0; SQWPhaseInput input = {0}; - void * rsp = NULL; + void *rsp = NULL; int32_t dataLen = 0; bool queryEnd = false; @@ -1138,8 +1138,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; int32_t dataLen = 0; bool locked = false; - SQWTaskCtx * ctx = NULL; - void * rsp = NULL; + SQWTaskCtx *ctx = NULL; + void *rsp = NULL; SQWPhaseInput input = {0}; QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); @@ -1274,7 +1274,7 @@ _return: int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); @@ -1300,7 +1300,7 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; if (qwMsg->code) { QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); @@ -1338,28 +1338,28 @@ _return: qwMsg->connInfo.handle = NULL; } - QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); + /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));*/ QW_RET(TSDB_CODE_SUCCESS); } void qwProcessHbTimerEvent(void *param, void *tmrId) { - SQWHbParam* hbParam = (SQWHbParam*)param; + SQWHbParam *hbParam = (SQWHbParam *)param; if (hbParam->qwrId != atomic_load_32(&gQwMgmt.qwRef)) { return; } - int64_t refId = hbParam->refId; + int64_t refId = hbParam->refId; SQWorker *mgmt = qwAcquire(refId); if (NULL == mgmt) { QW_DLOG("qwAcquire %" PRIx64 "failed", refId); taosMemoryFree(param); return; } - + SQWSchStatus *sch = NULL; int32_t taskNum = 0; - SQWHbInfo * rspList = NULL; + SQWHbInfo *rspList = NULL; int32_t code = 0; qwDbgDumpMgmtInfo(mgmt); @@ -1383,7 +1383,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; @@ -1413,29 +1413,27 @@ _return: for (int32_t j = 0; j < i; ++j) { qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); - QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), - (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); + /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/ + /*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/ tFreeSSchedulerHbRsp(&rspList[j].rsp); } taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); - qwRelease(refId); + qwRelease(refId); } void qwCloseRef(void) { taosWLockLatch(&gQwMgmt.lock); if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { taosCloseRef(gQwMgmt.qwRef); - gQwMgmt.qwRef= -1; + gQwMgmt.qwRef = -1; } taosWUnLockLatch(&gQwMgmt.lock); } -void qwDestroySchStatus(SQWSchStatus *pStatus) { - taosHashCleanup(pStatus->tasksHash); -} +void qwDestroySchStatus(SQWSchStatus *pStatus) { taosHashCleanup(pStatus->tasksHash); } void qwDestroyImpl(void *pMgmt) { SQWorker *mgmt = (SQWorker *)pMgmt; @@ -1454,12 +1452,12 @@ void qwDestroyImpl(void *pMgmt) { SQWSchStatus *sch = (SQWSchStatus *)pIter; qwDestroySchStatus(sch); pIter = taosHashIterate(mgmt->schHash, pIter); - } + } taosHashCleanup(mgmt->schHash); taosMemoryFree(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); qwCloseRef(); } @@ -1467,7 +1465,7 @@ void qwDestroyImpl(void *pMgmt) { int32_t qwOpenRef(void) { taosWLockLatch(&gQwMgmt.lock); if (gQwMgmt.qwRef < 0) { - gQwMgmt.qwRef= taosOpenRef(100, qwDestroyImpl); + gQwMgmt.qwRef = taosOpenRef(100, qwDestroyImpl); if (gQwMgmt.qwRef < 0) { taosWUnLockLatch(&gQwMgmt.lock); qError("init qworker ref failed"); @@ -1475,14 +1473,14 @@ int32_t qwOpenRef(void) { } } taosWUnLockLatch(&gQwMgmt.lock); - + return TSDB_CODE_SUCCESS; } void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { int32_t paramIdx = 0; int32_t newParamIdx = 0; - + while (true) { paramIdx = atomic_load_32(&gQwMgmt.paramIdx); if (paramIdx == tListLen(gQwMgmt.param)) { @@ -1490,7 +1488,7 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { } else { newParamIdx = paramIdx + 1; } - + if (paramIdx == atomic_val_compare_exchange_32(&gQwMgmt.paramIdx, paramIdx, newParamIdx)) { break; } @@ -1577,12 +1575,12 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW SQWHbParam *param = NULL; qwSetHbParam(mgmt->refId, ¶m); - mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void*)param, mgmt->timer); + mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer); if (NULL == mgmt->hbTimer) { qError("start hb timer failed"); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + *qWorkerMgmt = mgmt; qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); @@ -1599,9 +1597,9 @@ _return: taosTmrCleanUp(mgmt->timer); taosMemoryFreeClear(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); } - + QW_RET(code); } @@ -1678,7 +1676,7 @@ int32_t qwUpdateSchLastAccess(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64 } int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -1705,7 +1703,7 @@ int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId } int32_t qwCancelTask(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim index cb084ad537..9965772c75 100644 --- a/tests/script/tsim/tstream/basic1.sim +++ b/tests/script/tsim/tstream/basic1.sim @@ -24,7 +24,7 @@ sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); sleep 1000 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then print ======$rows @@ -137,7 +137,7 @@ endi sql insert into t1 values(1648791223001,12,14,13,11.1); sleep 500 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then print ======$rows @@ -250,7 +250,7 @@ endi sql insert into t1 values(1648791223002,12,14,13,11.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 2 then @@ -280,7 +280,7 @@ endi sql insert into t1 values(1648791223003,12,14,13,11.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 3 then @@ -312,7 +312,7 @@ sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 3 then @@ -344,7 +344,7 @@ sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 2 if $data21 != 2 then @@ -374,7 +374,7 @@ endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 0 if $data01 != 4 then @@ -404,7 +404,7 @@ endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 4 then From 8fca7d9a5107dd4161232762748aecac969e0864 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 21:25:21 +0800 Subject: [PATCH 082/103] fix: incorrect sync status --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 5 ++--- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 ++-- source/dnode/mnode/impl/src/mnode.c | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index bd026b9725..38e71754b6 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -116,8 +116,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; dmGetMnodeLoads(pMgmt, &minfo); - if (minfo.isMnode && minfo.load.syncState != TAOS_SYNC_STATE_LEADER && - minfo.load.syncState != TAOS_SYNC_STATE_FOLLOWER) { + if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; @@ -127,7 +126,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { dmGetVnodeLoads(pMgmt, &vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); - if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { + if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId, syncStr(pLoad->syncState)); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 87e69f8360..04c0a93485 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -181,7 +181,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { return -1; } - pObj->role = TAOS_SYNC_STATE_CANDIDATE; + pObj->role = TAOS_SYNC_STATE_FOLLOWER; return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 830551ad56..62021c6a7e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -416,7 +416,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } else { - pVgid->role = TAOS_SYNC_STATE_CANDIDATE; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); @@ -514,7 +514,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; pVgid->dnodeId = pDnode->id; - pVgid->role = TAOS_SYNC_STATE_CANDIDATE; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; pDnode->numOfVnodes++; mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3b479d42a4..8c805dd8c7 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -489,7 +489,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; } - if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_FOLLOWER) { + if (pVgid->role != TAOS_SYNC_STATE_ERROR) { pClusterInfo->vnodes_alive++; } pClusterInfo->vnodes_total++; From 3502a58cd526bbafe890dd9f07677fa17e264a6c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 18 May 2022 21:49:28 +0800 Subject: [PATCH 083/103] fix: bad log print flags --- include/libs/qcom/query.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 711db65e97..daf008108b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -220,23 +220,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ } \ } while (0) -#define qDebug(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebug(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qTrace(...) \ - do { \ - if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qTrace(...) \ + do { \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qDebugL(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebugL(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) #define QRY_ERR_RET(c) \ From 534dfae90769d495f172ca65d79ce6536e04d5dc Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 18 May 2022 22:06:59 +0800 Subject: [PATCH 084/103] [test: set days to 300 for default db] --- tests/pytest/util/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index adbab04e07..bdda7c453b 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -57,7 +57,7 @@ class TDSql: tdLog.notice("'reset query cache' is not supported") s = 'drop database if exists db' self.cursor.execute(s) - s = 'create database db' + s = 'create database db days 300' self.cursor.execute(s) s = 'use db' self.cursor.execute(s) From f41caed14aeec9dc72a0d005390802411c44f587 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 18 May 2022 22:52:20 +0800 Subject: [PATCH 085/103] fix: merge bitmap when data col have data --- source/common/src/trow.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d1516403c1..22bdd960ea 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1063,7 +1063,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { if (isAllRowsNone(pCol)) { - pVal->valType = TD_VTYPE_NULL; + pVal->valType = TD_VTYPE_NONE; #ifdef TD_SUPPORT_READ2 pVal->val = (void *)getNullValue(pCol->type); #else diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index bebdfb3b63..c1a1e7570e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -330,12 +330,12 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } - if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - if (pDataCol->bitmap) { + if (pDataCol->len > 0 && pDataCol->bitmap) { ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(pDataCol->pBitmap); tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdDataColsSetBitmapI(pReadh->pDCols[0]); } From 54420f06fc13983e3c104a0cd0c25d216241e514 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 19 May 2022 09:23:15 +0800 Subject: [PATCH 086/103] feat: put dup record in the last --- include/util/tskiplist.h | 2 +- source/util/src/tskiplist.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index eeae1b47da..10d3dcdbaa 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -56,10 +56,10 @@ typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutS typedef struct SSkipList { uint32_t seed; + uint16_t len; __compar_fn_t comparFn; __sl_key_fn_t keyFn; TdThreadRwlock *lock; - uint16_t len; uint8_t maxLevel; uint8_t flags; uint8_t type; // static info above diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 13637b8fe4..4ce668e6ba 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -185,10 +185,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it pKey = SL_GET_NODE_KEY(pSkipList, p); compare = pSkipList->comparFn(pKey, pDataKey); - if (compare >= 0) { - if (compare == 0 && !hasDup) hasDup = true; + if (compare > 0) { break; } else { + if (compare == 0 && !hasDup) hasDup = true; px = p; p = SL_NODE_GET_FORWARD_POINTER(px, i); } From fb944e85bc247378fb92eb67ba3237e5a93ec7a8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 19 May 2022 09:39:28 +0800 Subject: [PATCH 087/103] fix explain res issue --- include/libs/scheduler/scheduler.h | 2 +- source/client/src/clientImpl.c | 2 +- source/libs/scheduler/inc/schedulerInt.h | 8 +++- source/libs/scheduler/src/scheduler.c | 46 +++++++++---------- source/libs/scheduler/test/schedulerTests.cpp | 2 +- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index b3f35025d1..dcd058a293 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, bool needRes, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3ccecb0d3..7693d26d3f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -291,7 +291,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, NULL != pRes, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index a90fb7fc2e..5a6fcee759 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -39,6 +39,12 @@ enum { SCH_WRITE, }; +typedef enum { + SCH_RES_TYPE_QUERY, + SCH_RES_TYPE_FETCH, +} SCH_RES_TYPE; + + typedef struct SSchTrans { void *transInst; void *transHandle; @@ -159,7 +165,6 @@ typedef struct SSchTask { typedef struct SSchJobAttr { EExplainMode explainMode; - bool needRes; bool syncSchedule; bool queryJob; bool needFlowCtrl; @@ -192,6 +197,7 @@ typedef struct SSchJob { int32_t errCode; SArray *errList; // SArray SRWLatch resLock; + SCH_RES_TYPE resType; void *resData; //TODO free it or not int32_t resNumOfRows; const char *sql; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index bdf674ab93..9354c1a875 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -70,7 +70,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * } int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { int32_t code = 0; int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); @@ -81,7 +81,6 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; - pJob->attr.needRes = needRes; pJob->transport = transport; pJob->sql = sql; @@ -1059,6 +1058,8 @@ _return: int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + pJob->resType = SCH_RES_TYPE_FETCH; + atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); atomic_store_ptr(&pJob->resData, pRsp); @@ -1179,23 +1180,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); - if (pJob->attr.needRes) { - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->resData) { - SSubmitRsp *sum = pJob->resData; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); - taosMemoryFree(rsp->pBlocks); - taosMemoryFree(rsp); - } else { - pJob->resData = rsp; - } - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + pJob->resType = SCH_RES_TYPE_QUERY; + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->resData) { + SSubmitRsp *sum = pJob->resData; + sum->affectedRows += rsp->affectedRows; + sum->nBlocks += rsp->nBlocks; + sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); + memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + taosMemoryFree(rsp->pBlocks); + taosMemoryFree(rsp); } else { - tFreeSSubmitRsp(rsp); + pJob->resData = rsp; } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); @@ -2412,7 +2410,7 @@ void schFreeJobImpl(void *job) { } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); if (pNodeList == NULL || taosArrayGetSize(pNodeList) <= 0) { @@ -2421,7 +2419,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD int32_t code = 0; SSchJob *pJob = NULL; - SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, needRes, syncSchedule)); + SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, syncSchedule)); SCH_ERR_JRET(schLaunchJob(pJob)); @@ -2463,6 +2461,8 @@ int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDa SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData)); + pJob->resType = SCH_RES_TYPE_FETCH; + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); @@ -2535,7 +2535,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - int64_t startTs, bool needRes, SQueryResult *pRes) { + int64_t startTs, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -2543,14 +2543,14 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, needRes, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true)); } SSchJob *job = schAcquireJob(*pJob); pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; - if (needRes) { + if (SCH_RES_TYPE_QUERY == job->resType) { pRes->res = job->resData; job->resData = NULL; } @@ -2568,7 +2568,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pD if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, pNodeList, pDag, pJob, sql, false)); } else { - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 09ecd9fffd..fc0e05aaf1 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -985,7 +985,7 @@ TEST(insertTest, normalCase) { taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, false, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); From 23e415b9272d1c544fa09a06f456a0de2199d066 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 19 May 2022 09:41:08 +0800 Subject: [PATCH 088/103] test:modify days to 300 --- tests/script/tsim/query/interval-offset.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 68860dc2cb..dcd88e5a0c 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -5,7 +5,7 @@ sleep 500 sql connect print =============== create database -sql create database d0 +sql create database d0 days 300 sql use d0 print =============== create super table and child table @@ -254,4 +254,4 @@ endi #sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 2216438c72c3ec950f1b4303b50c5a26de861524 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 May 2022 10:26:21 +0800 Subject: [PATCH 089/103] fix(stream): skip update tb uid for db topic --- example/src/tmq.c | 4 ++-- source/dnode/vnode/src/tq/tq.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 00eb8462f4..b79d21d051 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -106,8 +106,8 @@ int32_t create_topic() { } taos_free_result(pRes); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + pRes = taos_query(pConn, "create topic topic_ctb_column as abc1"); + /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c69f7d71d5..30de2dcfa0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -111,6 +111,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) { pIter = taosHashIterate(pTq->execs, pIter); if (pIter == NULL) break; pExec = (STqExec*)pIter; + if (pExec->subType == TOPIC_SUB_TYPE__DB) continue; for (int32_t i = 0; i < 5; i++) { int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true); ASSERT(code == 0); From 51721dfc477a8c300559300649e2ed11bafd9928 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 10:37:11 +0800 Subject: [PATCH 090/103] fix(os): win run error --- cmake/cmake.install | 21 ++++--- include/os/osString.h | 18 ++++++ packaging/tools/make_install.bat | 6 ++ source/client/src/clientSml.c | 4 +- source/common/src/tname.c | 2 +- source/common/src/ttime.c | 4 +- source/common/src/tvariant.c | 8 +-- source/libs/index/src/indexComm.c | 8 +-- source/libs/parser/src/parAstCreater.c | 56 +++++++++--------- source/libs/parser/src/parInsert.c | 6 +- source/libs/parser/src/parTranslater.c | 22 +++---- source/libs/parser/src/sql.c | 2 +- source/libs/scalar/src/sclfunc.c | 8 +-- source/libs/scalar/src/sclvector.c | 22 +++---- source/os/src/osString.c | 79 ++++++++++++++++++++++++++ source/util/src/tjson.c | 4 +- tools/shell/src/shellEngine.c | 6 +- 17 files changed, 192 insertions(+), 84 deletions(-) create mode 100644 packaging/tools/make_install.bat diff --git a/cmake/cmake.install b/cmake/cmake.install index f51c6566fa..b2421fac25 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -5,22 +5,27 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(CMAKE_INSTALL_PREFIX C:/TDengine) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taos.h DESTINATION include) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taoserror.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) IF (TD_MVN_INSTALLED) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) ENDIF () + SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") + INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/include/os/osString.h b/include/os/osString.h index 5f65f97bec..1b518f9b81 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -38,6 +38,13 @@ typedef int32_t TdUcs4; #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #define strcasestr STR_CASE_STR_FORBID + #define strtoll STR_TO_LL_FUNC_TAOS_FORBID + #define strtoull STR_TO_ULL_FUNC_TAOS_FORBID + #define strtol STR_TO_L_FUNC_TAOS_FORBID + #define strtoul STR_TO_UL_FUNC_TAOS_FORBID + #define strtod STR_TO_LD_FUNC_TAOS_FORBID + #define strtold STR_TO_D_FUNC_TAOS_FORBID + #define strtof STR_TO_F_FUNC_TAOS_FORBID #endif #ifdef WINDOWS @@ -72,6 +79,17 @@ int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); char *taosStrCaseStr(const char *str, const char *pattern); +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix); +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix); +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix); +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix); +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix); +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix); +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix); +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix); +double taosStr2Double(const char *str, char** pEnd); +float taosStr2Float(const char *str, char** pEnd); + #ifdef __cplusplus } #endif diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat new file mode 100644 index 0000000000..64f30b8465 --- /dev/null +++ b/packaging/tools/make_install.bat @@ -0,0 +1,6 @@ +@echo off +goto %1 +:needAdmin +mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&goto :eof +:hasAdmin +cp -f C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 \ No newline at end of file diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f6f8db8590..d5377c99a6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -580,7 +580,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; - double result = strtod(pVal, &endptr); + double result = taosStr2Double(pVal, &endptr); if(pVal == endptr){ smlBuildInvalidDataMsg(msg, "invalid data", pVal); return false; @@ -714,7 +714,7 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { char *endPtr = NULL; - int64_t tsInt64 = strtoll(value, &endPtr, 10); + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); if(value + len != endPtr){ return -1; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fa9b6e1e63..0764ea84b9 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -250,7 +250,7 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return -1; } - dst->acctId = strtoll(str, NULL, 10); + dst->acctId = taosStr2Int32(str, NULL, 10); } if ((type & T_NAME_DB) == T_NAME_DB) { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 62a7179626..69ba964187 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -590,7 +590,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura char* endPtr = NULL; /* get the basic numeric value */ - int64_t timestamp = strtoll(token, &endPtr, 10); + int64_t timestamp = taosStr2Int64(token, &endPtr, 10); if (errno != 0) { return -1; } @@ -608,7 +608,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati errno = 0; /* get the basic numeric value */ - *duration = strtoll(token, NULL, 10); + *duration = taosStr2Int64(token, NULL, 10); if (errno != 0) { return -1; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index e44450fe6e..7b0bef4918 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -39,7 +39,7 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; - *value = strtoll(z, &endPtr, base); + *value = taosStr2Int64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -58,7 +58,7 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { return -1; } - *value = strtoull(z, &endPtr, base); + *value = taosStr2UInt64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -434,7 +434,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val // return -1; // } // - // *value = strtod(pStr, NULL); + // *value = taosStr2Double(pStr, NULL); return 0; } @@ -911,7 +911,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { case TSDB_DATA_TYPE_DOUBLE: { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { errno = 0; - double v = strtod(pVariant->pz, NULL); + double v = taosStr2Double(pVariant->pz, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { taosMemoryFree(pVariant->pz); return -1; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 802cdf7aad..4c23e4ba4b 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -118,21 +118,21 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_FLOAT) { - float va = strtod(a, NULL); + float va = taosStr2Float(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - float vb = strtod(b, NULL); + float vb = taosStr2Float(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_DOUBLE) { - double va = strtod(a, NULL); + double va = taosStr2Double(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - double vb = strtod(b, NULL); + double vb = taosStr2Double(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a438a1e843..894659ddd1 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -123,7 +123,7 @@ static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, ch pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); } else { strncpy(pFqdn, ep, pColon - ep); - *pPort = strtol(pColon + 1, NULL, 10); + *pPort = taosStr2Int32(pColon + 1, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -147,7 +147,7 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t if (NULL == pPortToken) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { - *pPort = strtol(pPortToken->z, NULL, 10); + *pPort = taosStr2Int32(pPortToken->z, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -476,9 +476,9 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); CHECK_OUT_OF_MEM(limitNode); - limitNode->limit = strtol(pLimit->z, NULL, 10); + limitNode->limit = taosStr2Int64(pLimit->z, NULL, 10); if (NULL != pOffset) { - limitNode->offset = strtol(pOffset->z, NULL, 10); + limitNode->offset = taosStr2Int64(pOffset->z, NULL, 10); } return (SNode*)limitNode; } @@ -694,59 +694,59 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { switch (type) { case DB_OPTION_BUFFER: - ((SDatabaseOptions*)pOptions)->buffer = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_CACHELAST: - ((SDatabaseOptions*)pOptions)->cachelast = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->cachelast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_COMP: - ((SDatabaseOptions*)pOptions)->compressionLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_DAYS: { SToken* pToken = pVal; if (TK_NK_INTEGER == pToken->type) { - ((SDatabaseOptions*)pOptions)->daysPerFile = strtol(pToken->z, NULL, 10) * 1440; + ((SDatabaseOptions*)pOptions)->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440; } else { ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken); } break; } case DB_OPTION_FSYNC: - ((SDatabaseOptions*)pOptions)->fsyncPeriod = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MAXROWS: - ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MINROWS: - ((SDatabaseOptions*)pOptions)->minRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_KEEP: ((SDatabaseOptions*)pOptions)->pKeep = pVal; break; case DB_OPTION_PAGES: - ((SDatabaseOptions*)pOptions)->pages = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PAGESIZE: - ((SDatabaseOptions*)pOptions)->pagesize = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PRECISION: copyStringFormStringToken((SToken*)pVal, ((SDatabaseOptions*)pOptions)->precisionStr, sizeof(((SDatabaseOptions*)pOptions)->precisionStr)); break; case DB_OPTION_REPLICA: - ((SDatabaseOptions*)pOptions)->replica = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_STRICT: - ((SDatabaseOptions*)pOptions)->strict = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->strict = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_WAL: - ((SDatabaseOptions*)pOptions)->walLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_VGROUPS: - ((SDatabaseOptions*)pOptions)->numOfVgroups = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_SINGLE_STABLE: - ((SDatabaseOptions*)pOptions)->singleStable = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_RETENTIONS: ((SDatabaseOptions*)pOptions)->pRetentions = pVal; @@ -827,16 +827,16 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType sizeof(((STableOptions*)pOptions)->comment)); break; case TABLE_OPTION_DELAY: - ((STableOptions*)pOptions)->delay = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->delay = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_FILE_FACTOR: - ((STableOptions*)pOptions)->filesFactor = strtod(((SToken*)pVal)->z, NULL); + ((STableOptions*)pOptions)->filesFactor = taosStr2Float(((SToken*)pVal)->z, NULL); break; case TABLE_OPTION_ROLLUP: ((STableOptions*)pOptions)->pRollupFuncs = pVal; break; case TABLE_OPTION_TTL: - ((STableOptions*)pOptions)->ttl = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->ttl = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_SMA: ((STableOptions*)pOptions)->pSma = pVal; @@ -868,7 +868,7 @@ SDataType createDataType(uint8_t type) { } SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { - SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = strtol(pLen->z, NULL, 10)}; + SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = taosStr2Int16(pLen->z, NULL, 10)}; return dt; } @@ -1119,7 +1119,7 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) { SDropDnodeStmt* pStmt = (SDropDnodeStmt*)nodesMakeNode(QUERY_NODE_DROP_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); if (TK_NK_INTEGER == pDnode->type) { - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); } else { if (!checkAndSplitEndpoint(pCxt, pDnode, pStmt->fqdn, &pStmt->port)) { nodesDestroyNode(pStmt); @@ -1133,7 +1133,7 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const const SToken* pValue) { SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); if (NULL != pValue) { trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); @@ -1183,7 +1183,7 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SCreateComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1191,7 +1191,7 @@ SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, co SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SDropComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1251,7 +1251,7 @@ SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* } SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) { - ((SExplainOptions*)pOptions)->ratio = strtod(pVal->z, NULL); + ((SExplainOptions*)pOptions)->ratio = taosStr2Double(pVal->z, NULL); return pOptions; } @@ -1347,7 +1347,7 @@ SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) { SKillStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->targetId = strtol(pId->z, NULL, 10); + pStmt->targetId = taosStr2Int32(pId->z, NULL, 10); return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 29951ccdee..34ae13580b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -442,7 +442,7 @@ static bool isNullStr(SToken* pToken) { static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { errno = 0; - *value = strtold(pToken->z, endPtr); + *value = taosStr2Double(pToken->z, endPtr); // not a valid integer number, return error if ((*endPtr - pToken->z) != pToken->n) { @@ -482,9 +482,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(pMsgBuf, ((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else if (pToken->type == TK_NK_FLOAT) { - return func(pMsgBuf, ((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18e0843e92..0d214fd85e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -498,7 +498,7 @@ static int32_t parseTimeFromValueNode(SValueNode* pVal) { return TSDB_CODE_SUCCESS; } char* pEnd = NULL; - pVal->datum.i = strtoll(pVal->literal, &pEnd, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &pEnd, 10); return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } else { return TSDB_CODE_FAILED; @@ -527,61 +527,61 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; case TSDB_DATA_TYPE_TINYINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int8_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_SMALLINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int16_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_INT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int32_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_UTINYINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint8_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_USMALLINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint16_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint64_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_FLOAT: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(float*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(double*)&pVal->typeData = pVal->datum.d; break; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 795b5ca641..2d844e1842 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3863,7 +3863,7 @@ static YYACTIONTYPE yy_reduce( { yymsp[1].minor.yy140 = 0; } break; case 245: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy140 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy140 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 246: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ { pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy617, &yymsp[-4].minor.yy105, yymsp[-2].minor.yy172, yymsp[-3].minor.yy172, yymsp[0].minor.yy172); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d4a88622e2..45742189d5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -724,7 +724,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(int64_t *)output = strtoll(output, NULL, 10); + *(int64_t *)output = taosStr2Int64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -733,7 +733,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(int64_t *)output = strtoll(newBuf, NULL, 10); + *(int64_t *)output = taosStr2Int64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); @@ -743,7 +743,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(uint64_t *)output = strtoull(output, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -752,7 +752,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(uint64_t *)output = strtoull(newBuf, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c9fcaeb32e..19453bf760 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -92,7 +92,7 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int tmp[len] = 0; ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); - double value = strtod(tmp, NULL); + double value = taosStr2Double(tmp, NULL); *((double *)outData) = value; taosMemoryFreeClear(tmp); @@ -267,22 +267,22 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t value = (int8_t)strtoll(buf, NULL, 10); + int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t value = (int16_t)strtoll(buf, NULL, 10); + int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_INT: { - int32_t value = (int32_t)strtoll(buf, NULL, 10); + int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t value = (int64_t)strtoll(buf, NULL, 10); + int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -292,22 +292,22 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_UTINYINT: { - uint8_t value = (uint8_t)strtoull(buf, NULL, 10); + uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t value = (uint16_t)strtoull(buf, NULL, 10); + uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_UINT: { - uint32_t value = (uint32_t)strtoull(buf, NULL, 10); + uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_UBIGINT: { - uint64_t value = (uint64_t)strtoull(buf, NULL, 10); + uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -315,12 +315,12 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro } static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { - double value = strtod(buf, NULL); + double value = taosStr2Double(buf, NULL); colDataAppendDouble(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowIndex) { - int64_t value = strtoll(buf, NULL, 10); + int64_t value = taosStr2Int64(buf, NULL, 10); bool v = (value != 0)? true:false; colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 7dbd301913..d5762ef87d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -254,4 +254,83 @@ char *taosStrCaseStr(const char *str, const char *pattern) { } } return NULL; +} + +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) { + int64_t tmp = strtoll(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) { + uint64_t tmp = strtoull(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + return tmp; +} + +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SHRT_MIN); + assert(tmp <= SHRT_MAX); + return (int16_t)tmp; +} + +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= USHRT_MAX); + return (uint16_t)tmp; +} + +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SCHAR_MIN); + assert(tmp <= SCHAR_MAX); + return tmp; +} + +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= UCHAR_MAX); + return tmp; +} + +double taosStr2Double(const char *str, char** pEnd) { + double tmp = strtod(str, pEnd); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VAL); + return tmp; +} + +float taosStr2Float(const char *str, char** pEnd) { + float tmp = strtof(str, pEnd); + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VALF); + assert(tmp != NAN); + return tmp; } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index c5cd968fd9..b15c188f04 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -187,7 +187,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal sscanf(p,"%lld",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtol(p, NULL, 10); + *pVal = taosStr2Int64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } @@ -222,7 +222,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV sscanf(p,"%llu",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtoul(p, NULL, 10); + *pVal = taosStr2UInt64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f9c8821b0..21474c316f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -195,16 +195,16 @@ void shellRunSingleCommandImp(char *command) { et = taosGetTimestampUs(); if (error_no == 0) { - printf("Query OK, %d row(s) in set (%.6fs)\n", numOfRows, (et - st) / 1E6); + printf("Query OK, %d rows affected (%.6fs)\n", numOfRows, (et - st) / 1E6); } else { - printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); + printf("Query interrupted (%s), %d rows affected (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } taos_free_result(pSql); } else { int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d of %d rows affected (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); } printf("\n"); From 86580cec090f96839e57f06935f90ef5fc6dd330 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 May 2022 10:53:24 +0800 Subject: [PATCH 091/103] enh(tmq): remove old tmq_commit api --- example/src/tmq.c | 2 +- include/client/taos.h | 6 ++-- tests/test/c/tmqDemo.c | 2 +- tests/test/c/tmqSim.c | 77 ++++++++++++++++++++---------------------- 4 files changed, 41 insertions(+), 46 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index b79d21d051..1abce3f188 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -239,7 +239,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { msg_process(tmqmessage); taos_free_result(tmqmessage); - tmq_commit(tmq, NULL, 1); + tmq_commit_async(tmq, NULL, tmq_commit_cb_print, NULL); /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/include/client/taos.h b/include/client/taos.h index 0194357841..0b8c67aa79 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -232,11 +232,11 @@ DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); -DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); -DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); +DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); + #if 0 -DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); +DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); #endif diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index dc1a77c231..96d7741897 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -368,7 +368,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { /*msg_process(tmqmessage);*/ taos_free_result(tmqmessage); - if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); + if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit_sync(tmq, NULL); } } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index cf113369bc..8322c1a60c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -37,10 +37,10 @@ typedef struct { TdThread thread; int32_t consumerId; - int32_t ifManualCommit; - //int32_t autoCommitIntervalMs; // 1000 ms - //char autoCommit[8]; // true, false - //char autoOffsetRest[16]; // none, earliest, latest + int32_t ifManualCommit; + // int32_t autoCommitIntervalMs; // 1000 ms + // char autoCommit[8]; // true, false + // char autoOffsetRest[16]; // none, earliest, latest int32_t ifCheckData; int64_t expectMsgCnt; @@ -99,21 +99,15 @@ static void printHelp() { } void initLogFile() { - time_t now; - struct tm curTime; - char filename[256]; + time_t now; + struct tm curTime; + char filename[256]; - now = taosTime(NULL); + now = taosTime(NULL); taosLocalTime(&now, &curTime); - sprintf(filename,"%s/../log/tmqlog_%04d-%02d-%02d %02d-%02d-%02d.txt", - configDir, - curTime.tm_year+1900, - curTime.tm_mon+1, - curTime.tm_mday, - curTime.tm_hour, - curTime.tm_min, - curTime.tm_sec); - //sprintf(filename, "%s/../log/tmqlog.txt", configDir); + sprintf(filename, "%s/../log/tmqlog_%04d-%02d-%02d %02d-%02d-%02d.txt", configDir, curTime.tm_year + 1900, + curTime.tm_mon + 1, curTime.tm_mday, curTime.tm_hour, curTime.tm_min, curTime.tm_sec); + // sprintf(filename, "%s/../log/tmqlog.txt", configDir); TdFilePtr pFile = taosOpenFile(filename, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", filename); @@ -137,9 +131,9 @@ void saveConfigToLogFile() { for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - //taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); - //taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); - //taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + // taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + // taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + // taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); taosFprintfFile(g_fp, " Topics: "); for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); @@ -234,17 +228,17 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) while (1) { TAOS_ROW row = taos_fetch_row(msg); - if (row == NULL) break; + if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - + taos_print_row(buf, row, fields, numOfFields); - - if (0 != g_stConfInfo.showRowFlag) { + + if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); } - + totalRows++; } @@ -276,7 +270,7 @@ void build_consumer(SThreadInfo* pInfo) { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); - //tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -299,7 +293,7 @@ void build_consumer(SThreadInfo* pInfo) { pInfo->tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); - + return; } @@ -322,10 +316,10 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); - time_t tTime = taosGetTimestampSec(); + time_t tTime = taosGetTimestampSec(); struct tm tm = *taosLocalTime(&tTime, NULL); taosFprintfFile(g_fp, "# save result: %d-%02d-%02d %02d:%02d:%02d, sql: %s\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr); + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { @@ -357,11 +351,11 @@ void loop_consume(SThreadInfo* pInfo) { totalMsgs++; if (totalRows >= pInfo->expectMsgCnt) { - taosFprintfFile(g_fp, "==== totalRows >= pInfo->expectMsgCnt, so break\n"); + taosFprintfFile(g_fp, "==== totalRows >= pInfo->expectMsgCnt, so break\n"); break; } - } else { - taosFprintfFile(g_fp, "==== delay over time, so break\n"); + } else { + taosFprintfFile(g_fp, "==== delay over time, so break\n"); break; } } @@ -389,7 +383,7 @@ void* consumeThreadFunc(void* param) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - + tmq_list_destroy(pInfo->topicList); pInfo->topicList = NULL; @@ -397,17 +391,18 @@ void* consumeThreadFunc(void* param) { if (pInfo->ifManualCommit) { taosFprintfFile(g_fp, "tmq_commit() manual commit when consume end.\n"); - pPrint("tmq_commit() manual commit when consume end.\n"); - tmq_commit(pInfo->tmq, NULL, 0); + pPrint("tmq_commit() manual commit when consume end.\n"); + /*tmq_commit(pInfo->tmq, NULL, 0);*/ + tmq_commit_sync(pInfo->tmq, NULL); } - + err = tmq_unsubscribe(pInfo->tmq); if (err) { pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); pInfo->consumeMsgCnt = -1; return NULL; } - + err = tmq_consumer_close(pInfo->tmq); if (err) { pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); @@ -485,9 +480,9 @@ int32_t getConsumeInfo() { int32_t* lengths = taos_fetch_lengths(pRes); // set default value - //g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; - //memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); - //memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + // g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + // memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + // memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { From a41dc27e6df6b2e143d2bb60431ff5a93fdbddbf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 11:10:13 +0800 Subject: [PATCH 092/103] refactor: dnode monitor --- source/dnode/mgmt/mgmt_bnode/src/bmHandle.c | 2 +- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 9 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 8 +- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 4 +- source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c | 104 ----------- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 14 +- source/dnode/mgmt/mgmt_qnode/src/qmHandle.c | 2 +- source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 +- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 17 +- source/dnode/mgmt/node_mgmt/inc/dmNodes.h | 45 +++++ source/dnode/mgmt/node_mgmt/src/dmEnv.c | 9 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 1 + source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 172 +++++++++++++++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 32 ++-- 16 files changed, 269 insertions(+), 158 deletions(-) delete mode 100644 source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c create mode 100644 source/dnode/mgmt/node_mgmt/inc/dmNodes.h create mode 100644 source/dnode/mgmt/node_mgmt/src/dmMonitor.c diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c index 2637e0af04..d2e547078e 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "bmInt.h" -static void bmGetMonitorInfo(SBnodeMgmt *pMgmt, SMonBmInfo *bmInfo) {} +void bmGetMonitorInfo(SBnodeMgmt *pMgmt, SMonBmInfo *bmInfo) {} int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonBmInfo bmInfo = {0}; diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index f90fd72c6b..ae8879326d 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -32,7 +32,9 @@ typedef struct SDnodeMgmt { SSingleWorker mgmtWorker; ProcessCreateNodeFp processCreateNodeFp; ProcessDropNodeFp processDropNodeFp; - IsNodeRequiredFp isNodeRequiredFp; + SendMonitorReportFp sendMonitorReportFp; + GetVnodeLoadsFp getVnodeLoadsFp; + GetMnodeLoadsFp getMnodeLoadsFp; } SDnodeMgmt; // dmHandle.c @@ -43,11 +45,6 @@ int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); -// dmMonitor.c -void dmGetVnodeLoads(SDnodeMgmt *pMgmt, SMonVloadInfo *pInfo); -void dmGetMnodeLoads(SDnodeMgmt *pMgmt, SMonMloadInfo *pInfo); -void dmSendMonitorReport(SDnodeMgmt *pMgmt); - // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 38e71754b6..7c1162ec10 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -72,11 +72,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { taosRUnLockLatch(&pMgmt->pData->latch); SMonVloadInfo vinfo = {0}; - dmGetVnodeLoads(pMgmt, &vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); req.pVloads = vinfo.pVloads; SMonMloadInfo minfo = {0}; - dmGetMnodeLoads(pMgmt, &minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); @@ -115,7 +115,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; - dmGetMnodeLoads(pMgmt, &minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); @@ -123,7 +123,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { } SMonVloadInfo vinfo = {0}; - dmGetVnodeLoads(pMgmt, &vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 3b343d4916..59c926545e 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -45,7 +45,9 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->name = pInput->name; pMgmt->processCreateNodeFp = pInput->processCreateNodeFp; pMgmt->processDropNodeFp = pInput->processDropNodeFp; - pMgmt->isNodeRequiredFp = pInput->isNodeRequiredFp; + pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; + pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; + pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; if (dmStartWorker(pMgmt) != 0) { return -1; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c b/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c deleted file mode 100644 index 3547c76937..0000000000 --- a/source/dnode/mgmt/mgmt_dnode/src/dmMonitor.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmInt.h" - -#define dmSendLocalRecv(pMgmt, mtype, func, pInfo) \ - if (!tsMultiProcess) { \ - SRpcMsg rsp = {0}; \ - SRpcMsg req = {.msgType = mtype}; \ - SEpSet epset = {.inUse = 0, .numOfEps = 1}; \ - tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); \ - epset.eps[0].port = tsServerPort; \ - rpcSendRecv(pMgmt->msgCb.clientRpc, &epset, &req, &rsp); \ - if (rsp.code == 0 && rsp.contLen > 0) { \ - func(rsp.pCont, rsp.contLen, pInfo); \ - } \ - rpcFreeCont(rsp.pCont); \ - } - -static void dmGetMonitorBasicInfo(SDnodeMgmt *pMgmt, SMonBasicInfo *pInfo) { - pInfo->protocol = 1; - pInfo->dnode_id = pMgmt->pData->dnodeId; - pInfo->cluster_id = pMgmt->pData->clusterId; - tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); -} - -static void dmGetMonitorDnodeInfo(SDnodeMgmt *pMgmt, SMonDnodeInfo *pInfo) { - pInfo->uptime = (taosGetTimestampMs() - pMgmt->pData->rebootTime) / (86400000.0f); - pInfo->has_mnode = (*pMgmt->isNodeRequiredFp)(MNODE); - pInfo->has_qnode = (*pMgmt->isNodeRequiredFp)(QNODE); - pInfo->has_snode = (*pMgmt->isNodeRequiredFp)(SNODE); - pInfo->has_bnode = (*pMgmt->isNodeRequiredFp)(BNODE); - tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); - pInfo->logdir.size = tsLogSpace.size; - tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); - pInfo->tempdir.size = tsTempSpace.size; -} - -static void dmGetMonitorInfo(SDnodeMgmt *pMgmt, SMonDmInfo *pInfo) { - dmGetMonitorBasicInfo(pMgmt, &pInfo->basic); - dmGetMonitorDnodeInfo(pMgmt, &pInfo->dnode); - dmGetMonitorSystemInfo(&pInfo->sys); -} - -void dmSendMonitorReport(SDnodeMgmt *pMgmt) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); - - SMonDmInfo dmInfo = {0}; - SMonMmInfo mmInfo = {0}; - SMonVmInfo vmInfo = {0}; - SMonQmInfo qmInfo = {0}; - SMonSmInfo smInfo = {0}; - SMonBmInfo bmInfo = {0}; - - dmGetMonitorInfo(pMgmt, &dmInfo); - dmSendLocalRecv(pMgmt, TDMT_MON_VM_INFO, tDeserializeSMonVmInfo, &vmInfo); - if (dmInfo.dnode.has_mnode) { - dmSendLocalRecv(pMgmt, TDMT_MON_MM_INFO, tDeserializeSMonMmInfo, &mmInfo); - } - if (dmInfo.dnode.has_qnode) { - dmSendLocalRecv(pMgmt, TDMT_MON_QM_INFO, tDeserializeSMonQmInfo, &qmInfo); - } - if (dmInfo.dnode.has_snode) { - dmSendLocalRecv(pMgmt, TDMT_MON_SM_INFO, tDeserializeSMonSmInfo, &smInfo); - } - if (dmInfo.dnode.has_bnode) { - dmSendLocalRecv(pMgmt, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); - } - - monSetDmInfo(&dmInfo); - monSetMmInfo(&mmInfo); - monSetVmInfo(&vmInfo); - monSetQmInfo(&qmInfo); - monSetSmInfo(&smInfo); - monSetBmInfo(&bmInfo); - tFreeSMonMmInfo(&mmInfo); - tFreeSMonVmInfo(&vmInfo); - tFreeSMonQmInfo(&qmInfo); - tFreeSMonSmInfo(&smInfo); - tFreeSMonBmInfo(&bmInfo); - monSendReport(); -} - -void dmGetVnodeLoads(SDnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { - dmSendLocalRecv(pMgmt, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); -} - -void dmGetMnodeLoads(SDnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { - dmSendLocalRecv(pMgmt, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); -} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 599fa07e1c..6a7e0ad322 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -50,7 +50,7 @@ static void *dmMonitorThreadFp(void *param) { int64_t curTime = taosGetTimestampMs(); float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsMonitorInterval) { - dmSendMonitorReport(pMgmt); + (*pMgmt->sendMonitorReportFp)(); lastTime = curTime; } } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index a09fd2627e..8b2d8f8bad 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -16,8 +16,13 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -static void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *mmInfo) { - mndGetMonitorInfo(pMgmt->pMnode, &mmInfo->cluster, &mmInfo->vgroup, &mmInfo->grant); +void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { + mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); +} + +void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { + pInfo->isMnode = 1; + mndGetLoad(pMgmt->pMnode, &pInfo->load); } int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pReq) { @@ -45,11 +50,6 @@ int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pReq) { return 0; } -static void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { - pInfo->isMnode = 1; - mndGetLoad(pMgmt->pMnode, &pInfo->load); -} - int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonMloadInfo mloads = {0}; mmGetMnodeLoads(pMgmt, &mloads); diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 4518a558bb..ac0868d8ef 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "qmInt.h" -static void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) {} +void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) {} int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonQmInfo qmInfo = {0}; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index ad83639a8d..d6ff73b132 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "smInt.h" -static void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {} +void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {} int32_t smProcessGetMonitorInfoReq(SSnodeMgmt *pMgmt, SRpcMsg *pReq) { SMonSmInfo smInfo = {0}; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 902998f28e..a4da6d089c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -static void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { +void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); if (pInfo->pVloads == NULL) return; @@ -37,7 +37,7 @@ static void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { taosRUnLockLatch(&pMgmt->latch); } -static void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { +void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { SMonVloadInfo vloads = {0}; vmGetVnodeLoads(pMgmt, &vloads); diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 7484c1e18f..9d092a93bc 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_DND_IMP_H_ -#define _TD_DND_IMP_H_ +#ifndef _TD_DND_MGMT_H_ +#define _TD_DND_MGMT_H_ // tobe deleted #include "uv.h" @@ -165,16 +165,13 @@ SMsgCb dmGetMsgcb(SDnode *pDnode); int32_t dmInitMsgHandle(SDnode *pDnode); int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -// mgmt nodes -SMgmtFunc dmGetMgmtFunc(); -SMgmtFunc bmGetMgmtFunc(); -SMgmtFunc qmGetMgmtFunc(); -SMgmtFunc smGetMgmtFunc(); -SMgmtFunc vmGetMgmtFunc(); -SMgmtFunc mmGetMgmtFunc(); +// dmMonitor.c +void dmSendMonitorReport(); +void dmGetVnodeLoads(SMonVloadInfo *pInfo); +void dmGetMnodeLoads(SMonMloadInfo *pInfo); #ifdef __cplusplus } #endif -#endif /*_TD_DND_IMP_H_*/ \ No newline at end of file +#endif /*_TD_DND_MGMT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h new file mode 100644 index 0000000000..3ac71de530 --- /dev/null +++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_NODES_H_ +#define _TD_DND_NODES_H_ + +#include "dmInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SMgmtFunc dmGetMgmtFunc(); +SMgmtFunc bmGetMgmtFunc(); +SMgmtFunc qmGetMgmtFunc(); +SMgmtFunc smGetMgmtFunc(); +SMgmtFunc vmGetMgmtFunc(); +SMgmtFunc mmGetMgmtFunc(); + +void mmGetMonitorInfo(void *pMgmt, SMonMmInfo *pInfo); +void vmGetMonitorInfo(void *pMgmt, SMonVmInfo *pInfo); +void qmGetMonitorInfo(void *pMgmt, SMonQmInfo *pInfo); +void smGetMonitorInfo(void *pMgmt, SMonSmInfo *pInfo); +void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo); + +void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo); +void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_NODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index af5c0f00db..07d0c43360 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -168,11 +168,6 @@ static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return code; } -static bool dmIsNodeRequired(EDndNodeType ntype) { - SDnode *pDnode = dmInstance(); - return pDnode->wrappers[ntype].required; -} - SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { SMgmtInputOpt opt = { .path = pWrapper->path, @@ -180,7 +175,9 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .pData = &pWrapper->pDnode->data, .processCreateNodeFp = dmProcessCreateNodeReq, .processDropNodeFp = dmProcessDropNodeReq, - .isNodeRequiredFp = dmIsNodeRequired, + .sendMonitorReportFp = dmSendMonitorReport, + .getVnodeLoadsFp = dmGetVnodeLoads, + .getMnodeLoadsFp = dmGetMnodeLoads, }; opt.msgCb = dmGetMsgcb(pWrapper->pDnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 30d7750f79..809c903e5c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" +#include "dmNodes.h" static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c new file mode 100644 index 0000000000..cb79de534d --- /dev/null +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmMgmt.h" +#include "dmNodes.h" + +#define dmSendLocalRecv(pDnode, mtype, func, pInfo) \ + SRpcMsg rsp = {0}; \ + SRpcMsg req = {.msgType = mtype}; \ + SEpSet epset = {.inUse = 0, .numOfEps = 1}; \ + tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); \ + epset.eps[0].port = tsServerPort; \ + rpcSendRecv(pDnode->trans.clientRpc, &epset, &req, &rsp); \ + if (rsp.code == 0 && rsp.contLen > 0) { \ + func(rsp.pCont, rsp.contLen, pInfo); \ + } \ + rpcFreeCont(rsp.pCont); + +static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { + pInfo->protocol = 1; + pInfo->dnode_id = pDnode->data.dnodeId; + pInfo->cluster_id = pDnode->data.clusterId; + tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); +} + +static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pDnode->data.rebootTime) / (86400000.0f); + pInfo->has_mnode = pDnode->wrappers[MNODE].required; + pInfo->has_qnode = pDnode->wrappers[QNODE].required; + pInfo->has_snode = pDnode->wrappers[SNODE].required; + pInfo->has_bnode = pDnode->wrappers[BNODE].required; + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; +} + +static void dmGetDmMonitorInfo(SDnode *pDnode) { + SMonDmInfo dmInfo = {0}; + dmGetMonitorBasicInfo(pDnode, &dmInfo.basic); + dmGetMonitorDnodeInfo(pDnode, &dmInfo.dnode); + dmGetMonitorSystemInfo(&dmInfo.sys); + monSetDmInfo(&dmInfo); +} + +static void dmGetMmMonitorInfo(SDnode *pDnode) { + SMonMmInfo mmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_MM_INFO, tDeserializeSMonMmInfo, &mmInfo); + } else if (pWrapper->pMgmt != NULL) { + mmGetMonitorInfo(pWrapper->pMgmt, &mmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetMmInfo(&mmInfo); + tFreeSMonMmInfo(&mmInfo); +} + +static void dmGetVmMonitorInfo(SDnode *pDnode) { + SMonVmInfo vmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_VM_INFO, tDeserializeSMonVmInfo, &vmInfo); + } else if (pWrapper->pMgmt != NULL) { + vmGetMonitorInfo(pWrapper->pMgmt, &vmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetVmInfo(&vmInfo); + tFreeSMonVmInfo(&vmInfo); +} + +static void dmGetQmMonitorInfo(SDnode *pDnode) { + SMonQmInfo qmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_QM_INFO, tDeserializeSMonQmInfo, &qmInfo); + } else if (pWrapper->pMgmt != NULL) { + qmGetMonitorInfo(pWrapper->pMgmt, &qmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetQmInfo(&qmInfo); + tFreeSMonQmInfo(&qmInfo); +} + +static void dmGetSmMonitorInfo(SDnode *pDnode) { + SMonSmInfo smInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_SM_INFO, tDeserializeSMonSmInfo, &smInfo); + } else if (pWrapper->pMgmt != NULL) { + smGetMonitorInfo(pWrapper->pMgmt, &smInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetSmInfo(&smInfo); + tFreeSMonSmInfo(&smInfo); +} + +static void dmGetBmMonitorInfo(SDnode *pDnode) { + SMonBmInfo bmInfo = {0}; + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); + } else if (pWrapper->pMgmt != NULL) { + bmGetMonitorInfo(pWrapper->pMgmt, &bmInfo); + } + } + dmReleaseWrapper(pWrapper); + monSetBmInfo(&bmInfo); + tFreeSMonBmInfo(&bmInfo); +} + +void dmSendMonitorReport() { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SDnode *pDnode = dmInstance(); + dmGetDmMonitorInfo(pDnode); + dmGetMmMonitorInfo(pDnode); + dmGetVmMonitorInfo(pDnode); + dmGetQmMonitorInfo(pDnode); + dmGetSmMonitorInfo(pDnode); + dmGetBmMonitorInfo(pDnode); + monSendReport(); +} + +void dmGetVnodeLoads(SMonVloadInfo *pInfo) { + SDnode *pDnode = dmInstance(); + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); + } else if (pWrapper->pMgmt != NULL) { + vmGetVnodeLoads(pWrapper->pMgmt, pInfo); + } + } + dmReleaseWrapper(pWrapper); +} + +void dmGetMnodeLoads(SMonMloadInfo *pInfo) { + SDnode *pDnode = dmInstance(); + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL && pWrapper->required) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); + } else if (pWrapper->pMgmt != NULL) { + mmGetMnodeLoads(pWrapper->pMgmt, pInfo); + } + } + dmReleaseWrapper(pWrapper); +} diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 45bd3e4f64..b0e764bf8e 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -89,21 +89,23 @@ typedef enum { typedef int32_t (*ProcessCreateNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef int32_t (*ProcessDropNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg); -typedef bool (*IsNodeRequiredFp)(EDndNodeType ntype); +typedef void (*SendMonitorReportFp)(); +typedef void (*GetVnodeLoadsFp)(); +typedef void (*GetMnodeLoadsFp)(); typedef struct { - int32_t dnodeId; - int64_t clusterId; - int64_t dnodeVer; - int64_t updateTime; - int64_t rebootTime; - bool dropped; - bool stopped; - SEpSet mnodeEps; - SArray *dnodeEps; - SHashObj *dnodeHash; - SRWLatch latch; - SMsgCb msgCb; + int32_t dnodeId; + int64_t clusterId; + int64_t dnodeVer; + int64_t updateTime; + int64_t rebootTime; + bool dropped; + bool stopped; + SEpSet mnodeEps; + SArray *dnodeEps; + SHashObj *dnodeHash; + SRWLatch latch; + SMsgCb msgCb; } SDnodeData; typedef struct { @@ -113,7 +115,9 @@ typedef struct { SMsgCb msgCb; ProcessCreateNodeFp processCreateNodeFp; ProcessDropNodeFp processDropNodeFp; - IsNodeRequiredFp isNodeRequiredFp; + SendMonitorReportFp sendMonitorReportFp; + GetVnodeLoadsFp getVnodeLoadsFp; + GetMnodeLoadsFp getMnodeLoadsFp; } SMgmtInputOpt; typedef struct { From 83f4d1286e18fda5a10f7d2b4d483f269e92af39 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 19 May 2022 11:20:27 +0800 Subject: [PATCH 093/103] fix: executor process express tree scalar execution error --- source/libs/executor/src/executorimpl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index db34f5dc67..3505af46e4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -869,8 +869,12 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); - + int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pBlockList); + return code; + } + int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); From 82b2ae419481238b617cb659621b5134d9c55bd9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 19 May 2022 11:20:47 +0800 Subject: [PATCH 094/103] enh(tmq): set and show client id --- include/common/tmsg.h | 3 +++ source/common/src/systable.c | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 9 ++++---- source/dnode/mnode/impl/src/mndTopic.c | 26 +++++++++++------------ tests/script/tsim/tstream/basic1.sim | 20 ++++++++--------- 6 files changed, 32 insertions(+), 30 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7a60542313..f7562a4b9b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1480,6 +1480,7 @@ typedef struct { typedef struct { int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; + char clientId[256]; SArray* topicNames; // SArray } SCMSubscribeReq; @@ -1487,6 +1488,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->cgroup); + tlen += taosEncodeString(buf, pReq->clientId); int32_t topicNum = taosArrayGetSize(pReq->topicNames); tlen += taosEncodeFixedI32(buf, topicNum); @@ -1500,6 +1502,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->cgroup); + buf = taosDecodeStringTo(buf, pReq->clientId); int32_t topicNum; buf = taosDecodeFixedI32(buf, &topicNum); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index e4e5abe148..9fe7645e2b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -263,7 +263,7 @@ static const SSysDbTableSchema topicSchema[] = { static const SSysDbTableSchema consumerSchema[] = { {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "consumer_group", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ec9139836a..a04417736a 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -463,7 +463,7 @@ typedef struct { typedef struct { int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; - char appId[TSDB_CGROUP_LEN]; + char clientId[256]; int8_t updateType; // used only for update int32_t epoch; int32_t status; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 274876567e..4fe7d8a399 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -427,6 +427,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerOld = mndAcquireConsumer(pMnode, consumerId); if (pConsumerOld == NULL) { pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256); pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; pConsumerNew->rebNewTopics = newSub; subscribe.topicNames = NULL; @@ -848,11 +849,11 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); // app id - char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN); - varDataSetLen(appId, strlen(varDataVal(appId))); + char clientId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(clientId), pConsumer->clientId, TSDB_CGROUP_LEN); + varDataSetLen(clientId, strlen(varDataVal(clientId))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)appId, false); + colDataAppend(pColInfo, numOfRows, (const char *)clientId, false); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 3f559cb6c0..c6eebb5c5d 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -492,8 +492,8 @@ static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTo } static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SMnode *pMnode = pReq->info.node; + /*SSdb *pSdb = pMnode->pSdb;*/ SMDropTopicReq dropReq = {0}; if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -502,16 +502,16 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name); - // if (pTopic == NULL) { - // if (dropReq.igNotExists) { - // mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); - // return 0; - // } else { - // terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - // mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); - // return -1; - // } - // } + if (pTopic == NULL) { + if (dropReq.igNotExists) { + mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); + return 0; + } else { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + return -1; + } + } if (pTopic->refConsumerCnt != 0) { mndReleaseTopic(pMnode, pTopic); @@ -528,12 +528,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); -#if 1 if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); return -1; } -#endif if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim index 9965772c75..0997e313c8 100644 --- a/tests/script/tsim/tstream/basic1.sim +++ b/tests/script/tsim/tstream/basic1.sim @@ -409,53 +409,53 @@ sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 4 then print ======$data11 - # return -1 + return -1 endi if $data12 != 4 then print ======$data12 - # return -1 + return -1 endi if $data13 != 10 then print ======$data13 - # return -1 + return -1 endi if $data14 != 3 then print ======$data14 - # return -1 + return -1 endi if $data15 != 1 then print ======$data15 - # return -1 + return -1 endi # row 2 if $data21 != 4 then print ======$data21 - # return -1 + return -1 endi if $data22 != 4 then print ======$data22 - # return -1 + return -1 endi if $data23 != 15 then print ======$data23 - # return -1 + return -1 endi if $data24 != 4 then print ======$data24 - # return -1 + return -1 endi if $data25 != 3 then print ======$data25 - # return -1 + return -1 endi From f1a788f16174350c6d4e85a4580b9e20d1c382f8 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 11:34:52 +0800 Subject: [PATCH 095/103] fix(os): win run error --- source/os/CMakeLists.txt | 3 +++ source/os/src/osString.c | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index ad6cfc8b95..90b8e9dd8a 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -27,6 +27,9 @@ if(BUILD_ADDR2LINE) os PUBLIC addr2line dl z ) endif () +if(CHECK_STR2INT_ERROR) + add_definitions(-DTD_CHECK_STR_TO_INT_ERROR) +endif() target_link_libraries( os PUBLIC pthread ) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d5762ef87d..da1fbd364f 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -258,79 +258,99 @@ char *taosStrCaseStr(const char *str, const char *pattern) { int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) { int64_t tmp = strtoll(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) { uint64_t tmp = strtoull(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); +#endif return tmp; } int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp >= SHRT_MIN); assert(tmp <= SHRT_MAX); +#endif return (int16_t)tmp; } uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp <= USHRT_MAX); +#endif return (uint16_t)tmp; } int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp >= SCHAR_MIN); assert(tmp <= SCHAR_MAX); +#endif return tmp; } uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp <= UCHAR_MAX); +#endif return tmp; } double taosStr2Double(const char *str, char** pEnd) { double tmp = strtod(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp != HUGE_VAL); +#endif return tmp; } float taosStr2Float(const char *str, char** pEnd) { float tmp = strtof(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR assert(errno != ERANGE); assert(errno != EINVAL); assert(tmp != HUGE_VALF); assert(tmp != NAN); +#endif return tmp; } \ No newline at end of file From fefc2dad116a8e0ec123884d95c19451b3941da3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 11:51:52 +0800 Subject: [PATCH 096/103] refactor: dnode monitor --- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index df377fefe7..0bab05f973 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -154,6 +154,6 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq, bool deployed) { return -1; } - dInfo("successed to write %s, deployed:%d", realfile, deployed); + dDebug("successed to write %s, deployed:%d", realfile, deployed); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index f6c7bb33e6..7a6c5f982e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -128,7 +128,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes = vnodesNum; code = 0; - dInfo("succcessed to read file %s", file); + dDebug("succcessed to read file %s", file); _OVER: if (content != NULL) taosMemoryFree(content); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 809c903e5c..3cbb9ff046 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -190,7 +190,7 @@ int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) { } dmReportStartup("dnode-transport", "initialized"); - dInfo("dnode is created, ptr:%p", pDnode); + dDebug("dnode is created, ptr:%p", pDnode); code = 0; _OVER: @@ -209,7 +209,7 @@ void dmCleanupDnode(SDnode *pDnode) { dmCleanupClient(pDnode); dmCleanupServer(pDnode); dmClearVars(pDnode); - dInfo("dnode is closed, ptr:%p", pDnode); + dDebug("dnode is closed, ptr:%p", pDnode); } void dmSetStatus(SDnode *pDnode, EDndRunStatus status) { From 2d8ed7a947834c5a6c14525f8b438f81c4c22f18 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 12:12:20 +0800 Subject: [PATCH 097/103] fix: monitor in multi-process mode --- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 78 ++++++++++----------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index cb79de534d..2497da13ec 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -57,78 +57,78 @@ static void dmGetDmMonitorInfo(SDnode *pDnode) { } static void dmGetMmMonitorInfo(SDnode *pDnode) { - SMonMmInfo mmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonMmInfo mmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_MM_INFO, tDeserializeSMonMmInfo, &mmInfo); } else if (pWrapper->pMgmt != NULL) { mmGetMonitorInfo(pWrapper->pMgmt, &mmInfo); } + dmReleaseWrapper(pWrapper); + monSetMmInfo(&mmInfo); + tFreeSMonMmInfo(&mmInfo); } - dmReleaseWrapper(pWrapper); - monSetMmInfo(&mmInfo); - tFreeSMonMmInfo(&mmInfo); } static void dmGetVmMonitorInfo(SDnode *pDnode) { - SMonVmInfo vmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonVmInfo vmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_VM_INFO, tDeserializeSMonVmInfo, &vmInfo); } else if (pWrapper->pMgmt != NULL) { vmGetMonitorInfo(pWrapper->pMgmt, &vmInfo); } + dmReleaseWrapper(pWrapper); + monSetVmInfo(&vmInfo); + tFreeSMonVmInfo(&vmInfo); } - dmReleaseWrapper(pWrapper); - monSetVmInfo(&vmInfo); - tFreeSMonVmInfo(&vmInfo); } static void dmGetQmMonitorInfo(SDnode *pDnode) { - SMonQmInfo qmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[QNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonQmInfo qmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_QM_INFO, tDeserializeSMonQmInfo, &qmInfo); } else if (pWrapper->pMgmt != NULL) { qmGetMonitorInfo(pWrapper->pMgmt, &qmInfo); } + dmReleaseWrapper(pWrapper); + monSetQmInfo(&qmInfo); + tFreeSMonQmInfo(&qmInfo); } - dmReleaseWrapper(pWrapper); - monSetQmInfo(&qmInfo); - tFreeSMonQmInfo(&qmInfo); } static void dmGetSmMonitorInfo(SDnode *pDnode) { - SMonSmInfo smInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[SNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonSmInfo smInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_SM_INFO, tDeserializeSMonSmInfo, &smInfo); } else if (pWrapper->pMgmt != NULL) { smGetMonitorInfo(pWrapper->pMgmt, &smInfo); } + dmReleaseWrapper(pWrapper); + monSetSmInfo(&smInfo); + tFreeSMonSmInfo(&smInfo); } - dmReleaseWrapper(pWrapper); - monSetSmInfo(&smInfo); - tFreeSMonSmInfo(&smInfo); } static void dmGetBmMonitorInfo(SDnode *pDnode) { - SMonBmInfo bmInfo = {0}; - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[BNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + SMonBmInfo bmInfo = {0}; if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); } else if (pWrapper->pMgmt != NULL) { bmGetMonitorInfo(pWrapper->pMgmt, &bmInfo); } + dmReleaseWrapper(pWrapper); + monSetBmInfo(&bmInfo); + tFreeSMonBmInfo(&bmInfo); } - dmReleaseWrapper(pWrapper); - monSetBmInfo(&bmInfo); - tFreeSMonBmInfo(&bmInfo); } void dmSendMonitorReport() { @@ -147,26 +147,24 @@ void dmSendMonitorReport() { void dmGetVnodeLoads(SMonVloadInfo *pInfo) { SDnode *pDnode = dmInstance(); - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, VNODE); - if (pWrapper != NULL && pWrapper->required) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE]; + if (dmMarkWrapper(pWrapper) == 0) { if (tsMultiProcess) { dmSendLocalRecv(pDnode, TDMT_MON_VM_LOAD, tDeserializeSMonVloadInfo, pInfo); } else if (pWrapper->pMgmt != NULL) { vmGetVnodeLoads(pWrapper->pMgmt, pInfo); } + dmReleaseWrapper(pWrapper); } - dmReleaseWrapper(pWrapper); } void dmGetMnodeLoads(SMonMloadInfo *pInfo) { SDnode *pDnode = dmInstance(); - SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL && pWrapper->required) { - if (tsMultiProcess) { - dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); - } else if (pWrapper->pMgmt != NULL) { - mmGetMnodeLoads(pWrapper->pMgmt, pInfo); - } + SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); + } else if (pWrapper->pMgmt != NULL) { + mmGetMnodeLoads(pWrapper->pMgmt, pInfo); } dmReleaseWrapper(pWrapper); } From 7002f32a92d77705ee6a1528579f27b6a4da8d19 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 12:30:18 +0800 Subject: [PATCH 098/103] enh: add tagver and colver to stb and its msg --- include/common/tmsg.h | 14 ++++++++++++++ source/dnode/mnode/impl/inc/mndDef.h | 2 ++ source/dnode/mnode/impl/src/mndStb.c | 19 ++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f7562a4b9b..09dd7c64d2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -301,6 +301,8 @@ typedef struct SSchema { typedef struct { int32_t nCols; int32_t sver; + int32_t tagVer; + int32_t colVer; SSchema* pSchema; } SSchemaWrapper; @@ -309,6 +311,8 @@ static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* p if (pSW == NULL) return pSW; pSW->nCols = pSchemaWrapper->nCols; pSW->sver = pSchemaWrapper->sver; + pSW->tagVer = pSchemaWrapper->tagVer; + pSW->colVer = pSchemaWrapper->colVer; pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { taosMemoryFree(pSW); @@ -364,6 +368,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr int32_t tlen = 0; tlen += taosEncodeVariantI32(buf, pSW->nCols); tlen += taosEncodeVariantI32(buf, pSW->sver); + tlen += taosEncodeVariantI32(buf, pSW->tagVer); + tlen += taosEncodeVariantI32(buf, pSW->colVer); for (int32_t i = 0; i < pSW->nCols; i++) { tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); } @@ -373,6 +379,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { buf = taosDecodeVariantI32(buf, &pSW->nCols); buf = taosDecodeVariantI32(buf, &pSW->sver); + buf = taosDecodeVariantI32(buf, &pSW->tagVer); + buf = taosDecodeVariantI32(buf, &pSW->colVer); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { return NULL; @@ -387,6 +395,8 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) { if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1; if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->tagVer) < 0) return -1; + if (tEncodeI32v(pEncoder, pSW->colVer) < 0) return -1; for (int32_t i = 0; i < pSW->nCols; i++) { if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; } @@ -397,6 +407,8 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSch static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) { if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->tagVer) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->colVer) < 0) return -1; pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) return -1; @@ -410,6 +422,8 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWra static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaWrapper* pSW) { if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->tagVer) < 0) return -1; + if (tDecodeI32v(pDecoder, &pSW->colVer) < 0) return -1; pSW->pSchema = (SSchema*)tDecoderMalloc(pDecoder, pSW->nCols * sizeof(SSchema)); if (pSW->pSchema == NULL) return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a04417736a..81f4c5ed1e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -365,6 +365,8 @@ typedef struct { int64_t uid; int64_t dbUid; int32_t version; + int32_t tagVer; + int32_t colVer; int32_t nextColId; float xFilesFactor; int32_t delay; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e8a0b31e2f..c8d4f91183 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -88,6 +88,8 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT64(pRaw, dataPos, pStb->uid, _OVER) SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->tagVer, _OVER) + SDB_SET_INT32(pRaw, dataPos, pStb->colVer, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER) SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER) @@ -166,6 +168,8 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pStb->uid, _OVER) SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->tagVer, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pStb->colVer, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, _OVER) int32_t xFilesFactor = 0; SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER) @@ -317,7 +321,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->updateTime = pNew->updateTime; pOld->version = pNew->version; - pOld->nextColId = pNew->nextColId; + pOld->tagVer = pNew->tagVer; + pOld->colVer = pNew->colVer; + pOld->colVer = pNew->nextColId; pOld->ttl = pNew->ttl; pOld->numOfColumns = pNew->numOfColumns; pOld->numOfTags = pNew->numOfTags; @@ -384,6 +390,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.rollup = pStb->ast1Len > 0 ? 1 : 0; req.schema.nCols = pStb->numOfColumns; req.schema.sver = pStb->version; + req.schema.tagVer = pStb->tagVer; + req.schema.colVer = pStb->colVer; req.schema.pSchema = pStb->pColumns; req.schemaTag.nCols = pStb->numOfTags; req.schemaTag.sver = 1; @@ -657,6 +665,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); pDst->dbUid = pDb->uid; pDst->version = 1; + pDst->tagVer = 1; + pDst->colVer = 1; pDst->nextColId = 1; pDst->xFilesFactor = pCreate->xFilesFactor; pDst->delay = pCreate->delay; @@ -949,6 +959,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p } pNew->version++; + pNew->tagVer++; return 0; } @@ -967,6 +978,7 @@ static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const ch pNew->numOfTags--; pNew->version++; + pNew->tagVer++; mDebug("stb:%s, start to drop tag %s", pNew->name, tagName); return 0; } @@ -1007,6 +1019,7 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF memcpy(pSchema->name, newTagName, TSDB_COL_NAME_LEN); pNew->version++; + pNew->tagVer++; mDebug("stb:%s, start to modify tag %s to %s", pNew->name, oldTagName, newTagName); return 0; } @@ -1036,6 +1049,7 @@ static int32_t mndAlterStbTagBytes(const SStbObj *pOld, SStbObj *pNew, const SFi pTag->bytes = pField->bytes; pNew->version++; + pNew->tagVer++; mDebug("stb:%s, start to modify tag len %s to %d", pNew->name, pField->name, pField->bytes); return 0; @@ -1075,6 +1089,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray } pNew->version++; + pNew->colVer++; return 0; } @@ -1103,6 +1118,7 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const pNew->numOfColumns--; pNew->version++; + pNew->colVer++; mDebug("stb:%s, start to drop col %s", pNew->name, colName); return 0; } @@ -1141,6 +1157,7 @@ static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const pCol->bytes = pField->bytes; pNew->version++; + pNew->colVer++; mDebug("stb:%s, start to modify col len %s to %d", pNew->name, pField->name, pField->bytes); return 0; From 1e7dd9084492fb49e8f9bbb9223846f43c4e4c97 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 19 May 2022 13:13:35 +0800 Subject: [PATCH 099/103] enh: return success if version in alter request is smaller than the stb --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 2 ++ source/dnode/mnode/impl/src/mndStb.c | 9 ++++++++- source/dnode/mnode/impl/test/stb/stb.cpp | 24 +++++++++++++++++------- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 09dd7c64d2..38609c2348 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -469,6 +469,7 @@ int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t alterType; + int32_t verInBlock; int32_t numOfFields; SArray* pFields; int32_t ttl; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 278fc2c193..677fbb43b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -600,6 +600,7 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1; + if (tEncodeI32(&encoder, pReq->verInBlock) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1; for (int32_t i = 0; i < pReq->numOfFields; ++i) { SField *pField = taosArrayGet(pReq->pFields, i); @@ -626,6 +627,7 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->verInBlock) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfFields) < 0) return -1; pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField)); if (pReq->pFields == NULL) { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index c8d4f91183..7485510bc6 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -323,7 +323,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->version = pNew->version; pOld->tagVer = pNew->tagVer; pOld->colVer = pNew->colVer; - pOld->colVer = pNew->nextColId; + pOld->nextColId = pNew->nextColId; pOld->ttl = pNew->ttl; pOld->numOfColumns = pNew->numOfColumns; pOld->numOfTags = pNew->numOfTags; @@ -1317,6 +1317,13 @@ static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq) { goto _OVER; } + if (alterReq.verInBlock > 0 && alterReq.verInBlock <= pStb->version) { + mDebug("stb:%s, already exist, verInBlock:%d smaller than verInStb:%d, alter success", alterReq.name, + alterReq.verInBlock, pStb->version); + code = 0; + goto _OVER; + } + pUser = mndAcquireUser(pMnode, pReq->conn.user); if (pUser == NULL) { goto _OVER; diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 16974ad541..b8873210ab 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -32,7 +32,7 @@ class MndTestStb : public ::testing::Test { void* BuildAlterStbUpdateTagBytesReq(const char* stbname, const char* tagname, int32_t bytes, int32_t* pContLen); void* BuildAlterStbAddColumnReq(const char* stbname, const char* colname, int32_t* pContLen); void* BuildAlterStbDropColumnReq(const char* stbname, const char* colname, int32_t* pContLen); - void* BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, int32_t* pContLen); + void* BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, int32_t* pContLen, int32_t verInBlock); }; Testbase MndTestStb::test; @@ -271,12 +271,13 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co } void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, - int32_t* pContLen) { + int32_t* pContLen, int32_t verInBlock) { SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); req.alterType = TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES; + req.verInBlock = verInBlock; SField field = {0}; field.bytes = bytes; @@ -781,31 +782,40 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen, 0); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES); } { - void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen); + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + EXPECT_EQ(test.GetShowRows(), 1); + } + + { + void* pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); From 29bdba7298ce83b17531c73052e090da97a52795 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 19 May 2022 14:26:10 +0800 Subject: [PATCH 100/103] fix: some problems of parser and planner --- include/libs/function/functionMgt.h | 2 ++ include/libs/nodes/cmdnodes.h | 4 ++-- include/util/taoserror.h | 2 ++ include/util/tdef.h | 4 ++-- source/libs/function/src/functionMgt.c | 4 ++++ source/libs/nodes/src/nodesCodeFuncs.c | 21 ++++++++++++++++++++- source/libs/parser/src/parAstCreater.c | 17 ++++++++++++++--- source/libs/parser/src/parTranslater.c | 13 ++++++++++--- source/libs/parser/src/parUtil.c | 4 ++++ source/libs/planner/src/planSpliter.c | 5 ++++- source/libs/planner/test/planSetOpTest.cpp | 9 +++++++++ source/libs/planner/test/planTestUtil.cpp | 1 + 12 files changed, 74 insertions(+), 12 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index aec1476663..89fbc92992 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -142,6 +142,8 @@ void fmFuncMgtDestroy(); int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc); +bool fmIsBuiltinFunc(const char* pFunc); + bool fmIsAggFunc(int32_t funcId); bool fmIsScalarFunc(int32_t funcId); bool fmIsNonstandardSQLFunc(int32_t funcId); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 5e294ae455..82bf4e1f45 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -78,7 +78,7 @@ typedef struct SAlterDatabaseStmt { typedef struct STableOptions { ENodeType type; - char comment[TSDB_STB_COMMENT_LEN]; + char comment[TSDB_TB_COMMENT_LEN]; int32_t delay; float filesFactor; SNodeList* pRollupFuncs; @@ -90,7 +90,7 @@ typedef struct SColumnDefNode { ENodeType type; char colName[TSDB_COL_NAME_LEN]; SDataType dataType; - char comments[TSDB_STB_COMMENT_LEN]; + char comments[TSDB_TB_COMMENT_LEN]; bool sma; } SColumnDefNode; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index be6005d46b..fd3e008e67 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -647,6 +647,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x264A) #define TSDB_CODE_PAR_INVALID_MODIFY_COL TAOS_DEF_ERROR_CODE(0, 0x264B) #define TSDB_CODE_PAR_INVALID_TBNAME TAOS_DEF_ERROR_CODE(0, 0x264C) +#define TSDB_CODE_PAR_INVALID_FUNCTION_NAME TAOS_DEF_ERROR_CODE(0, 0x264D) +#define TSDB_CODE_PAR_COMMENT_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x264E) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tdef.h b/include/util/tdef.h index f95d96be56..5cc687d7ab 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -218,8 +218,8 @@ typedef enum ELogicConditionType { #define TSDB_MAX_SQL_SHOW_LEN 1024 #define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb -#define TSDB_APP_NAME_LEN TSDB_UNI_LEN -#define TSDB_STB_COMMENT_LEN 1024 +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN +#define TSDB_TB_COMMENT_LEN 1025 /** * In some scenarios uint16_t (0~65535) is used to store the row len. diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 73ec7f510b..e46cb81561 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -100,6 +100,10 @@ int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { return getUdfInfo(pParam, pFunc); } +bool fmIsBuiltinFunc(const char* pFunc) { + return NULL != taosHashGet(gFunMgtService.pFuncNameHashTable, pFunc, strlen(pFunc)); +} + EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { if (fmIsUserDefinedFunc(pFunc->funcId) || pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) { return FUNC_DATA_REQUIRED_DATA_LOAD; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 763ccbf7a0..bc49f36afe 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -946,6 +946,23 @@ static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { return code; } +static const char* jkLogicPlanSubplans = "Subplans"; + +static int32_t logicPlanToJson(const void* pObj, SJson* pJson) { + const SQueryLogicPlan* pNode = (const SQueryLogicPlan*)pObj; + return tjsonAddObject(pJson, jkLogicPlanSubplans, nodeToJson, nodesListGetNode(pNode->pTopSubplans, 0)); +} + +static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { + SQueryLogicPlan* pNode = (SQueryLogicPlan*)pObj; + SNode* pChild = NULL; + int32_t code = jsonToNodeObject(pJson, jkLogicPlanSubplans, &pChild); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pNode->pTopSubplans, pChild); + } + return code; +} + static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; @@ -3029,7 +3046,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN: - break; + return logicPlanToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return physiTagScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: @@ -3126,6 +3143,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicPartitionNode(pJson, pObj); case QUERY_NODE_LOGIC_SUBPLAN: return jsonToLogicSubplan(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN: + return jsonToLogicPlan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a438a1e843..9a9a11c7b0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -196,6 +196,15 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { return true; } +static bool checkComment(SAstCreateContext* pCxt, const SToken* pCommentToken, bool demand) { + if (NULL == pCommentToken) { + pCxt->errCode = demand ? TSDB_CODE_PAR_SYNTAX_ERROR : TSDB_CODE_SUCCESS; + } else if (pCommentToken->n >= (TSDB_TB_COMMENT_LEN + 2)) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_COMMENT_TOO_LONG); + } + return TSDB_CODE_SUCCESS == pCxt->errCode; +} + SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode) { SRawExprNode* target = (SRawExprNode*)nodesMakeNode(QUERY_NODE_RAW_EXPR); CHECK_OUT_OF_MEM(target); @@ -823,8 +832,10 @@ SNode* createAlterTableOptions(SAstCreateContext* pCxt) { SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal) { switch (type) { case TABLE_OPTION_COMMENT: - copyStringFormStringToken((SToken*)pVal, ((STableOptions*)pOptions)->comment, - sizeof(((STableOptions*)pOptions)->comment)); + if (checkComment(pCxt, (SToken*)pVal, true)) { + copyStringFormStringToken((SToken*)pVal, ((STableOptions*)pOptions)->comment, + sizeof(((STableOptions*)pOptions)->comment)); + } break; case TABLE_OPTION_DELAY: ((STableOptions*)pOptions)->delay = strtol(((SToken*)pVal)->z, NULL, 10); @@ -848,7 +859,7 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType } SNode* createColumnDefNode(SAstCreateContext* pCxt, SToken* pColName, SDataType dataType, const SToken* pComment) { - if (!checkColumnName(pCxt, pColName)) { + if (!checkColumnName(pCxt, pColName) || !checkComment(pCxt, pComment, false)) { return NULL; } SColumnDefNode* pCol = (SColumnDefNode*)nodesMakeNode(QUERY_NODE_COLUMN_DEF); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18e0843e92..b270e811d9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -272,6 +272,10 @@ static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isScanPseudoColumnFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsScanPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isDistinctOrderBy(STranslateContext* pCxt) { return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); } @@ -892,7 +896,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { return DEAL_RES_IGNORE_CHILD; } } - if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pCxt->selectFuncNum > 1) { return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); } else { @@ -930,7 +934,7 @@ static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) { if (isAggFunc(*pNode)) { return DEAL_RES_IGNORE_CHILD; } - if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { return rewriteColToSelectValFunc((STranslateContext*)pContext, NULL, pNode); } return DEAL_RES_CONTINUE; @@ -958,7 +962,7 @@ static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (QUERY_NODE_COLUMN == nodeType(pNode)) { + if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -3261,6 +3265,9 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { } static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionStmt* pStmt) { + if (fmIsBuiltinFunc(pStmt->funcName)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FUNCTION_NAME); + } SCreateFuncReq req = {0}; strcpy(req.name, pStmt->funcName); req.igExists = pStmt->ignoreExists; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1ae066a8c2..7b9147beab 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -160,6 +160,10 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Only binary/nchar column length could be modified"; case TSDB_CODE_PAR_INVALID_TBNAME: return "Invalid tbname pseudo column"; + case TSDB_CODE_PAR_INVALID_FUNCTION_NAME: + return "Invalid function name"; + case TSDB_CODE_PAR_COMMENT_TOO_LONG: + return "Comment too long"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d0ac239bcb..fb716af3e5 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -24,6 +24,7 @@ #define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) typedef struct SSplitContext { + int32_t queryId; int32_t groupId; bool split; } SSplitContext; @@ -62,6 +63,7 @@ static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* if (NULL == pSubplan) { return NULL; } + pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); @@ -242,6 +244,7 @@ static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) if (NULL == pSubplan) { return NULL; } + pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = pNode; @@ -406,7 +409,7 @@ static const SSplitRule splitRuleSet[] = {{.pName = "SuperTableScan", .splitFunc static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); static int32_t applySplitRule(SLogicSubplan* pSubplan) { - SSplitContext cxt = {.groupId = pSubplan->id.groupId + 1, .split = false}; + SSplitContext cxt = {.queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false}; do { cxt.split = false; for (int32_t i = 0; i < splitRuleNum; ++i) { diff --git a/source/libs/planner/test/planSetOpTest.cpp b/source/libs/planner/test/planSetOpTest.cpp index 3c7bf9e43a..717384aae6 100644 --- a/source/libs/planner/test/planSetOpTest.cpp +++ b/source/libs/planner/test/planSetOpTest.cpp @@ -32,6 +32,15 @@ TEST_F(PlanSetOpTest, unionAllSubquery) { run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION ALL SELECT c1, c2 FROM t1)"); } +TEST_F(PlanSetOpTest, unionAllWithSubquery) { + useDb("root", "test"); + + // child table + run("SELECT ts FROM (SELECT ts FROM st1s1) UNION ALL SELECT ts FROM (SELECT ts FROM st1s2)"); + // super table + run("SELECT ts FROM (SELECT ts FROM st1) UNION ALL SELECT ts FROM (SELECT ts FROM st1)"); +} + TEST_F(PlanSetOpTest, union) { useDb("root", "test"); diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index af8ec87158..94a28f46a8 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -322,6 +322,7 @@ class PlannerTestBaseImpl { } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { + pCxt->queryId = 1; if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; pCxt->topicQuery = true; From f81f0978434efead48baf05c06dffd7e993db8bd Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 14:31:59 +0800 Subject: [PATCH 101/103] fix(os): file path format error --- source/libs/tfs/src/tfs.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 18a3a28bab..0054a94b80 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -160,7 +160,12 @@ bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { if (pFile1 == NULL || pFile2 == NULL || pFile1->pTfs != pFile2->pTfs) return false; if (pFile1->did.level != pFile2->did.level) return false; if (pFile1->did.id != pFile2->did.id) return false; - if (strncmp(pFile1->rname, pFile2->rname, TSDB_FILENAME_LEN) != 0) return false; + char nameBuf1[TMPNAME_LEN], nameBuf2[TMPNAME_LEN]; + memset(nameBuf1, 0, TMPNAME_LEN); + memset(nameBuf2, 0, TMPNAME_LEN); + taosRealPath(pFile1->rname, nameBuf1, TMPNAME_LEN); + taosRealPath(pFile2->rname, nameBuf2, TMPNAME_LEN); + if (strncmp(nameBuf1, nameBuf2, TMPNAME_LEN) != 0) return false; return true; } From 5d1b09be03ccd83c4bf8d4cee67a857ada13d46c Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 14:41:42 +0800 Subject: [PATCH 102/103] fix(os): file path format error --- source/libs/tfs/src/tfs.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 0054a94b80..92beeffa0c 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -161,10 +161,12 @@ bool tfsIsSameFile(const STfsFile *pFile1, const STfsFile *pFile2) { if (pFile1->did.level != pFile2->did.level) return false; if (pFile1->did.id != pFile2->did.id) return false; char nameBuf1[TMPNAME_LEN], nameBuf2[TMPNAME_LEN]; - memset(nameBuf1, 0, TMPNAME_LEN); - memset(nameBuf2, 0, TMPNAME_LEN); - taosRealPath(pFile1->rname, nameBuf1, TMPNAME_LEN); - taosRealPath(pFile2->rname, nameBuf2, TMPNAME_LEN); + strncpy(nameBuf1, pFile1->rname, TMPNAME_LEN); + strncpy(nameBuf2, pFile2->rname, TMPNAME_LEN); + nameBuf1[TMPNAME_LEN - 1] = 0; + nameBuf2[TMPNAME_LEN - 1] = 0; + taosRealPath(nameBuf1, NULL, TMPNAME_LEN); + taosRealPath(nameBuf2, NULL, TMPNAME_LEN); if (strncmp(nameBuf1, nameBuf2, TMPNAME_LEN) != 0) return false; return true; } From c6f23145f3be3c72ca700fab54c5a41e8d6374f4 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 19 May 2022 15:18:18 +0800 Subject: [PATCH 103/103] fix(os): file path format error --- source/dnode/vnode/src/meta/metaOpen.c | 1 + source/dnode/vnode/src/tsdb/tsdbOpen.c | 1 + source/dnode/vnode/src/vnd/vnodeOpen.c | 2 ++ tests/tsim/src/simParse.c | 1 + 4 files changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 135f44e939..9a97357b97 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -44,6 +44,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { pMeta->path = (char *)&pMeta[1]; sprintf(pMeta->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, VNODE_META_DIR); + taosRealPath(pMeta->path, NULL, slen); pMeta->pVnode = pVnode; // create path if not created yet diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 180eea3237..fa54c811ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -55,6 +55,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee memcpy(pTsdb->dir, dir, strlen(dir)); pTsdb->path = (char *)&pTsdb[1]; sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); + taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; pTsdb->repoLocked = false; taosThreadMutexInit(&pTsdb->mutex, NULL); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 739f7f9fa3..9412c39b27 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -109,6 +109,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open wal sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR); + taosRealPath(tdir, NULL, sizeof(tdir)); pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { vError("vgId:%d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); @@ -117,6 +118,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open tq sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR); + taosRealPath(tdir, NULL, sizeof(tdir)); pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); if (pVnode->pTq == NULL) { vError("vgId:%d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index a0721941e3..638c4a1ccb 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -183,6 +183,7 @@ SScript *simParseScript(char *fileName) { strcpy(name, fileName); } else { sprintf(name, "%s" TD_DIRSEP "%s", simScriptDir, fileName); + taosRealPath(name, NULL, sizeof(name)); } // if ((fd = fopen(name, "r")) == NULL) {