From 7a8f8e11a5d0727ceec022231f49f49d84d35d1e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 16:00:35 +0800 Subject: [PATCH 01/63] feat: support update in mem/file --- include/common/tdataformat.h | 6 +- include/common/trow.h | 49 ++-- include/util/tdef.h | 2 + source/common/src/trow.c | 273 ++++++++++++++------- source/common/src/ttypes.c | 2 +- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 7 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 257 ++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 38 ++- source/dnode/vnode/src/tsdb/tsdbSma.c | 9 +- 11 files changed, 460 insertions(+), 196 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 36286d5944..0472c7ea52 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,7 +27,7 @@ extern "C" { // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TD_SUPPORT_READ2 +#undef TD_SUPPORT_READ2 #define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) @@ -387,8 +387,6 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; - - #define isAllRowsNull(pCol) ((pCol)->len == 0) #define isAllRowsNone(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } @@ -482,7 +480,7 @@ void tdResetDataCols(SDataCols *pCols); int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| diff --git a/include/common/trow.h b/include/common/trow.h index 02f84b372e..fe1cf0d967 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -42,7 +42,7 @@ extern "C" { * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available */ -#define TD_VTYPE_NORM 0x00U // normal val: not none, not null(no need assign value) +#define TD_VTYPE_NORM 0x00U // normal val: not none, not null #define TD_VTYPE_NULL 0x01U // null val #define TD_VTYPE_NONE 0x02U // none or unknown/undefined #define TD_VTYPE_MAX 0x03U // @@ -122,6 +122,8 @@ typedef struct { typedef struct { /// timestamp TSKEY ts; + /// row version + uint64_t ver; union { /// union field for encode and decode uint32_t info; @@ -140,8 +142,6 @@ typedef struct { }; /// row total length uint32_t len; - /// row version - uint64_t ver; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -241,13 +241,13 @@ static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colI static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, - int8_t bitmapMode); + int8_t bitmapMode, bool isMerge); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); /** * @brief @@ -622,7 +622,6 @@ static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols return TSDB_CODE_SUCCESS; } - /** * @brief To judge row type: STpRow/SKvRow * @@ -719,6 +718,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } + return TSDB_CODE_SUCCESS; } @@ -758,7 +758,6 @@ static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } - /** * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数 * @@ -1250,16 +1249,16 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in } /** - * @brief - * - * @param pRow - * @param colId - * @param colType - * @param flen - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param colType + * @param flen + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1273,14 +1272,14 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t } /** - * @brief - * - * @param pRow - * @param colId - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1397,14 +1396,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } } -static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) { +static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>", tag); for (int i = 0; i < pSchema->numOfCols; ++i) { STColumn *stCol = pSchema->columns + i; - SCellVal sVal = { 255, NULL}; + SCellVal sVal = {255, NULL}; if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { break; } diff --git a/include/util/tdef.h b/include/util/tdef.h index 316640797b..0ab277e0c3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -29,6 +29,8 @@ extern "C" { #define TSKEY_MAX (INT64_MAX - 1) #define TSKEY_INITIAL_VAL TSKEY_MIN +#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle + // Bytes for each type. extern const int32_t TYPE_BYTES[15]; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 7157c1e0f0..9f4f053028 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -33,6 +33,24 @@ const uint8_t tdVTypeByte[2][3] = {{ // declaration static uint8_t tdGetBitmapByte(uint8_t byte); +// static void dataColSetNEleNull(SDataCol *pCol, int nEle); + +/** + * @brief src2 data has more priority than src1 + * + * @param target + * @param src1 + * @param iter1 + * @param limit1 + * @param src2 + * @param iter2 + * @param limit2 + * @param tRows + * @param update + */ +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows, bool update); + // implementation /** * @brief Compress bitmap bytes comprised of 2-bits to counterpart of 1-bit. @@ -229,23 +247,23 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) { void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { int32_t i = 0, j = 0; int32_t nBytes = TD_BITMAP_BYTES(nBits); - int32_t nStrictBytes = nBits / 4; - int32_t nPartialBits = nBits - nStrictBytes * 4; + int32_t nRoundBytes = nBits / 4; + int32_t nRemainderBits = nBits - nRoundBytes * 4; - switch (nPartialBits) { + switch (nRemainderBits) { case 0: // NOTHING TODO break; case 1: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xC0; } break; case 2: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xF0; } break; case 3: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xFC; } break; default: @@ -266,10 +284,6 @@ void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { } } -// static void dataColSetNEleNull(SDataCol *pCol, int nEle); -static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull); - static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; @@ -410,11 +424,12 @@ STSRow *tdRowDup(STSRow *row) { * @param val * @param numOfRows * @param maxPoints - * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) + * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) + * @param isMerge merge to current row * @return int */ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, - int8_t bitmapMode) { + int8_t bitmapMode, bool isMerge) { TASSERT(pCol != NULL); // Assume that the columns not specified during insert/upsert mean None. @@ -436,18 +451,35 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int // 2. later on, considering further optimization, don't save Null/None for VarType. val = getNullValue(pCol->type); } - if (IS_VAR_DATA_TYPE(pCol->type)) { - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); - // Update the length - pCol->len += varDataTLen(val); + if (!isMerge) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + // set offset + pCol->dataOff[numOfRows] = pCol->len; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + // Update the length + pCol->len += varDataTLen(val); + } else { + ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes); + pCol->len += pCol->bytes; + } } else { - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes); - pCol->len += pCol->bytes; + if (IS_VAR_DATA_TYPE(pCol->type)) { + // keep the last offset + // discard the last var data + int32_t lastVarLen = varDataTLen(POINTER_SHIFT(pCol->pData, pCol->dataOff[numOfRows])); + pCol->len -= lastVarLen; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + // Update the length + pCol->len += varDataTLen(val); + } else { + ASSERT(pCol->len - TYPE_BYTES[pCol->type] == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), val, pCol->bytes); + } } + #ifdef TD_SUPPORT_BITMAP tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); #endif @@ -455,8 +487,13 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int } // internal -static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { +#if 0 ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow)); +#endif + + // Multi-Version rows with the same key and different versions supported + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) <= TD_ROW_KEY(pRow)); int rcol = 1; int dcol = 1; @@ -464,12 +501,14 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; continue; } @@ -480,13 +519,15 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; ++rcol; } else if (pRowCol->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; } } @@ -495,7 +536,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols return TSDB_CODE_SUCCESS; } // internal -static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow)); int rcol = 0; @@ -506,12 +547,14 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= tRowCols || rcol >= tSchemaCols) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; continue; } @@ -527,13 +570,15 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; ++rcol; } else if (pIdx->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; } } @@ -548,20 +593,30 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols * @param pRow * @param pSchema * @param pCols - * @param forceSetNull */ -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { if (TD_IS_TP_ROW(pRow)) { - return tdAppendTpRowToDataCol(pRow, pSchema, pCols); + return tdAppendTpRowToDataCol(pRow, pSchema, pCols, isMerge); } else if (TD_IS_KV_ROW(pRow)) { - return tdAppendKvRowToDataCol(pRow, pSchema, pCols); + return tdAppendKvRowToDataCol(pRow, pSchema, pCols, isMerge); } else { ASSERT(0); } return TSDB_CODE_SUCCESS; } -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, +/** + * @brief source data has more priority than target + * + * @param target + * @param source + * @param rowsToMerge + * @param pOffset + * @param update + * @param maxVer + * @return int + */ +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool update, TDRowVerT maxVer) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); @@ -576,17 +631,35 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); // TODO: filter the maxVer - for (int i = 0; i < rowsToMerge; i++) { + TSKEY lastKey = TSKEY_INITIAL_VAL; + for (int i = 0; i < rowsToMerge; ++i) { + bool merge = false; for (int j = 0; j < source->numOfCols; j++) { if (source->cols[j].len > 0 || target->cols[j].len > 0) { SCellVal sVal = {0}; if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) { TASSERT(0); } + + if (j == 0) { + if (lastKey == *(TSKEY *)sVal.val) { + if (!update) { + break; + } + merge = true; + } else if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; + } + + lastKey = *(TSKEY *)sVal.val; + } + tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); + target->bitmapMode, merge); } } + } + if (rowsToMerge > 0) { ++target->numOfRows; } (*pOffset) += rowsToMerge; @@ -596,7 +669,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * int iter1 = 0; tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows, - pTarget->numOfRows + rowsToMerge, forceSetNull); + pTarget->numOfRows + rowsToMerge, update); } tdFreeDataCols(pTarget); @@ -606,68 +679,95 @@ _err: tdFreeDataCols(pTarget); return -1; } - -// src2 data has more priority than src1 +static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, bool isMerge) { + for (int i = 0; i < src->numOfCols; ++i) { + ASSERT(target->cols[i].type == src->cols[i].type); + if (src->cols[i].len > 0 || target->cols[i].len > 0) { + SCellVal sVal = {0}; + if (tdGetColDataOfRow(&sVal, src->cols + i, iter, src->bitmapMode) < 0) { + TASSERT(0); + } + if (isMerge) { + if (!tdValTypeIsNone(sVal.valType)) { + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode, isMerge); + } else { + // Keep the origi value for None + } + } else { + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode, isMerge); + } + } + } +} +/** + * @brief src2 data has more priority than src1 + * + * @param target + * @param src1 + * @param iter1 + * @param limit1 + * @param src2 + * @param iter2 + * @param limit2 + * @param tRows + * @param update + */ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull) { + int limit2, int tRows, bool update) { tdResetDataCols(target); + target->bitmapMode = src1->bitmapMode; ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); + int32_t nRows = 0; - while (target->numOfRows < tRows) { + // TODO: filter the maxVer + // TODO: handle the delete function + TSKEY lastKey = TSKEY_INITIAL_VAL; + while (nRows < tRows) { if (*iter1 >= limit1 && *iter2 >= limit2) break; TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); - TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); + // TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); // TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); - ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - // TODO: filter the maxVer - if (key1 < key2) { - for (int i = 0; i < src1->numOfCols; ++i) { - ASSERT(target->cols[i].type == src1->cols[i].type); - if (src1->cols[i].len > 0 || target->cols[i].len > 0) { - SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } - } + // ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - ++target->numOfRows; + if (key1 <= key2) { + // select key1 if not delete + if (update && (lastKey == key1)) { + tdAppendValToDataCols(target, src1, *iter1, true); + } else if (lastKey != key1) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; + } + tdAppendValToDataCols(target, src1, *iter1, false); + } + ++nRows; ++(*iter1); - } else if (key1 >= key2) { - // TODO: filter the maxVer - if ((key1 > key2) || ((key1 == key2) && !TKEY_IS_DELETED(key2))) { - for (int i = 0; i < src2->numOfCols; ++i) { - SCellVal sVal = {0}; - ASSERT(target->cols[i].type == src2->cols[i].type); - if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) { - TASSERT(0); - } - if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) { - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } else if (target->cols[i].len > 0) { - dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode); - } + lastKey = key1; + } else { + // use key2 if not deleted + // TODO: handle the delete function + if (update && (lastKey == key2)) { + tdAppendValToDataCols(target, src2, *iter2, true); + } else if (lastKey != key2) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; } - ++target->numOfRows; + tdAppendValToDataCols(target, src2, *iter2, false); } + ++nRows; ++(*iter2); - if (key1 == key2) ++(*iter1); + lastKey = key2; } - ASSERT(target->numOfRows <= target->maxPoints); + ASSERT(target->numOfRows <= target->maxPoints - 1); + } + if (nRows > 0) { + ++target->numOfRows; } } @@ -777,7 +877,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->sversion = pDataCols->sversion; if (keepData) pRet->numOfRows = pDataCols->numOfRows; - for (int i = 0; i < pDataCols->numOfCols; i++) { + for (int i = 0; i < pDataCols->numOfCols; ++i) { pRet->cols[i].type = pDataCols->cols[i].type; pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; pRet->cols[i].colId = pDataCols->cols[i].colId; @@ -797,8 +897,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); } if (!TD_COL_ROWS_NORM(pRet->cols + i)) { - int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows); - memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); + memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, TD_BITMAP_BYTES(pDataCols->numOfRows)); } } } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index e3d67cd488..3fd2ef8e73 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -557,7 +557,7 @@ static const void *nullValues[] = { }; const void *getNullValue(int32_t type) { - assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); // TODO: extend the types return nullValues[type - 1]; } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 078a493773..ffa6f81c0d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -274,7 +274,8 @@ typedef enum { typedef struct { uint8_t last : 1; - uint8_t blkVer : 7; + uint8_t hasDupKey : 1; // 0: no dup TS key, 1: has dup TS key(since supporting Multi-Version) + uint8_t blkVer : 6; uint8_t numOfSubBlocks; col_id_t numOfCols; // not including timestamp column uint32_t len; // data block length diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index fc23534247..754023c49d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1360,7 +1360,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt TASSERT(0); } tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, - pTarget->bitmapMode); + pTarget->bitmapMode, false); } ++pTarget->numOfRows; @@ -1371,7 +1371,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendSTSRowToDataCol(row, pSchema, pTarget); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); tSkipListIterNext(pCommitIter->pIter); } else { @@ -1409,7 +1409,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt } // TODO: tdAppendValToDataCol may fail tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, - pTarget->bitmapMode); + pTarget->bitmapMode, false); } if (TD_SUPPORT_UPDATE(update)) { @@ -1427,9 +1427,9 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter); if (key2 < TD_ROW_KEY(nextRow)) { - tdAppendSTSRowToDataCol(row, pSchema, pTarget); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); } else { - tdAppendSTSRowToDataCol(row, pSchema, pTarget); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); } // TODO: merge with Multi-Version } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ff4d99f510..ebe5ce9a8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -256,9 +256,12 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { pTbData->keyMin = TSKEY_MAX; pTbData->keyMax = TSKEY_MIN; pTbData->nrows = 0; - +#if 0 pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); +#endif + pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, + tsdbGetTsTupleKey); if (pTbData->pData == NULL) { taosMemoryFree(pTbData); return NULL; @@ -303,7 +306,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } } - tdAppendSTSRowToDataCol(row, *ppSchema, pCols); + tdAppendSTSRowToDataCol(row, *ppSchema, pCols, false); } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b6dcc52b5e..72fada4e97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -163,7 +163,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); // static void* destroyTableCheckInfo(SArray* pTableCheckInfo); static bool tsdbGetExternalRow(tsdbReaderT pHandle); -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions); +static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -351,36 +351,43 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } -#if 0 -int nQUERY = 0; +#if 1 +int nQUERY = 0; #endif -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { +static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { int level = 0; -#if 1 +#if 0 int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + i; if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { break; } + ++level; } #endif -#if 0 - ++nQUERY; - if(nQUERY%3 == 0) { - level = 2; - } else if(nQUERY%2 == 0) { - level = 1; - } else { - level = 0; +#if 1 + switch ((nQUERY++) % 3) { + case 0: + level = 0; + break; + case 1: + level = 1; + break; + default: + level = 2; + break; } #endif if (level == TSDB_RETENTION_L0) { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA1(pVnode); } else { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA2(pVnode); } } @@ -393,7 +400,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* goto _end; } - STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); + STsdb* pTsdb = getTsdbByRetentions(pVnode, pReadHandle, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); pReadHandle->order = pCond->order; pReadHandle->pTsdb = pTsdb; @@ -803,12 +810,15 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { +static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) { STSRow *rmem = NULL, *rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_KEY(rmem) > maxVer) { + rmem = NULL; + } } } @@ -816,6 +826,9 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_KEY(rimem) > maxVer) { + rimem = NULL; + } } } @@ -837,6 +850,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, TSKEY r2 = TD_ROW_KEY(rimem); if (r1 == r2) { +#if 0 if (update == TD_ROW_DISCARD_UPDATE) { pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; tSkipListIterNext(pCheckInfo->iter); @@ -846,6 +860,13 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, } else { pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; } +#endif + if (TD_SUPPORT_UPDATE(update)) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + } else { + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + tSkipListIterNext(pCheckInfo->iter); + } return r1; } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) { pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; @@ -856,12 +877,16 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, } } -static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow) { + +static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow, TDRowVerT maxVer) { STSRow *rmem = NULL, *rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_VER(rmem) > maxVer) { + rmem = NULL; + } } } @@ -869,6 +894,9 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_VER(rimem) > maxVer) { + rimem = NULL; + } } } @@ -890,6 +918,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int TSKEY r2 = TD_ROW_KEY(rimem); if (r1 == r2) { +#if 0 if (update == TD_ROW_DISCARD_UPDATE) { tSkipListIterNext(pCheckInfo->iter); pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; @@ -903,6 +932,16 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int *extraRow = rimem; return rmem; } +#endif + if (TD_SUPPORT_UPDATE(update)) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + *extraRow = rimem; + return rmem; + } else { + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + return rimem; + } } else { if (ASCENDING_TRAVERSE(order)) { if (r1 < r2) { @@ -973,7 +1012,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL); + STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL, TD_VER_MAX); if (row == NULL) { return false; } @@ -1250,7 +1289,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); + key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update, TD_VER_MAX); if (key != TSKEY_INITIAL_VAL) { tsdbDebug("%p key in mem:%" PRId64 ", %s", pTsdbReadHandle, key, pTsdbReadHandle->idStr); @@ -1497,10 +1536,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t TASSERT(0); } - if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, rowIndex); - } else { + if (sVal.valType == TD_VTYPE_NORM) { colDataAppend(pColInfo, rowIndex, sVal.val, false); + } else { + colDataAppendNULL(pColInfo, rowIndex); } } } else { // handle the var-string @@ -1513,10 +1552,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t TASSERT(0); } - if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, rowIndex); - } else { + if (sVal.valType == TD_VTYPE_NORM) { colDataAppend(pColInfo, rowIndex, sVal.val, false); + } else { + colDataAppendNULL(pColInfo, rowIndex); } } } @@ -1541,11 +1580,26 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t return numOfRows + num; } -// TODO fix bug for reverse copy data problem -// Note: row1 always has high priority -static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1, - STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, - bool forceSetNull) { +/** + * @brief // TODO fix bug for reverse copy data problem + * Note: row1 always has high priority + * + * @param pTsdbReadHandle + * @param capacity + * @param curRow + * @param row1 + * @param row2 + * @param numOfCols + * @param uid + * @param pSchema1 + * @param pSchema2 + * @param update + * @param lastRowKey + * @return int32_t The quantity of rows appended + */ +static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t* curRow, STSRow* row1, + STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, + bool update, TSKEY* lastRowKey) { #if 1 STSchema* pSchema; STSRow* row; @@ -1557,12 +1611,16 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit bool isChosenRowDataRow; int32_t chosen_itr; SCellVal sVal = {0}; + TSKEY rowKey = TSKEY_INITIAL_VAL; + int32_t nResult = 0; // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); + // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); + // TODO: use the real schemaVersion + pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); } #ifdef TD_DEBUG_PRINT_ROW @@ -1579,7 +1637,9 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit if (row2) { isRow2DataRow = TD_IS_TP_ROW(row2); if (pSchema2 == NULL) { - pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); + // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); + // TODO: use the real schemaVersion + pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); } if (isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1610,19 +1670,19 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit colIdOfRow2 = tdKvRowColIdAt(row2, k); } - if (colIdOfRow1 == colIdOfRow2) { + if (colIdOfRow1 < colIdOfRow2) { // the most probability if (colIdOfRow1 < pColInfo->info.colId) { - j++; - k++; + ++j; continue; } row = row1; pSchema = pSchema1; isChosenRowDataRow = isRow1DataRow; chosen_itr = j; - } else if (colIdOfRow1 < colIdOfRow2) { + } else if (colIdOfRow1 == colIdOfRow2) { if (colIdOfRow1 < pColInfo->info.colId) { - j++; + ++j; + ++k; continue; } row = row1; @@ -1631,7 +1691,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit chosen_itr = j; } else { if (colIdOfRow2 < pColInfo->info.colId) { - k++; + ++k; continue; } row = row2; @@ -1639,16 +1699,35 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit chosen_itr = k; isChosenRowDataRow = isRow2DataRow; } + if (isChosenRowDataRow) { colId = pSchema->columns[chosen_itr].colId; offset = pSchema->columns[chosen_itr].offset; // TODO: use STSRowIter tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr - 1, &sVal); + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + rowKey = *(TSKEY*)sVal.val; + if (rowKey != *lastRowKey) { + if (*lastRowKey != TSKEY_INITIAL_VAL) { + ++(*curRow); + } + ++nResult; + } + *lastRowKey = rowKey; + } } else { // TODO: use STSRowIter if (chosen_itr == 0) { colId = PRIMARYKEY_TIMESTAMP_COL_ID; tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); + rowKey = *(TSKEY*)sVal.val; + if (rowKey != *lastRowKey) { + if (*lastRowKey != TSKEY_INITIAL_VAL) { + ++(*curRow); + } + ++nResult; + } + *lastRowKey = rowKey; } else { SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1); colId = pColIdx->colId; @@ -1657,35 +1736,46 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit } } + ASSERT(rowKey != TSKEY_INITIAL_VAL); + if (colId == pColInfo->info.colId) { if (tdValTypeIsNorm(sVal.valType)) { - colDataAppend(pColInfo, numOfRows, sVal.val, false); - } else if (forceSetNull) { - colDataAppend(pColInfo, numOfRows, NULL, true); + colDataAppend(pColInfo, *curRow, sVal.val, false); + } else if (tdValTypeIsNull(sVal.valType)) { + colDataAppend(pColInfo, *curRow, NULL, true); + } else if (tdValTypeIsNone(sVal.valType)) { + // TODO: Set null if nothing append for this row + if (*lastRowKey != rowKey) { + colDataAppend(pColInfo, *curRow, NULL, true); + } + } else { + ASSERT(0); } - i++; + ++i; if (row == row1) { - j++; + ++j; } else { - k++; + ++k; } } else { - if (forceSetNull) { - colDataAppend(pColInfo, numOfRows, NULL, true); + if (*lastRowKey != rowKey) { + colDataAppend(pColInfo, *curRow, NULL, true); } - i++; + ++i; } } - if (forceSetNull) { + if (*lastRowKey != rowKey) { while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - colDataAppend(pColInfo, numOfRows, NULL, true); - i++; + colDataAppend(pColInfo, *curRow, NULL, true); + ++i; } } + + return nResult; #endif } @@ -1859,6 +1949,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; + int32_t curRow = 0; int16_t rv1 = -1; int16_t rv2 = -1; @@ -1874,9 +1965,11 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + do { STSRow* row2 = NULL; - STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2); + STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2, TD_VER_MAX); if (row1 == NULL) { break; } @@ -1905,9 +1998,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf rv2 = TD_ROW_SVER(row2); } - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true); - numOfRows += 1; + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + // numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; } @@ -1918,6 +2011,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it +#if 0 if (pCfg->update) { if (pCfg->update == TD_ROW_PARTIAL_UPDATE) { doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos); @@ -1933,7 +2027,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull); + pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull, &lastRowKey); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1943,6 +2037,35 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->lastKey = key + step; cur->mixBlock = true; + moveToNextRowInMem(pCheckInfo); + pos += step; + } else { + moveToNextRowInMem(pCheckInfo); + } +#endif + if (TD_SUPPORT_UPDATE(pCfg->update)) { + doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); + + if (rv1 != TD_ROW_SVER(row1)) { + // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + rv1 = TD_ROW_SVER(row1); + } + if (row2 && rv2 != TD_ROW_SVER(row2)) { + // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + rv2 = TD_ROW_SVER(row2); + } + + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + // ++numOfRows; + if (cur->win.skey == TSKEY_INITIAL_VAL) { + cur->win.skey = key; + } + + cur->win.ekey = key; + cur->lastKey = key + step; + cur->mixBlock = true; + moveToNextRowInMem(pCheckInfo); pos += step; } else { @@ -1958,11 +2081,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf assert(end != -1); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it +#if 0 if (pCfg->update == TD_ROW_DISCARD_UPDATE) { moveToNextRowInMem(pCheckInfo); } else { end -= step; } +#endif + if (!TD_SUPPORT_UPDATE(pCfg->update)) { + moveToNextRowInMem(pCheckInfo); + } else { + end -= step; + } } int32_t qstart = 0, qend = 0; @@ -2572,6 +2702,7 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) { static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; + int curRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; @@ -2579,9 +2710,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int int64_t st = taosGetTimestampUs(); int16_t rv = -1; STSchema* pSchema = NULL; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + do { - STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL); + STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL, TD_VER_MAX); if (row == NULL) { break; } @@ -2604,16 +2737,18 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); rv = TD_ROW_SVER(row); } - mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, - NULL, true); + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, + NULL, true, &lastRowKey); - if (++numOfRows >= maxRowsToRead) { + if (numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); break; } } while (moveToNextRowInMem(pCheckInfo)); + taosMemoryFreeClear(pSchema); // free the STSChema + assert(numOfRows <= maxRowsToRead); // if the buffer is not full in case of descending order query, move the data in the front of the buffer @@ -2731,6 +2866,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { STSRow* pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + int32_t curRow = 0; if (++pTsdbReadHandle->activeIndex < numOfTables) { STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); @@ -2738,8 +2875,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // if (ret != TSDB_CODE_SUCCESS) { // return false; // } - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId, - NULL, NULL, true); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, pRow, NULL, numOfCols, pCheckInfo->tableId, + NULL, NULL, true, &lastRowKey); taosMemoryFreeClear(pRow); // update the last key value diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index d7c9a70c18..8f2013e8e0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -266,11 +266,22 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, - update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0) + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; } + // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line + if (pBlock->numOfSubBlocks == 1) { + tdResetDataCols(pReadh->pDCols[1]); + pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode; + if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL, + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { + return -1; + } + memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + ASSERT(pReadh->pDCols[0]->bitmapMode != 0); + } - ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); @@ -297,9 +308,20 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, - update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0) + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; } + // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line + if (pBlock->numOfSubBlocks == 1) { + tdResetDataCols(pReadh->pDCols[1]); + pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode; + if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL, + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { + return -1; + } + memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + ASSERT(pReadh->pDCols[0]->bitmapMode != 0); + } if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { @@ -312,7 +334,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } } - ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); @@ -623,15 +645,15 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat } if (dcol != 0) { - ccol++; + ++ccol; } - dcol++; + ++dcol; } else if (tcolId < pDataCol->colId) { - ccol++; + ++ccol; } else { // Set current column as NULL and forward dataColReset(pDataCol); - dcol++; + ++dcol; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 362e31ff61..83adb545ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1635,17 +1635,20 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { SSmaCfg vCreateSmaReq = {0}; if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); + tsdbWarn("vgId:%d tsma create msg received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); return -1; } - tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName, - vCreateSmaReq.tSma.indexUid); + + tsdbDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", REPO_ID(pTsdb), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); // record current timezone of server side vCreateSmaReq.tSma.timezoneInt = tsTimezone; if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) { // TODO: handle error + tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); tdDestroyTSma(&vCreateSmaReq.tSma); return -1; } From 319d2785abd24ec3ca82af084d2990258bf79027 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 18:33:31 +0800 Subject: [PATCH 02/63] feat: update support merge in mem --- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 72fada4e97..7f733e0e4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1613,6 +1613,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa SCellVal sVal = {0}; TSKEY rowKey = TSKEY_INITIAL_VAL; int32_t nResult = 0; + bool isMerge = true; // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; @@ -1708,6 +1709,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { + isMerge = false; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } @@ -1722,6 +1724,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { + isMerge = false; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } @@ -1745,7 +1748,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa colDataAppend(pColInfo, *curRow, NULL, true); } else if (tdValTypeIsNone(sVal.valType)) { // TODO: Set null if nothing append for this row - if (*lastRowKey != rowKey) { + if (!isMerge) { colDataAppend(pColInfo, *curRow, NULL, true); } } else { @@ -1760,7 +1763,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa ++k; } } else { - if (*lastRowKey != rowKey) { + if (!isMerge) { colDataAppend(pColInfo, *curRow, NULL, true); } ++i; From 140a7458c3b8068ea68eb0407d66eaf036ac31be Mon Sep 17 00:00:00 2001 From: cpwu Date: Thu, 5 May 2022 20:08:23 +0800 Subject: [PATCH 03/63] fix join case --- tests/system-test/2-query/join.py | 144 +++++++++++++++++++++--------- 1 file changed, 103 insertions(+), 41 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index b4878e42c9..c6431cdc8d 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -5,6 +5,7 @@ from util.sql import * from util.cases import * from util.dnodes import * +PRIMARY_COL = "ts" INT_COL = "c1" BINT_COL = "c2" @@ -18,9 +19,10 @@ BINARY_COL = "c8" NCHAR_COL = "c9" TS_COL = "c10" -UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] -TS_TYPE_COL = [TS_COL] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] class TDTestCase: @@ -28,50 +30,78 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - def __length_condition(self): - length_condition = [] + def __query_condition(self,tbname): + query_condition = [] for char_col in CHAR_COL: - length_condition.extend( + query_condition.extend( ( - char_col, - f"upper( {char_col} )", + f"{tbname}.{char_col}", + f"upper( {tbname}.{char_col} )", ) ) - length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) - length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) - length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL ) + for num_col in NUM_COL: + query_condition.extend( + ( + f"{tbname}.{num_col}", + f"sin( {tbname}.{num_col} )" + ) + ) + query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL ) - length_condition.append('''"test1234!@#$%^&*():'> 0 " - return "" + def __join_condition(self, tb_list, filter=PRIMARY_COL): + # sourcery skip: flip-comparison + if 1 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[0]}.{filter} " + elif 2 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + else: + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + for i in range(1, len(tb_list)-1 ): + join_filter += f"and {tb_list[i]}.{filter} = {tb_list[i+1]}.{filter}" - def __group_condition(self, col, having = ""): - return f" group by {col} having {having}" if having else f" group by {col} " + return join_filter - def __length_current_check(self, tbname): - length_condition = self.__length_condition() - for condition in length_condition: - where_condition = self.__where_condition(condition) - group_having = self.__group_condition(condition, having=f"{condition} is not null " ) - group_no_having= self.__group_condition(condition ) + def __where_condition(self, col, tbname): + if col in NUM_COL: + return f" abs( {tbname}.{col} ) >= 0" + elif col in CHAR_COL: + return f" lower( {tbname}.{col} ) is not null" + elif col in BOOLEAN_COL: + return f" {tbname}.{col} in (false, true) " + elif col in TS_TYPE_COL or col in PRIMARY_COL: + return f" abs( cast( {tbname}.{col} as bigint ) ) >= 0 " + else: + return "" + + def __group_condition(self, tbname, col, having = ""): + return f" group by {tbname}.{col} having {having}" if having else f" group by {tbname}.{col} " + + def __join_check(self, tblist, checkrows, join_flag=True): + query_conditions = self.__query_condition(tblist[0]) + join_condition = self.__join_condition(tb_list=tblist) if join_flag else " " + for condition in query_conditions: + where_condition = self.__where_condition(col=condition, tbname=tblist[0]) + group_having = self.__group_condition(tbname=tblist[0], col=condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(tbname=tblist[0], col=condition ) groups = ["", group_having, group_no_having] - for group_condition in groups: - tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") - datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] - length_data = [ len(str(data)) if data else None for data in datas ] - tdSql.query(f"select length( {condition} ) from {tbname} {where_condition} {group_condition}") - for i in range(len(length_data)): - tdSql.checkData(i, 0, length_data[i] ) if length_data[i] else tdSql.checkData(i, 0, None) + sql = f"select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition}" + if len(tblist) == 2: + self.__join_current(sql, checkrows) + elif len(tblist) > 2 or len(tblist) < 1: + tdSql.error(sql=sql) - def __length_err_check(self,tbname): + def __join_err_check(self,tbname): sqls = [] - for un_char_col in UN_CHAR_COL: + for un_char_col in NUM_COL: sqls.extend( ( f"select length( {un_char_col} ) from {tbname} ", @@ -80,12 +110,12 @@ class TDTestCase: ) ) - sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) + sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) + sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) @@ -101,15 +131,46 @@ class TDTestCase: return sqls + def __join_current(self, sql, checkrows): + tdSql.query(sql=sql) + tdSql.checkRows(checkrows) + + def __test_current(self): + # sourcery skip: extract-duplicate-method, inline-immediately-returned-variable tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") - tbname = ["ct1", "ct2", "ct4", "t1"] - for tb in tbname: - self.__length_current_check(tb) - tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + tblist_1 = ["ct1", "ct2"] + self.__join_check(tblist_1, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========") + tblist_2 = ["ct2", "ct4"] + self.__join_check(tblist_2, self.rows - 3) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========") + tblist_3 = ["t1", "ct4"] + self.__join_check(tblist_3, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_3} over==========") + tblist_4 = ["t1", "ct1"] + self.__join_check(tblist_4, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_4} over==========") def __test_error(self): + # sourcery skip: extract-duplicate-method, move-assign-in-block tdLog.printNoPrefix("==========err sql condition check , must return error==========") + err_list_1 = ["ct1","ct2", "ct4"] + err_list_2 = ["ct1","ct2", "t1"] + err_list_3 = ["ct1","ct4", "t1"] + err_list_4 = ["ct2","ct4", "t1"] + err_list_5 = ["ct1", "ct2","ct4", "t1"] + self.__join_check(err_list_1, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_1} over==========") + self.__join_check(err_list_2, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_2} over==========") + self.__join_check(err_list_3, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_3} over==========") + self.__join_check(err_list_4, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========") + self.__join_check(err_list_5, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========") + tbname = ["ct1", "ct2", "ct4", "t1"] for tb in tbname: @@ -168,7 +229,7 @@ class TDTestCase: tdSql.execute( f'''insert into ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, @@ -184,7 +245,7 @@ class TDTestCase: tdSql.execute( f'''insert into ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, @@ -228,7 +289,8 @@ class TDTestCase: self.__create_tb() tdLog.printNoPrefix("==========step2:insert data") - self.__insert_data(10) + self.rows = 10 + self.__insert_data(self.rows) tdLog.printNoPrefix("==========step3:all check") self.all_test() From 4a649ec925d18d20a1c9e29918c6591357d48dca Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 21:07:55 +0800 Subject: [PATCH 04/63] fix(tsdb): swap pointer of data cols --- source/common/src/trow.c | 5 +++-- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9f4f053028..6391744851 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -24,7 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{ }, { // 1 bit - TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I, + TD_VTYPE_NORM_BYTE_I, // normal + TD_VTYPE_NULL_BYTE_I, TD_VTYPE_NULL_BYTE_I, // padding } @@ -343,7 +344,7 @@ bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode if (*((uint8_t *)pBitmap) != vTypeByte) { return false; } - pBitmap = POINTER_SHIFT(pBitmap, 1); + pBitmap = POINTER_SHIFT(pBitmap, i); } int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 8f2013e8e0..8f68883fde 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -246,6 +246,12 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { return 0; } +static FORCE_INLINE void tsdbSwapDataCols(SDataCols *pDest, SDataCols *pSrc) { + SDataCol *pCols = pDest->cols; + memcpy(pDest, pSrc, sizeof(SDataCols)); + pSrc->cols = pCols; +} + int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { ASSERT(pBlock->numOfSubBlocks > 0); STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo); @@ -277,7 +283,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { return -1; } - memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } @@ -319,10 +325,11 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { return -1; } - memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } + if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; From 934636d23f7b2277a482e86fb658fe4ba43abe3c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 23:25:55 +0800 Subject: [PATCH 05/63] fix(tsdb): update function --- source/common/src/trow.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6391744851..590e6097c4 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -446,23 +446,24 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int dataColSetNEleNone(pCol, numOfRows, bitmapMode); } } - if (!tdValTypeIsNorm(valType)) { + const void *value = val; + if (!tdValTypeIsNorm(valType) || !val) { // TODO: // 1. back compatibility and easy to debug with codes of 2.0 to save NULL values. // 2. later on, considering further optimization, don't save Null/None for VarType. - val = getNullValue(pCol->type); + value = getNullValue(pCol->type); } if (!isMerge) { if (IS_VAR_DATA_TYPE(pCol->type)) { // set offset pCol->dataOff[numOfRows] = pCol->len; // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); // Update the length - pCol->len += varDataTLen(val); + pCol->len += varDataTLen(value); } else { ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; } } else { @@ -472,12 +473,12 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int int32_t lastVarLen = varDataTLen(POINTER_SHIFT(pCol->pData, pCol->dataOff[numOfRows])); pCol->len -= lastVarLen; // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); // Update the length - pCol->len += varDataTLen(val); + pCol->len += varDataTLen(value); } else { ASSERT(pCol->len - TYPE_BYTES[pCol->type] == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), val, pCol->bytes); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), value, pCol->bytes); } } @@ -654,6 +655,9 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * lastKey = *(TSKEY *)sVal.val; } + if (i == 0) { + (target->cols + j)->bitmap = (source->cols + j)->bitmap; + } tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode, merge); From e4f0a0fc676f60b248d616ebe966171716f68c9e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 May 2022 23:47:44 +0800 Subject: [PATCH 06/63] fix(query): enable table name query for super table. --- include/libs/scalar/scalar.h | 2 + source/libs/executor/inc/executorimpl.h | 16 +++-- source/libs/executor/src/executorimpl.c | 26 ++++++-- source/libs/executor/src/scanoperator.c | 82 ++++++++++++++----------- source/libs/function/src/builtins.c | 2 +- source/libs/scalar/src/sclfunc.c | 6 ++ 6 files changed, 82 insertions(+), 52 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 0c7db45c4b..555274599a 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -91,6 +91,8 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bc6139c304..eefa38d802 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -87,9 +87,7 @@ typedef struct SResultInfo { // TODO refactor typedef struct STableQueryInfo { TSKEY lastKey; // last check ts, todo remove it later SResultRowPosition pos; // current active time window -// int32_t groupIndex; // group id in table list // SVariant tag; -// SResultRowInfo resInfo; // result info } STableQueryInfo; typedef enum { @@ -363,11 +361,12 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo *pCols; - SSDataBlock *pRes; - int32_t totalTables; - int32_t curPos; - void *pReader; + SColumnInfo *pCols; + SSDataBlock *pRes; + SArray *pColMatchInfo; + int32_t curPos; + SReadHandle readHandle; + STableGroupInfo *pTableGroups; } STagScanInfo; typedef struct SStreamBlockScanInfo { @@ -704,7 +703,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); @@ -717,7 +716,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); -void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a237eb0e7d..9aa251e1b6 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4739,7 +4739,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { return NULL; @@ -4783,6 +4782,25 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { + STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); + + int32_t code = + doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); + + int32_t numOfOutputCols = 0; + SArray* colList = + extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); + return pOperator; } else { ASSERT(0); } @@ -5088,7 +5106,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchInfo c = {0}; c.output = true; - c.colId = pColNode->colId; + c.colId = pColNode->colId; c.targetSlotId = pNode->slotId; taosArrayPush(pList, &c); } @@ -5166,9 +5184,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* if (code != TSDB_CODE_SUCCESS) { goto _error; } -#if 0 - return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId); -#endif + return tsdbQueryTables(pHandle->vnode, &cond, pTableGroupInfo, queryId, taskId); _error: diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8c9fdfe4e6..b728daa3bb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ -#include "ttime.h" +#include #include "filter.h" #include "function.h" #include "functionMgt.h" #include "os.h" #include "querynodes.h" +#include "systable.h" #include "tglobal.h" #include "tname.h" -#include "systable.h" +#include "ttime.h" #include "tdatablock.h" #include "tmsg.h" @@ -1159,16 +1160,17 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe } static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { -#if 0 if (pOperator->status == OP_EXEC_DONE) { return NULL; } + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + +#if 0 int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; STagScanInfo *pInfo = pOperator->info; SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; int32_t count = 0; SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); @@ -1237,55 +1239,54 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo +#endif - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; + STagScanInfo* pInfo = pOperator->info; + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + SSDataBlock* pRes = pInfo->pRes; - STableQueryInfo* item = taosArrayGetP(pa, i); + SArray* pa = taosArrayGetP(pInfo->pTableGroups->pGroupList, 0); - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfExprs; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { - continue; - } + char str[512] = {0}; + int32_t count = 0; + SMetaReader mr = {0}; - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resSchema.type; - bytes = pExprInfo[j].base.resSchema.bytes; + while (pInfo->curPos < pInfo->pTableGroups->numOfTables && count < pOperator->resultInfo.capacity) { + STableKeyInfo* item = taosArrayGet(pa, pInfo->curPos); - if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); - } + for (int32_t j = 0; j < pOperator->numOfExprs; ++j) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); - dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; - doSetTagValueToResultBuf(dst, data, type, bytes); + // refactor later + if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&mr, item->uid); + + STR_TO_VARSTR(str, mr.me.name); + metaReaderClear(&mr); + + colDataAppend(pDst, count, str, false); + + // data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); + // dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + // doSetTagValueToResultBuf(dst, data, type, bytes); } count += 1; } - if (pInfo->curPos >= pInfo->totalTables) { + if (++pInfo->curPos >= pInfo->pTableGroups->numOfTables) { pOperator->status = OP_EXEC_DONE; } - - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } + // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); + setTaskStatus(pTaskInfo, TASK_COMPLETED); } pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; - -#endif - return TSDB_CODE_SUCCESS; + return (pRes->info.rows == 0) ? NULL : pInfo->pRes; } static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { @@ -1293,14 +1294,18 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, + SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pReader = readHandle; + pInfo->pTableGroups = pTableGroupInfo; + pInfo->pColMatchInfo = pColMatchInfo; + pInfo->pRes = pResBlock; + pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -1308,9 +1313,12 @@ SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExpr; - pOperator->numOfExprs = numOfOutput; + pOperator->numOfExprs = numOfOutput; pOperator->pTaskInfo = pTaskInfo; + initResultSizeInfo(pOperator, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 38922833f9..eac11558cb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -917,7 +917,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = qTbnameFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index a507b41342..4df6148a6e 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1504,3 +1504,9 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t*) colDataGetData(pInput->columnData, 4)); return TSDB_CODE_SUCCESS; } + +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + ASSERT(inputNum == 1); + colDataAppend(pOutput->columnData, pOutput->numOfRows, colDataGetData(pInput->columnData, 0), false); + return TSDB_CODE_SUCCESS; +} From e33586922ab1975ba5690b072d1e491812ae3b01 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:08:33 +0800 Subject: [PATCH 07/63] fix case --- tests/system-test/2-query/join.py | 74 +++++++++++++++++++------------ 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index c6431cdc8d..d66a300afb 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -93,43 +93,48 @@ class TDTestCase: groups = ["", group_having, group_no_having] for group_condition in groups: sql = f"select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition}" + if not join_flag : + tdSql.error(sql=sql) + return if len(tblist) == 2: self.__join_current(sql, checkrows) - elif len(tblist) > 2 or len(tblist) < 1: + return + if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) + return - def __join_err_check(self,tbname): - sqls = [] + # def __join_err_check(self,tbname): + # sqls = [] - for un_char_col in NUM_COL: - sqls.extend( - ( - f"select length( {un_char_col} ) from {tbname} ", - f"select length(ceil( {un_char_col} )) from {tbname} ", - f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", - ) - ) + # for un_char_col in NUM_COL: + # sqls.extend( + # ( + # f"select length( {un_char_col} ) from {tbname} ", + # f"select length(ceil( {un_char_col} )) from {tbname} ", + # f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", + # ) + # ) - sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) - sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) + # sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) - sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) - sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) - sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) - sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) - sqls.extend( - ( - f"select length() from {tbname} ", - f"select length(*) from {tbname} ", - f"select length(ccccccc) from {tbname} ", - f"select length(111) from {tbname} ", - f"select length(c8, 11) from {tbname} ", - ) - ) + # sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) + # sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) + # sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + # sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + # sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + # sqls.extend( + # ( + # f"select length() from {tbname} ", + # f"select length(*) from {tbname} ", + # f"select length(ccccccc) from {tbname} ", + # f"select length(111) from {tbname} ", + # f"select length(c8, 11) from {tbname} ", + # ) + # ) - return sqls + # return sqls def __join_current(self, sql, checkrows): tdSql.query(sql=sql) @@ -170,6 +175,17 @@ class TDTestCase: tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========") self.__join_check(err_list_5, -1) tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========") + self.__join_check(["ct2", "ct4"], -1, join_flag=False) + tdLog.printNoPrefix("==========err sql condition check in has no join condition over==========") + + tdSql.error( f"select c1, c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{INT_COL}=ct4.{INT_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{TS_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct1.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and c1 is not null " ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and ct1.c1 is not null " ) + tbname = ["ct1", "ct2", "ct4", "t1"] From c97518a81eff7f3524d1197b0ab39cd9158b5ca0 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:41:46 +0800 Subject: [PATCH 08/63] fix case --- tests/system-test/2-query/join.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index d66a300afb..733f15bfc6 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -227,18 +227,18 @@ class TDTestCase: now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( f'''insert into ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) @@ -249,11 +249,11 @@ class TDTestCase: ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} ) ( { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} ) ''' ) @@ -265,11 +265,11 @@ class TDTestCase: ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) @@ -277,7 +277,7 @@ class TDTestCase: for i in range(rows): insert_data = f'''insert into t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( @@ -287,12 +287,12 @@ class TDTestCase: ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) From 3f29e09b41a46ce3d9ccd757413648aed87d18f7 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:52:17 +0800 Subject: [PATCH 09/63] fix case --- tests/system-test/2-query/join.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 733f15bfc6..7ed614c7a5 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -72,11 +72,11 @@ class TDTestCase: if col in NUM_COL: return f" abs( {tbname}.{col} ) >= 0" elif col in CHAR_COL: - return f" lower( {tbname}.{col} ) is not null" + return f" lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' " elif col in BOOLEAN_COL: return f" {tbname}.{col} in (false, true) " elif col in TS_TYPE_COL or col in PRIMARY_COL: - return f" abs( cast( {tbname}.{col} as bigint ) ) >= 0 " + return f" cast( {tbname}.{col} as binary(16) ) is not null " else: return "" @@ -92,7 +92,11 @@ class TDTestCase: group_no_having= self.__group_condition(tbname=tblist[0], col=condition ) groups = ["", group_having, group_no_having] for group_condition in groups: - sql = f"select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition}" + if where_condition: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition} " + else: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} {group_condition} " + if not join_flag : tdSql.error(sql=sql) return From d2ef75b578718f9aa4a8b302dadc08339b3b21e1 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:52:59 +0800 Subject: [PATCH 10/63] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 7ed614c7a5..c263a0b412 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -51,7 +51,7 @@ class TDTestCase: ) query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL ) - query_condition.append('''"test1234!@#$%^&*():'> Date: Fri, 6 May 2022 13:57:45 +0800 Subject: [PATCH 11/63] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index c263a0b412..7744360e65 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -101,7 +101,7 @@ class TDTestCase: tdSql.error(sql=sql) return if len(tblist) == 2: - self.__join_current(sql, checkrows) + self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) return if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) From 01ea52e33d26dcdecfba35ab4354755488bf60d3 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:59:19 +0800 Subject: [PATCH 12/63] fix case --- tests/system-test/2-query/join.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 7744360e65..1ab8fe6dd8 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -99,13 +99,10 @@ class TDTestCase: if not join_flag : tdSql.error(sql=sql) - return if len(tblist) == 2: self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) - return if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) - return # def __join_err_check(self,tbname): # sqls = [] From 8ee2ae550f4c02436123457729ee4ebe2b604537 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 14:06:46 +0800 Subject: [PATCH 13/63] fix case --- tests/system-test/2-query/join.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 1ab8fe6dd8..aea454b9b7 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -100,7 +100,10 @@ class TDTestCase: if not join_flag : tdSql.error(sql=sql) if len(tblist) == 2: - self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) + if "ct1" in tblist or "t1" in tblist: + self.__join_current(sql, checkrows) + else: + self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) From f755df6176c2593fb7c198c0468bba583a39dd99 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 14:10:00 +0800 Subject: [PATCH 14/63] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index aea454b9b7..ebaa2e8b9f 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -81,7 +81,7 @@ class TDTestCase: return "" def __group_condition(self, tbname, col, having = ""): - return f" group by {tbname}.{col} having {having}" if having else f" group by {tbname}.{col} " + return f" group by {col} having {having}" if having else f" group by {col} " def __join_check(self, tblist, checkrows, join_flag=True): query_conditions = self.__query_condition(tblist[0]) From cbd55c47a5e2caaf02cc18bb5d9835e1440a52c8 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 14:21:10 +0800 Subject: [PATCH 15/63] fix case --- tests/system-test/2-query/join.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index ebaa2e8b9f..015538ec1e 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,8 +102,10 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) + elif where_condition: + self.__join_current(sql, checkrows + 5 ) else: - self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) + self.__join_current(sql, checkrows + 2 ) if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) From c4b4c008c3bc2b8ece89ecf31e6f70b83d8f178a Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 15:00:23 +0800 Subject: [PATCH 16/63] fix case --- tests/system-test/2-query/join.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 015538ec1e..57e433804d 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -103,9 +103,9 @@ class TDTestCase: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) elif where_condition: - self.__join_current(sql, checkrows + 5 ) - else: self.__join_current(sql, checkrows + 2 ) + else: + self.__join_current(sql, checkrows + 5 ) if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) @@ -154,7 +154,7 @@ class TDTestCase: self.__join_check(tblist_1, 1) tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========") tblist_2 = ["ct2", "ct4"] - self.__join_check(tblist_2, self.rows - 3) + self.__join_check(tblist_2, self.rows) tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========") tblist_3 = ["t1", "ct4"] self.__join_check(tblist_3, 1) From 1436796da72e4775fcce02cefef2984fcb1b30c6 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 15:20:26 +0800 Subject: [PATCH 17/63] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 57e433804d..61c0ecbf2c 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,7 +102,7 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) - elif where_condition: + elif where_condition or "null" in groups: self.__join_current(sql, checkrows + 2 ) else: self.__join_current(sql, checkrows + 5 ) From 666b0b676dacec6a56e40fc972d86926f09d1627 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 15:30:00 +0800 Subject: [PATCH 18/63] fix: alter db replications --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 -- source/dnode/mnode/impl/src/mndDb.c | 50 ++++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e3816396c9..9329e7b15d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -293,14 +293,10 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); // sync integration dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index af7efb5543..96a8969b56 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -627,6 +627,7 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { pDb->cfg.replications = pAlter->replications; + pDb->vgVersion++; terrno = 0; } @@ -652,26 +653,30 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p } static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (pVgroup->replica == pDb->cfg.replications) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_VNODE; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } } + } else if (pVgroup->replica < pDb->cfg.replications) { + } else { } return 0; @@ -726,6 +731,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { SDbObj *pDb = NULL; SUserObj *pUser = NULL; SAlterDbReq alterReq = {0}; + SDbObj dbObj = {0}; if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -749,15 +755,14 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { goto _OVER; } - SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - dbObj.cfg.numOfRetensions = 0; - dbObj.cfg.pRetensions = NULL; + if (dbObj.cfg.pRetensions != NULL) { + dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions); + if (dbObj.cfg.pRetensions == NULL) goto _OVER; + } code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); - if (code != 0) { - goto _OVER; - } + if (code != 0) goto _OVER; dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); @@ -771,6 +776,7 @@ _OVER: mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); + taosArrayDestroy(dbObj.cfg.pRetensions); return code; } From a1b3d0254aea1313a40801b32f8262b56beaf524 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 6 May 2022 15:36:10 +0800 Subject: [PATCH 19/63] feat: extract row version from trow --- include/common/trow.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 8b1d612433..f87ea1f009 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -141,7 +141,7 @@ typedef struct { /// row total length uint32_t len; /// row version - uint64_t ver; + // uint64_t ver; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -176,7 +176,7 @@ typedef struct { #define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_KEY(r) ((r)->ts) -#define TD_ROW_VER(r) ((r)->ver) +// #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and From f4f4bccdbb29c9c7bca06b57727acaa90a7ccdd5 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 15:45:16 +0800 Subject: [PATCH 20/63] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 61c0ecbf2c..83a33cc6ea 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,7 +102,7 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) - elif where_condition or "null" in groups: + elif where_condition or "not null" in groups: self.__join_current(sql, checkrows + 2 ) else: self.__join_current(sql, checkrows + 5 ) From ea88da15dbc4ec32fdd11a86c3bdb96450d629fb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 6 May 2022 15:51:18 +0800 Subject: [PATCH 21/63] feat(query): add histogram function --- source/libs/function/inc/builtinsimpl.h | 5 +++ source/libs/function/src/builtins.c | 30 +++++++++++++++ source/libs/function/src/builtinsimpl.c | 49 +++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 87cd3e24a8..abb9525cc5 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -73,6 +73,11 @@ bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) int32_t spreadFunction(SqlFunctionCtx* pCtx); int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t histogramFunction(SqlFunctionCtx* pCtx); +int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 38922833f9..e0d9fb60f5 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -225,6 +225,26 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (4 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY || + ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY || + ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE }; + return TSDB_CODE_SUCCESS; +} + static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // todo return TSDB_CODE_SUCCESS; @@ -600,6 +620,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = diffFunction, .finalizeFunc = functionFinalize }, + { + .name = "histogram", + .type = FUNCTION_TYPE_HISTOGRAM, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateHistogram, + .getEnvFunc = getHistogramFuncEnv, + .initFunc = histogramFunctionSetup, + .processFunc = histogramFunction, + .finalizeFunc = histogramFinalize + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0eba442e66..3c9eca85dd 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -20,6 +20,8 @@ #include "tdatablock.h" #include "tpercentile.h" +#define HISTOGRAM_MAX_BINS_NUM 100 + typedef struct SSumRes { union { int64_t isum; @@ -89,6 +91,22 @@ typedef struct SSpreadInfo { double max; } SSpreadInfo; +typedef struct SHistoFuncBin { + double lower; + double upper; + union { + int64_t count; + double percentage; + }; +} SHistoFuncBin; + +typedef struct SHistoFuncInfo { + int32_t numOfBins; + bool normalized; + SHistoFuncBin bins[]; +} SHistoFuncInfo; + + #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -1777,3 +1795,34 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } return functionFinalize(pCtx, pBlock); } + +bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin); + return true; +} + +bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + char* binType = pCtx->param[1].param.pz; + char* binDesc = pCtx->param[2].param.pz; + int64_t nornalized = pCtx->param[3].param.i; + + + return true; +} + +int32_t histogramFunction(SqlFunctionCtx *pCtx) { + return TSDB_CODE_SUCCESS; +} + +int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + //if (pInfo->hasResult == true) { + // SET_DOUBLE_VAL(&pInfo->result, pInfo->max - pInfo->min); + //} + return functionFinalize(pCtx, pBlock); +} From b6b3247d41b270dd83b89bccb8356147fdb5c97a Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 16:26:13 +0800 Subject: [PATCH 22/63] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 83a33cc6ea..2d91f754af 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,7 +102,7 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) - elif where_condition or "not null" in groups: + elif where_condition or "not null" in group_condition: self.__join_current(sql, checkrows + 2 ) else: self.__join_current(sql, checkrows + 5 ) From b41c2f32791fbf8339eab25b7be5ee7e096326fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 08:34:38 +0000 Subject: [PATCH 23/63] fix invalid read --- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 06da3b0e1d..8460400b59 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -339,7 +339,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs); + rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); if (rsp.pArray == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; From 43fd14759b82a919e6bd3cb5e516364faf8eef34 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 16:35:45 +0800 Subject: [PATCH 24/63] fix case --- tests/system-test/2-query/join.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 2d91f754af..a39bc21946 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -104,6 +104,8 @@ class TDTestCase: self.__join_current(sql, checkrows) elif where_condition or "not null" in group_condition: self.__join_current(sql, checkrows + 2 ) + elif group_condition: + self.__join_current(sql, checkrows + 3 ) else: self.__join_current(sql, checkrows + 5 ) if len(tblist) > 2 or len(tblist) < 1: From dd6349254182f6b4b7ea6182bd404275a327f4b1 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 16:48:51 +0800 Subject: [PATCH 25/63] add case for math function sqrt --- tests/system-test/2-query/sqrt.py | 551 ++++++++++++++++++++++++++++++ 1 file changed, 551 insertions(+) create mode 100644 tests/system-test/2-query/sqrt.py diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py new file mode 100644 index 0000000000..b41a41010e --- /dev/null +++ b/tests/system-test/2-query/sqrt.py @@ -0,0 +1,551 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_sqrt(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem < 0: + elem = None + else: + elem = math.sqrt(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("sqrt function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("sqrt value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select sqrt from t1", + # "select sqrt(-+--+c1 ) from t1", + # "select +-sqrt(c1) from t1", + # "select ++-sqrt(c1) from t1", + # "select ++--sqrt(c1) from t1", + # "select - -sqrt(c1)*0 from t1", + # "select sqrt(tbname+1) from t1 ", + "select sqrt(123--123)==1 from t1", + "select sqrt(c1) as 'd1' from t1", + "select sqrt(c1 ,c2) from t1", + "select sqrt(c1 ,NULL ) from t1", + "select sqrt(,) from t1;", + "select sqrt(sqrt(c1) ab from t1)", + "select sqrt(c1 ) as int from t1", + "select sqrt from stb1", + # "select sqrt(-+--+c1) from stb1", + # "select +-sqrt(c1) from stb1", + # "select ++-sqrt(c1) from stb1", + # "select ++--sqrt(c1) from stb1", + # "select - -sqrt(c1)*0 from stb1", + # "select sqrt(tbname+1) from stb1 ", + "select sqrt(123--123)==1 from stb1", + "select sqrt(c1) as 'd1' from stb1", + "select sqrt(c1 ,c2 ) from stb1", + "select sqrt(c1 ,NULL) from stb1", + "select sqrt(,) from stb1;", + "select sqrt(sqrt(c1) ab from stb1)", + "select sqrt(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select sqrt(ts) from t1" , + "select sqrt(c7) from t1", + "select sqrt(c8) from t1", + "select sqrt(c9) from t1", + "select sqrt(ts) from ct1" , + "select sqrt(c7) from ct1", + "select sqrt(c8) from ct1", + "select sqrt(c9) from ct1", + "select sqrt(ts) from ct3" , + "select sqrt(c7) from ct3", + "select sqrt(c8) from ct3", + "select sqrt(c9) from ct3", + "select sqrt(ts) from ct4" , + "select sqrt(c7) from ct4", + "select sqrt(c8) from ct4", + "select sqrt(c9) from ct4", + "select sqrt(ts) from stb1" , + "select sqrt(c7) from stb1", + "select sqrt(c8) from stb1", + "select sqrt(c9) from stb1" , + + "select sqrt(ts) from stbbb1" , + "select sqrt(c7) from stbbb1", + + "select sqrt(ts) from tbname", + "select sqrt(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select sqrt(c1) from t1", + "select sqrt(c2) from t1", + "select sqrt(c3) from t1", + "select sqrt(c4) from t1", + "select sqrt(c5) from t1", + "select sqrt(c6) from t1", + + "select sqrt(c1) from ct1", + "select sqrt(c2) from ct1", + "select sqrt(c3) from ct1", + "select sqrt(c4) from ct1", + "select sqrt(c5) from ct1", + "select sqrt(c6) from ct1", + + "select sqrt(c1) from ct3", + "select sqrt(c2) from ct3", + "select sqrt(c3) from ct3", + "select sqrt(c4) from ct3", + "select sqrt(c5) from ct3", + "select sqrt(c6) from ct3", + + "select sqrt(c1) from stb1", + "select sqrt(c2) from stb1", + "select sqrt(c3) from stb1", + "select sqrt(c4) from stb1", + "select sqrt(c5) from stb1", + "select sqrt(c6) from stb1", + + "select sqrt(c6) as alisb from stb1", + "select sqrt(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_sqrt_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select sqrt(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select sqrt(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 1.732050808) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,sqrt(c2) from ct1") + tdSql.checkData(0, 1, 298.140906284) + tdSql.checkData(1 , 1, 278.885281074) + tdSql.checkData(3 , 1, 235.701081881) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,sqrt(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 2.979932904) + tdSql.checkData(2 , 2, 2.787471970) + tdSql.checkData(3 , 2, 2.580697551) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_sqrt( "select c1, c2, c3 , c4, c5 from ct1", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c4), sqrt(c5) from ct1") + + # nest query for sqrt functions + tdSql.query("select c4 , sqrt(c4) ,sqrt(sqrt(c4)) , sqrt(sqrt(sqrt(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 9.380831520) + tdSql.checkData(0 , 2 , 3.062814314) + tdSql.checkData(0 , 3 , 1.750089802) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 8.774964387) + tdSql.checkData(1 , 2 , 2.962256638) + tdSql.checkData(1 , 3 , 1.721120750) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select sqrt(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select sqrt(c1) from stbbb1") + tdSql.error("select sqrt(c1) from tbname") + tdSql.error("select sqrt(c1) from ct5") + + # mix with common col + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,2.828427125) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c2, sqrt(c2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,55555) + tdSql.checkData(4 , 1 ,235.701081881) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, sqrt(c1),sqrt(c1), sqrt(sqrt(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,2.449489743) + tdSql.checkData(3 , 2 ,2.449489743) + tdSql.checkData(3 , 3 ,1.565084580) + + tdSql.query("select c1, sqrt(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, sqrt(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, sqrt(c1),c5, count(c5) from ct1 ") + tdSql.error("select sqrt(c1), count(c5) from stb1 ") + tdSql.error("select sqrt(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, sqrt(c1) -0 ,sqrt(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.000000000) + + tdSql.query(" select c1, sqrt(c1) -0 ,sqrt(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.710693865) + + tdSql.query("select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, sqrt(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 10000.000000000) + + + tdSql.query("select c1, sqrt(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 3162277.660168380) + + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, 3162277660171.025390625) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, 100000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, 100000000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0, 1,2.828427125) + tdSql.checkRows(13) + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, sqrt(1) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start sqrt(x,y) + tdSql.query("select c1,c2, sqrt(c2) from ct1") + tdSql.checkData(0, 2, 298.140906284) + tdSql.checkData(1, 2, 278.885281074) + tdSql.checkData(4, 2, 0.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=sqrt(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,11111) + tdSql.checkData(0,2,1.000000000) + tdSql.checkData(0,3,1.000000000) + tdSql.checkData(0,4,0.900000000) + tdSql.checkData(0,5,1.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from sub1_bound") + + self.check_result_auto_sqrt( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c3), sqrt(c2) ,sqrt(c1) from sub1_bound") + + self.check_result_auto_sqrt("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select sqrt(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select sqrt(abs(c1)) ,sqrt(abs(c2)) , sqrt(abs(c3)) , sqrt(abs(c4)), sqrt(abs(c5)), sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483647)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767)) + tdSql.checkData(0,3,math.sqrt(127)) + tdSql.checkData(0,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.sqrt(2147483647)) + tdSql.checkData(1,1,math.sqrt(9223372036854775807)) + tdSql.checkData(1,2,math.sqrt(32767)) + tdSql.checkData(1,3,math.sqrt(127)) + tdSql.checkData(1,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.sqrt(2147483646)) + tdSql.checkData(3,1,math.sqrt(9223372036854775806)) + tdSql.checkData(3,2,math.sqrt(32766)) + tdSql.checkData(3,3,math.sqrt(126)) + tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483648.000000000)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767.000000000)) + tdSql.checkData(0,3,math.sqrt(63.500000000)) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: sqrt basic query ============") + + self.basic_sqrt_function() + + tdLog.printNoPrefix("==========step5: big number sqrt query ============") + + self.test_big_number() + + tdLog.printNoPrefix("==========step6: base number for sqrt query ============") + + self.pow_base_test() + + tdLog.printNoPrefix("==========step7: sqrt boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step8: sqrt filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 390c68a8c085a30febb78a8cb6f4037a749a6c5a Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 16:52:09 +0800 Subject: [PATCH 26/63] add test case for math function sqrt --- tests/system-test/2-query/sqrt.py | 2 +- tests/system-test/fulltest.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index b41a41010e..28e869e044 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -497,7 +497,7 @@ class TDTestCase: tdSql.checkData(3,2,math.sqrt(32766)) tdSql.checkData(3,3,math.sqrt(126)) tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000)) - + # check + - * / in functions tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ") tdSql.checkData(0,0,math.sqrt(2147483648.000000000)) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1dfb160987..51f1649cc8 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -24,3 +24,4 @@ python3 ./test.py -f 2-query/floor.py python3 ./test.py -f 2-query/round.py python3 ./test.py -f 2-query/log.py python3 ./test.py -f 2-query/pow.py +python3 ./test.py -f 2-query/sqrt.py From 4e90982c22be6fe41c15f2ea04104735d8b29108 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 May 2022 17:23:20 +0800 Subject: [PATCH 27/63] fix(query): sort according to the generated column data in order by operator. --- source/common/src/tdatablock.c | 15 ++++--- source/libs/executor/inc/executorimpl.h | 5 +-- source/libs/executor/inc/tsort.h | 15 +++++-- source/libs/executor/src/executorimpl.c | 37 +++++----------- source/libs/executor/src/sortoperator.c | 59 +++++++++++++++++-------- source/libs/executor/src/tsort.c | 40 ++++++++--------- source/libs/scalar/src/sclvector.c | 6 +-- 7 files changed, 94 insertions(+), 83 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index dcea167e81..4946c9690b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -363,9 +363,9 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { int32_t mapIndex = i; - if (pIndexMap) { - mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); - } +// if (pIndexMap) { +// mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); +// } SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); @@ -491,9 +491,14 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { - bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); - char* p = colDataGetData(pColData, j); + bool isNull = false; + if (pBlock->pBlockAgg == NULL) { + isNull = colDataIsNull_s(pColData, pBlock->info.rows); + } else { + isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); + } + char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); } } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index eefa38d802..3b5d0c209f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -578,9 +578,8 @@ typedef struct SSortOperatorInfo { uint32_t sortBufSize; // max buffer size for in-memory sort SArray* pSortInfo; SSortHandle* pSortHandle; - SArray* inputSlotMap; // for index map from table scan output + SArray* pColMatchInfo; // for index map from table scan output int32_t bufPageSize; -// int32_t numOfRowsInRes; // TODO extact struct int64_t startTs; // sort start time @@ -645,7 +644,7 @@ void cleanupAggSup(SAggSupporter* pAggSup); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity); +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo); SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 2072707b30..d74628a72f 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -117,18 +117,25 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex); +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex); +void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); + +/** + * + * @param pSortHandle + * @return + */ +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9aa251e1b6..943d4b2783 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3520,7 +3520,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSortedMergeOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL); } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; @@ -4701,7 +4701,7 @@ static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols); -static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); +static SArray* createSortInfo(SNodeList* pNodeList); static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); @@ -4870,16 +4870,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SArray* info = createSortInfo(pSortPhyNode->pSortKeys, pSortPhyNode->pTargets); - SArray* slotMap = createIndexMap(pSortPhyNode->pTargets); + SArray* info = createSortInfo(pSortPhyNode->pSortKeys); int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; - if (pSortPhyNode->pExprs != NULL) { - pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); - } + SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); - pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, slotMap, pTaskInfo); + int32_t numOfOutputCols = 0; + SArray* pColList = + extractColMatchInfo(pSortPhyNode->pTargets, pSortPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -5037,7 +5037,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { return pList; } -SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { +SArray* createSortInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { @@ -5052,22 +5052,7 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST); SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr; - - bool found = false; - for (int32_t j = 0; j < LIST_LENGTH(pNodeListTarget); ++j) { - STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j); - - SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr; - if (pColNode->slotId == pColNodeT->slotId) { // to find slotId in PhysiSort OutputDataBlockDesc - bi.slotId = pTarget->slotId; - found = true; - break; - } - } - - if (!found) { - qError("sort slot id does not found"); - } + bi.slotId = pColNode->slotId; taosArrayPush(pList, &bi); } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0f973b0cf0..619651f11f 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -5,7 +5,7 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, - SArray* pIndexMap, SExecTaskInfo* pTaskInfo) { + SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); int32_t rowSize = pResBlock->info.rowSize; @@ -20,17 +20,19 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header - pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer pInfo->pSortInfo = pSortInfo; - pInfo->inputSlotMap = pIndexMap; + pInfo->pColMatchInfo= pColMatchColInfo; pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; + // lazy evaluation for the following parameter since the input datablock is not known till now. +// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header +// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, NULL); @@ -45,14 +47,12 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR return NULL; } -// TODO merge aggregate super table void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { - colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + colDataAppendNULL(pColInfo, pBlock->info.rows); } else { char* pData = tsortGetValue(pTupleHandle, i); colDataAppend(pColInfo, pBlock->info.rows, pData, false); @@ -62,11 +62,12 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { pBlock->info.rows += 1; } -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) { +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) { blockDataCleanup(pDataBlock); - blockDataEnsureCapacity(pDataBlock, capacity); + ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); - blockDataEnsureCapacity(pDataBlock, capacity); + SSDataBlock* p = tsortGetSortedDataBlock(pHandle); + blockDataEnsureCapacity(p, capacity); while (1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); @@ -74,12 +75,32 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i break; } - appendOneRowToDataBlock(pDataBlock, pTupleHandle); - if (pDataBlock->info.rows >= capacity) { + appendOneRowToDataBlock(p, pTupleHandle); + if (p->info.rows >= capacity) { return pDataBlock; } } + if (p->info.rows > 0) { + int32_t numOfCols = taosArrayGetSize(pColMatchInfo); + for(int32_t i = 0; i < numOfCols; ++i) { + SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); + + for(int32_t j = 0; j < p->info.numOfCols; ++j) { + SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, j); + if (pSrc->info.colId == pmInfo->colId) { + SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); + colDataAssign(pDst, pSrc, p->info.rows); + break; + } + } + } + + pDataBlock->info.rows = p->info.rows; + pDataBlock->info.capacity = p->info.rows; + } + + blockDataDestroy(p); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } @@ -106,16 +127,16 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { SSortOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); } - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT, - pInfo->bufPageSize, numOfBufPage, pInfo->binfo.pRes, pTaskInfo->id.str); +// pInfo->binfo.pRes is not equalled to the input datablock. +// int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, + -1, -1, NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[0]; tsortAddSource(pInfo->pSortHandle, ps); @@ -127,7 +148,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { } pOperator->status = OP_RES_TO_RETURN; - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); } void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { @@ -135,5 +156,5 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); taosArrayDestroy(pInfo->pSortInfo); - taosArrayDestroy(pInfo->inputSlotMap); + taosArrayDestroy(pInfo->pColMatchInfo); } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 50aa4cfc01..040ee8c7f5 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -64,25 +64,8 @@ struct SSortHandle { static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); -static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - - colInfo.info.type = pSchema[i].type; - colInfo.info.bytes = pSchema[i].bytes; - colInfo.info.colId = pSchema[i].colId; - taosArrayPush(pBlock->pDataBlock, &colInfo); - - if (IS_VAR_DATA_TYPE(colInfo.info.type)) { - pBlock->info.hasVarCol = true; - } - } - - return pBlock; +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) { + return createOneDataBlock(pSortHandle->pDataBlock, false); } /** @@ -98,7 +81,10 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, SArray* pIndexMap, int32_t pSortHandle->numOfPages = numOfPages; pSortHandle->pSortInfo = pSortInfo; pSortHandle->pIndexMap = pIndexMap; - pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + + if (pBlock != NULL) { + pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + } pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); pSortHandle->cmpParam.orderInfo = pSortInfo; @@ -530,6 +516,17 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { if (pHandle->pDataBlock == NULL) { pHandle->pDataBlock = createOneDataBlock(pBlock, false); + + // calculate the buffer pages according to the total available buffers. + int32_t rowSize = blockDataGetRowSize(pBlock); + if (rowSize * 4 > 4096) { + pHandle->pageSize = rowSize * 4; + } else { + pHandle->pageSize = 4096; + } + // todo!! + pHandle->numOfPages = 1024; + sortBufSize = pHandle->numOfPages * pHandle->pageSize; } // perform the scalar function calculation before apply the sort @@ -538,7 +535,6 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { } // todo relocate the columns - int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock, pHandle->pIndexMap); if (code != 0) { return code; @@ -689,7 +685,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex); - return colDataIsNull(pColInfoSrc, 0, pVHandle->rowIndex, NULL); + return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex); } void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 84aa5559d0..1e37533f2c 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1023,8 +1023,7 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, 0); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0); } } } @@ -1050,8 +1049,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); From ae23dd2382a5d8a06dbb4e281bea2901c898ee80 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 17:28:58 +0800 Subject: [PATCH 28/63] fix: alter db replications --- source/dnode/mnode/impl/inc/mndVgroup.h | 5 +- source/dnode/mnode/impl/src/mndDb.c | 235 +++++++++++++++--------- source/dnode/mnode/impl/src/mndVgroup.c | 8 + source/dnode/mnode/impl/test/db/db.cpp | 13 +- 4 files changed, 173 insertions(+), 88 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 8aecc22454..87be15a4fd 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -27,10 +27,13 @@ void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist); + void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 96a8969b56..cf9dd664e3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,6 +261,104 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_VNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_VNODE; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { @@ -278,6 +376,8 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { } static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1; if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; @@ -300,9 +400,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; - if (pCfg->strict > pCfg->replications) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; + + terrno = 0; return TSDB_CODE_SUCCESS; } @@ -381,24 +482,8 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } @@ -412,24 +497,8 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, false) != 0) { return -1; } } @@ -482,7 +551,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate } if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -570,37 +638,37 @@ _OVER: static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) { + if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) { pDb->cfg.buffer = pAlter->buffer; terrno = 0; } - if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) { + if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) { pDb->cfg.pages = pAlter->pages; terrno = 0; } - if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) { + if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) { pDb->cfg.pageSize = pAlter->pageSize; terrno = 0; } - if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { + if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { pDb->cfg.daysPerFile = pAlter->daysPerFile; terrno = 0; } - if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { + if (pAlter->daysToKeep0 > 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; terrno = 0; } - if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { + if (pAlter->daysToKeep1 > 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; terrno = 0; } - if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { + if (pAlter->daysToKeep2 > 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; terrno = 0; } @@ -625,7 +693,7 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } - if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { + if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) { pDb->cfg.replications = pAlter->replications; pDb->vgVersion++; terrno = 0; @@ -653,30 +721,44 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p } static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - if (pVgroup->replica == pDb->cfg.replications) { + if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } - } else if (pVgroup->replica < pDb->cfg.replications) { } else { + SVgObj newVgroup = {0}; + memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + if (newVgroup.replica < pDb->cfg.replications) { + SVnodeGid new1 = {0}; + SVnodeGid new2 = {0}; + SVnodeGid exist = {0}; + if (mndAddVnodeToVgroup(pMnode, &newVgroup, &new1, &new2, &exist) != 0) { + mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new1, true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new2, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + } else { + SVnodeGid del1 = {0}; + SVnodeGid del2 = {0}; + SVnodeGid exist = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, &del1, &del2, &exist) != 0) { + mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + } + + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } return 0; @@ -764,6 +846,9 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) goto _OVER; + code = mndCheckDbCfg(pMnode, &dbObj.cfg); + if (code != 0) goto _OVER; + dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); @@ -905,24 +990,8 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index d1e4be1161..ca60a3b9ed 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -491,6 +491,14 @@ _OVER: return code; } +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist) { + return 0; +} + +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist) { + return 0; +} + SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 3c5bf5c083..545f9f22bb 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -73,14 +73,19 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { SAlterDbReq alterdbReq = {0}; strcpy(alterdbReq.db, "1.d1"); + alterdbReq.buffer = 12; - alterdbReq.daysToKeep0 = 300; - alterdbReq.daysToKeep1 = 400; - alterdbReq.daysToKeep2 = 500; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; - alterdbReq.strict = 2; + alterdbReq.strict = 1; alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); void* pReq = rpcMallocCont(contLen); From 1315f8e510794c73aeb32ca083972b8d829b770a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 May 2022 17:37:57 +0800 Subject: [PATCH 29/63] fix(query): fix an syntax error. --- source/client/test/clientTests.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d7ffa282df..fc5781cb4d 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -734,6 +734,5 @@ TEST(testCase, agg_query_tables) { taos_free_result(pRes); taos_close(pConn); } -#endif #pragma GCC diagnostic pop From c9ee1b0deef8eadb48638708c4efeeb262abc04b Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 6 May 2022 17:41:49 +0800 Subject: [PATCH 30/63] fix: core dump when pass mulitiple columns from taosd to udfd --- include/libs/function/tudf.h | 11 +++++++---- source/libs/function/src/tudf.c | 4 +++- source/libs/function/src/udfd.c | 25 ++++++++++++++++++++++--- tests/script/tsim/query/udf.sim | 21 +++++++++++++++++++++ 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 0000972f5e..d59a7c23f7 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -29,7 +29,11 @@ extern "C" { #endif #define UDF_LISTEN_PIPE_NAME_LEN 32 -#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." +#ifdef _WIN32 +#define UDF_LISTEN_PIPE_NAME_PREFIX "\\\\?\\pipe\\udfd.sock" +#else +#define UDF_LISTEN_PIPE_NAME_PREFIX ".udfd.sock." +#endif #define UDF_DNODE_ID_ENV_NAME "DNODE_ID" //====================================================================================== @@ -129,8 +133,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); // begin API to UDF writer. // dynamic lib init and destroy -typedef int32_t (*TUdfSetupFunc)(); -typedef int32_t (*TUdfTeardownFunc)(); +typedef int32_t (*TUdfInitFunc)(); +typedef int32_t (*TUdfDestroyFunc)(); //TODO: add API to check function arguments type, number etc. @@ -242,7 +246,6 @@ static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRo return 0; } -typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5b1573c88f..0a99ef61ce 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -594,7 +594,9 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS //TODO: free the array output->pDataBlock output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - taosArrayPush(output->pDataBlock, input->columnData); + for (int32_t i = 0; i < numOfCols; ++i) { + taosArrayPush(output->pDataBlock, (input + i)->columnData); + } return 0; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ba9fca2969..f5e4a9c6e6 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -81,6 +81,9 @@ typedef struct SUdf { TUdfAggStartFunc aggStartFunc; TUdfAggProcessFunc aggProcFunc; TUdfAggFinishFunc aggFinishFunc; + + TUdfInitFunc initFunc; + TUdfDestroyFunc destroyFunc; } SUdf; // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix @@ -101,7 +104,19 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); return UDFC_CODE_LOAD_UDF_FAILURE; } - //TODO: init and destroy function + + char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *initSuffix = "_init"; + strcpy(initFuncName, udfName); + strncat(initFuncName, initSuffix, strlen(initSuffix)); + uv_dlsym(&udf->lib, initFuncName, (void**)(&udf->initFunc)); + + char destroyFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *destroySuffix = "_destroy"; + strcpy(destroyFuncName, udfName); + strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + uv_dlsym(&udf->lib, destroyFuncName, (void**)(&udf->destroyFunc)); + if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); @@ -159,6 +174,9 @@ void udfdProcessRequest(uv_work_t *req) { if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; udfdLoadUdf(setup->udfName, udf); + if (udf->initFunc) { + udf->initFunc(); + } udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -170,7 +188,6 @@ void udfdProcessRequest(uv_work_t *req) { } SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); handle->udf = udf; - // TODO: allocate private structure and call init function and set it to handle SUdfResponse rsp; rsp.seqNum = request.seqNum; rsp.type = request.type; @@ -275,10 +292,12 @@ void udfdProcessRequest(uv_work_t *req) { if (unloadUdf) { uv_cond_destroy(&udf->condReady); uv_mutex_destroy(&udf->lock); + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } uv_dlclose(&udf->lib); taosMemoryFree(udf); } - // TODO: call destroy and free udf private taosMemoryFree(handle); SUdfResponse response; diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 8acd07cfe4..c76569b40f 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -43,6 +43,27 @@ if $data00 != 2.236067977 then return -1 endi +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); +sql select udf1(f1, f2) from t2; +if $rows != 2 then + return -1 +endi +if $data00 != 88 then + return -1 +endi +if $data10 != 88 then + return -1 +endi + +sql select udf2(f1, f2) from t2; +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi + #sql drop function udf1; #sql drop function udf2; system sh/exec.sh -n dnode1 -s stop -x SIGKILL From 5f063879401cbc1f870aa615f92ce8ff826318af Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 6 May 2022 17:52:59 +0800 Subject: [PATCH 31/63] fix: some problems of parser and planner --- include/util/taoserror.h | 8 ++ source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/parser/src/parTranslater.c | 124 ++++++++++++++++-- source/libs/parser/src/parUtil.c | 18 ++- source/libs/planner/src/planLogicCreater.c | 5 + source/libs/planner/src/planOptimizer.c | 39 +++++- source/libs/planner/test/planOptimizeTest.cpp | 18 ++- source/libs/scheduler/src/scheduler.c | 42 +++--- source/util/src/terror.c | 3 + 9 files changed, 218 insertions(+), 41 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index abc752955d..441e87eff7 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -621,6 +621,14 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639) #define TSDB_CODE_PAR_INVALID_DROP_STABLE TAOS_DEF_ERROR_CODE(0, 0x263A) #define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE TAOS_DEF_ERROR_CODE(0, 0x263B) +#define TSDB_CODE_PAR_DUPLICATED_COLUMN TAOS_DEF_ERROR_CODE(0, 0x263C) +#define TSDB_CODE_PAR_INVALID_TAGS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x263D) +#define TSDB_CODE_PAR_INVALID_ROW_LENGTH TAOS_DEF_ERROR_CODE(0, 0x263E) +#define TSDB_CODE_PAR_INVALID_COLUMNS_NUM TAOS_DEF_ERROR_CODE(0, 0x263F) +#define TSDB_CODE_PAR_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x2640) +#define TSDB_CODE_PAR_INVALID_FIRST_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2641) +#define TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN TAOS_DEF_ERROR_CODE(0, 0x2642) +#define TSDB_CODE_PAR_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x2643) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 878dd5fca4..d7eddc280f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -339,7 +339,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs); + rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); if (rsp.pArray == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fbb1f34217..6874b5b7d4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2174,17 +2174,6 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS return TSDB_CODE_SUCCESS; } -static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) { - SNode* pNode; - FOREACH(pNode, pStmt->pTags) { - SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if (pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); - } - } - return TSDB_CODE_SUCCESS; -} - static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) { if (NULL == pFuncs) { return TSDB_CODE_SUCCESS; @@ -2196,6 +2185,113 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs return TSDB_CODE_SUCCESS; } +static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pTags) { + int32_t ntags = LIST_LENGTH(pTags); + if (0 == ntags) { + return TSDB_CODE_SUCCESS; + } else if (ntags > TSDB_MAX_TAGS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM); + } + + int32_t code = TSDB_CODE_SUCCESS; + int32_t tagsSize = 0; + SNode* pNode = NULL; + FOREACH(pNode, pTags) { + SColumnDefNode* pTag = (SColumnDefNode*)pNode; + int32_t len = strlen(pTag->colName); + if (NULL != taosHashGet(pHash, pTag->colName, len)) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); + } + if (TSDB_CODE_SUCCESS == code && pTag->dataType.type == TSDB_DATA_TYPE_JSON && ntags > 1) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); + } + if (TSDB_CODE_SUCCESS == code) { + if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_NCHAR_LEN)) { + code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pHash, pTag->colName, len, &pTag, POINTER_BYTES); + } + if (TSDB_CODE_SUCCESS == code) { + tagsSize += pTag->dataType.bytes; + } else { + break; + } + } + + if (TSDB_CODE_SUCCESS == code && tagsSize > TSDB_MAX_TAGS_LEN) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_LENGTH, TSDB_MAX_TAGS_LEN); + } + + return code; +} + +static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pCols) { + int32_t ncols = LIST_LENGTH(pCols); + if (ncols < TSDB_MIN_COLUMNS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } else if (ncols > TSDB_MAX_COLUMNS) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TOO_MANY_COLUMNS); + } + + int32_t code = TSDB_CODE_SUCCESS; + + bool first = true; + int32_t rowSize = 0; + SNode* pNode = NULL; + FOREACH(pNode, pCols) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if (first) { + first = false; + if (TSDB_DATA_TYPE_TIMESTAMP != pCol->dataType.type) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN); + } + } + int32_t len = strlen(pCol->colName); + if (TSDB_CODE_SUCCESS == code && NULL != taosHashGet(pHash, pCol->colName, len)) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); + } + if (TSDB_CODE_SUCCESS == code) { + if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_NCHAR_LEN)) { + code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); + } + if (TSDB_CODE_SUCCESS == code) { + rowSize += pCol->dataType.bytes; + } else { + break; + } + } + + if (TSDB_CODE_SUCCESS == code && rowSize > TSDB_MAX_BYTES_PER_ROW) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); + } + + return code; +} + +static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + SHashObj* pHash = taosHashInit(LIST_LENGTH(pStmt->pTags) + LIST_LENGTH(pStmt->pCols), + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pHash) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = checkTableTagsSchema(pCxt, pHash, pStmt->pTags); + if (TSDB_CODE_SUCCESS == code) { + code = checkTableColsSchema(pCxt, pHash, pStmt->pCols); + } + + taosHashCleanup(pHash); + return code; +} + static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { int32_t code = checkRangeOption(pCxt, "delay", pStmt->pOptions->delay, TSDB_MIN_ROLLUP_DELAY, TSDB_MAX_ROLLUP_DELAY); if (TSDB_CODE_SUCCESS == code) { @@ -2211,7 +2307,7 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt code = checkTableSmaOption(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code) { - code = checkTableTags(pCxt, pStmt); + code = checkTableSchema(pCxt, pStmt); } return code; } @@ -3838,6 +3934,10 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl goto over; } + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code && pClause->ignoreNotExists) { + code = TSDB_CODE_SUCCESS; + } + *pIsSuperTable = false; SVgroupInfo info = {0}; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e839536218..e37dbd1edd 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -129,7 +129,23 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INVALID_DROP_STABLE: return "Cannot drop super table in batch"; case TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE: - return "start(end) time of query range required or time range too large"; + return "Start(end) time of query range required or time range too large"; + case TSDB_CODE_PAR_DUPLICATED_COLUMN: + return "Duplicated column names"; + case TSDB_CODE_PAR_INVALID_TAGS_LENGTH: + return "Tags length exceeds max length %d"; + case TSDB_CODE_PAR_INVALID_ROW_LENGTH: + return "Row length exceeds max length %d"; + case TSDB_CODE_PAR_INVALID_COLUMNS_NUM: + return "Illegal number of columns"; + case TSDB_CODE_PAR_TOO_MANY_COLUMNS: + return "Too many columns"; + case TSDB_CODE_PAR_INVALID_FIRST_COLUMN: + return "First column must be timestamp"; + case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN: + return "Invalid binary/nchar column length"; + case TSDB_CODE_PAR_INVALID_TAGS_NUM: + return "Invalid number of tag columns"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 0bc700696b..3e19ccbd82 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -277,6 +277,11 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); } + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM); + } + pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 67609355d7..dbce9abf36 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -123,14 +123,40 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { return NULL; } +static bool needOptimizeDataRequire(const SFunctionNode* pFunc) { + if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) { + return false; + } + SNode* pPara = NULL; + FOREACH(pPara, pFunc->pParameterList) { + if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) { + return false; + } + } + return true; +} + +static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) { + if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) { + return false; + } + SNode* pPara = NULL; + FOREACH(pPara, pFunc->pParameterList) { + if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) { + return false; + } + } + return true; +} + static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent); SNode* pFunc = NULL; FOREACH(pFunc, pAllFuncs) { int32_t code = TSDB_CODE_SUCCESS; - if (fmIsSpecialDataRequiredFunc(((SFunctionNode*)pFunc)->funcId)) { + if (needOptimizeDataRequire((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(pSdrFuncs, nodesCloneNode(pFunc)); - } else if (fmIsDynamicScanOptimizedFunc(((SFunctionNode*)pFunc)->funcId)) { + } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) { code = nodesListMakeStrictAppend(pDsoFuncs, nodesCloneNode(pFunc)); } if (TSDB_CODE_SUCCESS != code) { @@ -541,9 +567,14 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } - SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; - SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) { return cpdIsPrimaryKey(pOper->pRight, pRightCols); } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) { diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index deb20c65a4..4938618db6 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -20,11 +20,21 @@ using namespace std; class PlanOptimizeTest : public PlannerTestBase {}; +TEST_F(PlanOptimizeTest, optimizeScanData) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1"); + + run("SELECT COUNT(c1) FROM t1"); + + run("SELECT COUNT(CAST(c1 AS BIGINT)) FROM t1"); +} + TEST_F(PlanOptimizeTest, orderByPrimaryKey) { useDb("root", "test"); - run("select * from t1 order by ts"); - run("select * from t1 order by ts desc"); - run("select c1 from t1 order by ts"); - run("select c1 from t1 order by ts desc"); + run("SELECT * FROM t1 ORDER BY ts"); + run("SELECT * FROM t1 ORDER BY ts DESC"); + run("SELECT c1 FROM t1 ORDER BY ts"); + run("SELECT c1 FROM t1 ORDER BY ts DESC"); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index a70d366e9b..53772601ca 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -22,7 +22,7 @@ #include "trpc.h" SSchedulerMgmt schMgmt = { - .jobRef = -1, + .jobRef = -1, }; FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } @@ -72,7 +72,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, int64_t startTs, bool syncSchedule) { int32_t code = 0; - int64_t refId = -1; + int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); @@ -124,7 +124,7 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray } atomic_add_fetch_32(&schMgmt.jobNum, 1); - + if (NULL == schAcquireJob(refId)) { SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); @@ -1085,19 +1085,22 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch case TDMT_VND_CREATE_TABLE_RSP: { SVCreateTbBatchRsp batchRsp = {0}; if (msg) { - SCH_ERR_JRET(tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp)); - if (batchRsp.pArray) { - int32_t num = taosArrayGetSize(batchRsp.pArray); - for (int32_t i = 0; i < num; ++i) { - SVCreateTbRsp *rsp = taosArrayGet(batchRsp.pArray, i); + SCoder coder = {0}; + tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER); + code = tDecodeSVCreateTbBatchRsp(&coder, &batchRsp); + if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { + for (int32_t i = 0; i < batchRsp.nRsps; ++i) { + SVCreateTbRsp *rsp = batchRsp.pRsps + i; if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { - taosArrayDestroy(batchRsp.pArray); + tCoderClear(&coder); SCH_ERR_JRET(rsp->code); + } else if (TSDB_CODE_SUCCESS != rsp->code) { + code = rsp->code; } } - - taosArrayDestroy(batchRsp.pArray); } + tCoderClear(&coder); + SCH_ERR_JRET(code); } SCH_ERR_JRET(rspCode); @@ -1110,13 +1113,14 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCoder coder = {0}; tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER); code = tDecodeSVDropTbBatchRsp(&coder, &batchRsp); - if (TSDB_CODE_SUCCESS == code && batchRsp.pArray) { - int32_t num = taosArrayGetSize(batchRsp.pArray); - for (int32_t i = 0; i < num; ++i) { - SVDropTbRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { + for (int32_t i = 0; i < batchRsp.nRsps; ++i) { + SVDropTbRsp *rsp = batchRsp.pRsps + i; if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) { tCoderClear(&coder); SCH_ERR_JRET(rsp->code); + } else if (TSDB_CODE_SUCCESS != rsp->code) { + code = rsp->code; } } } @@ -2282,10 +2286,10 @@ int32_t schCancelJob(SSchJob *pJob) { } void schCloseJobRef(void) { - if (!atomic_load_8((int8_t*)&schMgmt.exit)) { + if (!atomic_load_8((int8_t *)&schMgmt.exit)) { return; } - + SCH_LOCK(SCH_WRITE, &schMgmt.lock); if (atomic_load_32(&schMgmt.jobNum) <= 0 && schMgmt.jobRef >= 0) { taosCloseRef(schMgmt.jobRef); @@ -2791,8 +2795,8 @@ void schedulerFreeTaskList(SArray *taskList) { } void schedulerDestroy(void) { - atomic_store_8((int8_t*)&schMgmt.exit, 1); - + atomic_store_8((int8_t *)&schMgmt.exit, 1); + if (schMgmt.jobRef >= 0) { SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); int64_t refId = 0; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 00fe8bd0e9..1470496c68 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -443,6 +443,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status erro TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") +// parser +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TABLE_NOT_EXIST, "Table does not exist") + //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error") From 2687ea395734d4baf97cc866639749e72871d522 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 6 May 2022 17:53:10 +0800 Subject: [PATCH 32/63] fix: remove python connector (#12157) update requirements.txt [TD-14358] --- tests/requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/requirements.txt diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000000..ce459414c4 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,5 @@ +taospy +numpy +fabric2 +psutil +pandas From 9295303f1e5a913f1044e99774ea653ffec23e4a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 6 May 2022 17:53:32 +0800 Subject: [PATCH 33/63] feat: taostools update for3.0 (#12160) * feat: update taos-tools for 3.0 [TD-13052] * update taos-tools taosdump uses taos_fetch_lengths() --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index 2f3dfddd4d..59e0ebaf49 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c +Subproject commit 59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20 From 79b70ac2875b9293914d88aac4e2cff67b3bb8d8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 10:03:54 +0000 Subject: [PATCH 34/63] refact more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 98 ++++++++++++++++++--- 1 file changed, 86 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index ae0d4630d1..fd753afb2f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,10 +15,10 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListCfg SMemSkipListCfg; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; struct SMemTable { STsdb *pTsdb; @@ -32,15 +32,16 @@ struct SMemTable { SMemData **pBuckets; }; -struct SMemSkipListCfg { - int8_t maxLevel; - int32_t nKey; - int32_t nData; +struct SMemSkipListNode { + int8_t level; + SMemSkipListNode *forwards[]; }; struct SMemSkipList { - int8_t level; - uint32_t seed; + uint32_t seed; + int8_t level; + int32_t size; + SMemSkipListNode pHead[]; }; struct SMemData { @@ -76,6 +77,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); if (pMemTb->pBuckets == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pMemTb); return -1; } @@ -83,11 +85,83 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { return 0; } -int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { - // TODO +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { + if (pMemTb) { + // loop to destroy the contents (todo) + taosMemoryFree(pMemTb->pBuckets); + taosMemoryFree(pMemTb); + } return 0; } +int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { + SMemData *pMemData; + SVBufPool *pPool = pMemTb->pTsdb->pVnode->inUse; + int32_t hash; + int32_t tlen; + uint8_t buf[16]; + int32_t rSize; + SMemSkipListNode *pSlNode; + const STSRow *pTSRow; + + // search hash + hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket; + for (pMemData = pMemTb->pBuckets[hash]; pMemData; pMemData = pMemData->pHashNext) { + if (pMemData->suid == pSubmitBlk->suid && pMemData->uid == pSubmitBlk->uid) break; + } + + // create pMemData if need + if (pMemData == NULL) { + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + if (pMemData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemData->pHashNext = NULL; + pMemData->suid = pSubmitBlk->suid; + pMemData->uid = pSubmitBlk->uid; + pMemData->minKey = TSKEY_MAX; + pMemData->maxKey = TSKEY_MIN; + pMemData->minVer = -1; + pMemData->maxVer = -1; + pMemData->nRows = 0; + pMemData->sl.level = 0; + pMemData->sl.seed = taosRand(); + pMemData->sl.size = 0; + + // add to MemTable + hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket; + pMemData->pHashNext = pMemTb->pBuckets[hash]; + pMemTb->pBuckets[hash] = pMemData; + } + + // loop to insert data to skiplist + for (;;) { + rSize = 0; + pTSRow = NULL; + if (pTSRow == NULL) break; + + // check the row (todo) + + // move the cursor to position to write (todo) + + // insert the row + int8_t level = 1; + tlen = 0; // sizeof(int64_t) + tsdbPutLen(rSize) + rSize; + pSlNode = vnodeBufPoolMalloc(pPool, tlen); + if (pSlNode == NULL) { + ASSERT(0); + } + } + + return 0; +} + +static void tsdbEncodeRow(int64_t version, int32_t rSize, const STSRow *pRow) {} + +static void tsdbDecodeRow(int64_t *version, int32_t *rSize, const STSRow **ppRow) {} + // SMemData // SMemSkipList \ No newline at end of file From 809a3eda257556a8eb09703c9497cedfde730665 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 May 2022 18:16:16 +0800 Subject: [PATCH 35/63] fix(query): set the correct result field length for first/last query. --- source/libs/function/src/builtins.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index eac11558cb..734136b296 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -242,8 +242,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l "The parameters of first/last can only be columns"); } - uint8_t paraType = ((SExprNode*)pPara)->resType.type; - pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType}; + pFunc->node.resType = ((SExprNode*)pPara)->resType; return TSDB_CODE_SUCCESS; } From 5ec83cbf11cf415faaf66ddd9895b64e3da09163 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 10:21:55 +0000 Subject: [PATCH 36/63] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index fd753afb2f..1c263b7d00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -39,6 +39,7 @@ struct SMemSkipListNode { struct SMemSkipList { uint32_t seed; + int8_t maxLevel; int8_t level; int32_t size; SMemSkipListNode pHead[]; @@ -96,11 +97,14 @@ int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { SMemData *pMemData; - SVBufPool *pPool = pMemTb->pTsdb->pVnode->inUse; + STsdb *pTsdb = pMemTb->pTsdb; + SVnode *pVnode = pTsdb->pVnode; + SVBufPool *pPool = pVnode->inUse; int32_t hash; int32_t tlen; uint8_t buf[16]; - int32_t rSize; + int32_t rlen; + const uint8_t *p; SMemSkipListNode *pSlNode; const STSRow *pTSRow; @@ -126,8 +130,9 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p pMemData->minVer = -1; pMemData->maxVer = -1; pMemData->nRows = 0; - pMemData->sl.level = 0; pMemData->sl.seed = taosRand(); + pMemData->sl.maxLevel = pVnode->config.tsdbCfg.slLevel; + pMemData->sl.level = 0; pMemData->sl.size = 0; // add to MemTable @@ -137,9 +142,13 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p } // loop to insert data to skiplist + p = pSubmitBlk->pData; for (;;) { - rSize = 0; - pTSRow = NULL; + if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break; + + // p = tGetLen(p, &rlen); + pTSRow = (STSRow *)p; + p += rlen; if (pTSRow == NULL) break; // check the row (todo) @@ -148,7 +157,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p // insert the row int8_t level = 1; - tlen = 0; // sizeof(int64_t) + tsdbPutLen(rSize) + rSize; + tlen = 0; // sizeof(int64_t) + tsdbPutLen(rlen) + rlen; pSlNode = vnodeBufPoolMalloc(pPool, tlen); if (pSlNode == NULL) { ASSERT(0); @@ -158,9 +167,9 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p return 0; } -static void tsdbEncodeRow(int64_t version, int32_t rSize, const STSRow *pRow) {} +static void tsdbEncodeRow(int64_t version, int32_t rlen, const STSRow *pRow) {} -static void tsdbDecodeRow(int64_t *version, int32_t *rSize, const STSRow **ppRow) {} +static void tsdbDecodeRow(int64_t *version, int32_t *rlen, const STSRow **ppRow) {} // SMemData From e08dd100a44d4b30f35964754683840b38279cd6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 May 2022 18:39:28 +0800 Subject: [PATCH 37/63] enh(index): support more data type --- include/libs/index/index.h | 7 +- source/libs/executor/src/indexoperator.c | 4 +- source/libs/index/inc/indexCache.h | 1 + source/libs/index/inc/indexInt.h | 4 +- source/libs/index/src/index.c | 4 +- source/libs/index/src/indexCache.c | 190 +++++++++++++++++------ source/libs/index/src/indexTfile.c | 14 +- source/libs/index/test/indexTests.cc | 44 +++--- source/libs/index/test/jsonUT.cc | 16 +- source/libs/transport/src/transSrv.c | 5 + 10 files changed, 194 insertions(+), 95 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c94d75338a..fa4cb1d2bd 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -56,7 +56,8 @@ typedef enum { QUERY_LESS_EQUAL, QUERY_GREATER_THAN, QUERY_GREATER_EQUAL, - QUERY_RANGE + QUERY_RANGE, + QUERY_MAX } EIndexQueryType; /* @@ -178,8 +179,8 @@ void indexOptsDestroy(SIndexOpts* opts); * @param: */ -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType, - const char* colName, int32_t nColName, const char* colVal, int32_t nColVal); +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal); void indexTermDestroy(SIndexTerm* p); /* diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 3b62bdd664..86a28605b2 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -252,8 +252,8 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu return TSDB_CODE_QRY_INVALID_INPUT; } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, - strlen(left->colName), right->condValue, strlen(right->condValue)); + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), + right->condValue, strlen(right->condValue)); if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 3ea986ad48..0daca26ec9 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -31,6 +31,7 @@ extern "C" { typedef struct MemTable { T_REF_DECLARE() SSkipList* mem; + void* pCache; } MemTable; typedef struct IndexCache { T_REF_DECLARE() diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 5c7b8b9afe..7b7050d80e 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -166,7 +166,9 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) + +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) + #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 83b5025ad0..e0c24ac3bd 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -244,8 +244,8 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde return 0; } -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType, - const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (tm == NULL) { return NULL; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 13768ce682..ad82dd0748 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -34,23 +34,58 @@ static char* indexCacheTermGet(const void* pData); static MemTable* indexInternalCacheCreate(int8_t type); -static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); /*comm func of compare, used in (LE/LT/GE/GT compare)*/ -static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, +static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s, RangeType type); +static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); + +static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, + RangeType type); typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); -static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { + int32_t ret = func(a, b); + switch (comType) { + case QUERY_LESS_THAN: { + if (ret < 0) return MATCH; + } break; + case QUERY_LESS_EQUAL: { + if (ret <= 0) return MATCH; + break; + } + case QUERY_GREATER_THAN: { + if (ret > 0) return MATCH; + break; + } + case QUERY_GREATER_EQUAL: { + if (ret >= 0) return MATCH; + } + } + return CONTINUE; +} +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_THAN, a, b); +} static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } @@ -58,20 +93,26 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MAT static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual}; -static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { - cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, - cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}; +static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = { + {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual, + cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}, + {cacheSearchTerm_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON, + cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON, + cacheSearchRange_JSON}}; static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); -static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { if (cache == NULL) { return 0; } + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + CacheTerm* pCt = &ct; - MemTable* mem = cache; - char* key = indexCacheTermGet(ct); + char* key = indexCacheTermGet(&ct); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { @@ -80,7 +121,7 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - if (0 == strcmp(c->colVal, ct->colVal)) { + if (0 == strcmp(c->colVal, pCt->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) // taosArrayPush(result, &c->uid); @@ -95,27 +136,33 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S tSkipListDestroyIter(iter); return 0; } -static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchSuffix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchRegex(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, +static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, RangeType type) { if (cache == NULL) { return 0; } _cache_range_compare cmpFn = rangeCompare[type]; - MemTable* mem = cache; - char* key = indexCacheTermGet(ct); + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* key = indexCacheTermGet(pCt); SSkipListIterator* iter = tSkipListCreateIter(mem->mem); while (tSkipListIterNext(iter)) { @@ -124,7 +171,7 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - TExeCond cond = cmpFn(c->colVal, ct->colVal, ct->colType); + TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) @@ -134,26 +181,61 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) } } else if (cond == CONTINUE) { + continue; } else if (cond == BREAK) { break; } } + taosMemoryFree(pCt); tSkipListDestroyIter(iter); return TSDB_CODE_SUCCESS; } -static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, LT); +static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, LT); } -static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, LE); +static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, LE); } -static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, GT); +static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, GT); } -static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, GE); +static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, GE); } -static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + +static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT); +} +static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, LE); +} +static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, GT); +} +static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE); +} +static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} + +static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, + RangeType type) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } @@ -167,6 +249,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in }; cache->mem = indexInternalCacheCreate(type); + cache->mem->pCache = cache; cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; @@ -379,11 +462,19 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u return 0; } -static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SIdxTempResult* tr, STermValueType* s) { +static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTempResult* tr, STermValueType* s) { if (mem == NULL) { return 0; } - return cacheSearch[qtype](mem, ct, tr, s); + + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + return cacheSearch[1][qtype](mem, term, tr, s); + } else { + return cacheSearch[0][qtype](mem, term, tr, s); + } } int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); @@ -400,25 +491,24 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result indexMemRef(imm); taosThreadMutexUnlock(&pCache->mtx); - SIndexTerm* term = query->term; - EIndexQueryType qtype = query->qType; + // SIndexTerm* term = query->term; + // EIndexQueryType qtype = query->qType; - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); - char* p = term->colVal; - if (hasJson) { - p = indexPackJsonData(term); - } - CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; + // bool isJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); + // char* p = term->colVal; + // if (isJson) { + // p = indexPackJsonData(term); + //} + // CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; - int ret = indexQueryMem(mem, &ct, qtype, result, s); + int ret = indexQueryMem(mem, query, result, s); if (ret == 0 && *s != kTypeDeletion) { // continue search in imm - ret = indexQueryMem(imm, &ct, qtype, result, s); - } - - if (hasJson) { - taosMemoryFreeClear(p); + ret = indexQueryMem(imm, query, result, s); } + // if (isJson) { + // taosMemoryFreeClear(p); + //} indexMemUnRef(mem); indexMemUnRef(imm); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 5aed2bd6b0..9edd868272 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -606,16 +606,16 @@ static bool tfileIteratorNext(Iterate* iiter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter)); - if (tIter == NULL) { + TFileFstIter* iter = taosMemoryCalloc(1, sizeof(TFileFstIter)); + if (iter == NULL) { return NULL; } - tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); - tIter->fb = fstSearch(reader->fst, tIter->ctx); - tIter->st = streamBuilderIntoStream(tIter->fb); - tIter->rdr = reader; - return tIter; + iter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); + iter->fb = fstSearch(reader->fst, iter->ctx); + iter->st = streamBuilderIntoStream(iter->fb); + iter->rdr = reader; + return iter; } Iterate* tfileIteratorCreate(TFileReader* reader) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index d8ea6a8233..896451c686 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -483,7 +483,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colName("voltage"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; @@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); @@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -623,7 +623,7 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); @@ -638,7 +638,7 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); @@ -670,7 +670,7 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -679,7 +679,7 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -701,7 +701,7 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -737,7 +737,7 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -759,7 +759,7 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -784,7 +784,7 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); @@ -792,7 +792,7 @@ class IndexObj { } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); @@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e1e5004701..f789d23136 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); @@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b78edf6cd0..9c905771d5 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -782,6 +782,11 @@ static void uvDestroyConn(uv_handle_t* handle) { tDebug("server conn %p destroy", conn); // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); + + if (conn->regArg.init == 1) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); taosMemoryFree(conn); From 041ecaa35f779f94e110fcb058d877050bfd79fd Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 6 May 2022 18:44:02 +0800 Subject: [PATCH 38/63] fix: some problems of parser and planner --- tests/script/tsim/query/explain.sim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index 66d3c48f5d..71f7969c83 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -25,15 +25,15 @@ sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=2 sql create table tb4 using st1 tags(4); sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)"); -sql create table tb1 using st2 tags(1); -sql insert into tb1 values (now, 1, "Hash Join (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)"); +#sql create table tb1 using st2 tags(1); +#sql insert into tb1 values (now, 1, "Hash Join (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)"); -sql create table tb2 using st2 tags(2); -sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2 (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)"); -sql create table tb3 using st2 tags(3); -sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)"); -sql create table tb4 using st2 tags(4); -sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)"); +#sql create table tb2 using st2 tags(2); +#sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2 (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)"); +#sql create table tb3 using st2 tags(3); +#sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)"); +#sql create table tb4 using st2 tags(4); +#sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)"); print ======== step2 From 3ac18d91a828ac17e0c0bf28da6c0423b4a8e664 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 6 May 2022 19:13:55 +0800 Subject: [PATCH 39/63] fix: some problems of parser and planner --- tests/script/tsim/query/session.sim | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index a69b6249fc..c39956c0df 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -65,6 +65,15 @@ sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.031', 5) sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.036', 6) sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.51', 7) +# vnode does not return the precision of the table +print ====> create database d1 precision 'us' +sql create database d1 precision 'us' +sql use d1 +sql create table dev_001 (ts timestamp ,i timestamp ,j int) +sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) +sql create table secondts(ts timestamp,t2 timestamp,i int) +sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) + $loop_test = 0 loop_test_pos: @@ -288,15 +297,6 @@ sql_error sql select count(*) from dev_001 session(ts,0s) sql_error select count(*) from dev_001 session(i,1y) sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0' -# vnode does not return the precision of the table -print ====> create database d1 precision 'us' -sql create database d1 precision 'us' -sql use d1 -sql create table dev_001 (ts timestamp ,i timestamp ,j int) -sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) -sql create table secondts(ts timestamp,t2 timestamp,i int) -sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4) - #print ====> select count(*) from dev_001 session(ts,1u) #sql select _wstartts, count(*) from dev_001 session(ts,1u) #print rows: $rows From 68fc2acc9caad5f1587f83d3767db1c5b529d0c4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 11:16:53 +0000 Subject: [PATCH 40/63] refact --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 114 ++++++++++++++------ 1 file changed, 81 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 1c263b7d00..e7b733369a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,10 +15,11 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemSkipListCurosr SMemSkipListCurosr; struct SMemTable { STsdb *pTsdb; @@ -57,6 +58,20 @@ struct SMemData { SMemSkipList sl; }; +struct SMemSkipListCurosr { + SMemSkipList *pSl; + SMemSkipListNode *pNodeC; +}; + +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) +#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +#define SL_HEAD_NODE(sl) ((sl)->pHead) +#define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel)) + // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { SMemTable *pMemTb = NULL; @@ -96,17 +111,18 @@ int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { } int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { - SMemData *pMemData; - STsdb *pTsdb = pMemTb->pTsdb; - SVnode *pVnode = pTsdb->pVnode; - SVBufPool *pPool = pVnode->inUse; - int32_t hash; - int32_t tlen; - uint8_t buf[16]; - int32_t rlen; - const uint8_t *p; - SMemSkipListNode *pSlNode; - const STSRow *pTSRow; + SMemData *pMemData; + STsdb *pTsdb = pMemTb->pTsdb; + SVnode *pVnode = pTsdb->pVnode; + SVBufPool *pPool = pVnode->inUse; + int32_t hash; + int32_t tlen; + uint8_t buf[16]; + int32_t rlen; + const uint8_t *p; + SMemSkipListNode *pSlNode; + const STSRow *pTSRow; + SMemSkipListCurosr slc = {0}; // search hash hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket; @@ -116,7 +132,11 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p // create pMemData if need if (pMemData == NULL) { - pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + int8_t maxLevel = pVnode->config.tsdbCfg.slLevel; + int32_t tsize = sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2; + SMemSkipListNode *pHead, *pTail; + + pMemData = vnodeBufPoolMalloc(pPool, tsize); if (pMemData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -131,46 +151,74 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p pMemData->maxVer = -1; pMemData->nRows = 0; pMemData->sl.seed = taosRand(); - pMemData->sl.maxLevel = pVnode->config.tsdbCfg.slLevel; + pMemData->sl.maxLevel = maxLevel; pMemData->sl.level = 0; pMemData->sl.size = 0; + pHead = SL_HEAD_NODE(&pMemData->sl); + pTail = SL_TAIL_NODE(&pMemData->sl); + pHead->level = maxLevel; + pTail->level = maxLevel; + for (int iLevel = 0; iLevel < maxLevel; iLevel++) { + SL_NODE_FORWARD(pHead, iLevel) = pTail; + SL_NODE_FORWARD(pTail, iLevel) = pHead; + } // add to MemTable hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket; pMemData->pHashNext = pMemTb->pBuckets[hash]; pMemTb->pBuckets[hash] = pMemData; + pMemTb->nHash++; } - // loop to insert data to skiplist +// loop to insert data to skiplist +#if 0 + tsdbMemSkipListCursorOpen(&slc, &pMemData->sl); p = pSubmitBlk->pData; for (;;) { if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break; - // p = tGetLen(p, &rlen); - pTSRow = (STSRow *)p; - p += rlen; - if (pTSRow == NULL) break; + const uint8_t *pt = p; + p = tGetBinary(p, &pTSRow, &rlen); // check the row (todo) // move the cursor to position to write (todo) + int32_t c; + tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c); + ASSERT(c); - // insert the row - int8_t level = 1; - tlen = 0; // sizeof(int64_t) + tsdbPutLen(rlen) + rlen; - pSlNode = vnodeBufPoolMalloc(pPool, tlen); - if (pSlNode == NULL) { - ASSERT(0); - } + // encode row + int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); + int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (p - pt); + pSlNode = vnodeBufPoolMalloc(pPool, tsize); + pSlNode->level = level; + + uint8_t *pData = SL_NODE_DATA(pSlNode); + *(int64_t *)pData = version; + pData += sizeof(version); + memcpy(pData, pt, p - pt); + + // insert row + tsdbMemSkipListCursorPut(&slc, pSlNode); + + // update status + if (pTSRow->ts < pMemData->minKey) pMemData->minKey = pTSRow->ts; + if (pTSRow->ts > pMemData->maxKey) pMemData->maxKey = pTSRow->ts; } + tsdbMemSkipListCursorClose(&slc); +#endif + + if (pMemData->minVer == -1) pMemData->minVer = version; + if (pMemData->maxVer == -1 || pMemData->maxVer < version) pMemData->maxVer = version; + + if (pMemTb->minKey < pMemData->minKey) pMemTb->minKey = pMemData->minKey; + if (pMemTb->maxKey < pMemData->maxKey) pMemTb->maxKey = pMemData->maxKey; + if (pMemTb->minVer == -1) pMemTb->minVer = version; + if (pMemTb->maxVer == -1 || pMemTb->maxVer < version) pMemTb->maxVer = version; return 0; } -static void tsdbEncodeRow(int64_t version, int32_t rlen, const STSRow *pRow) {} - -static void tsdbDecodeRow(int64_t *version, int32_t *rlen, const STSRow **ppRow) {} - // SMemData // SMemSkipList \ No newline at end of file From bcdf9c19b2dc0eab830877a620f1d8a3cc2d51db Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 6 May 2022 19:31:20 +0800 Subject: [PATCH 41/63] feat: update refactor --- Jenkinsfile2 | 279 +++++--- cmake/cmake.define | 2 + include/common/tdataformat.h | 7 +- include/common/tmsg.h | 37 +- include/common/trow.h | 4 +- include/libs/catalog/catalog.h | 17 + include/libs/function/tudf.h | 15 +- include/libs/scalar/scalar.h | 2 + include/libs/transport/trpc.h | 1 + include/os/osSocket.h | 1 + source/client/src/clientHb.c | 77 +++ source/client/test/clientTests.cpp | 75 +- source/common/src/tdatablock.c | 10 +- source/common/src/tmsg.c | 204 +++++- source/common/src/trow.c | 1 + source/dnode/mgmt/implement/src/dmTransport.c | 12 +- source/dnode/mnode/impl/inc/mndDef.h | 2 + source/dnode/mnode/impl/inc/mndSma.h | 1 - source/dnode/mnode/impl/inc/mndUser.h | 1 + source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 219 ++---- source/dnode/mnode/impl/src/mndFunc.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 10 + source/dnode/mnode/impl/src/mndSma.c | 49 +- source/dnode/mnode/impl/src/mndTrans.c | 4 +- source/dnode/mnode/impl/src/mndUser.c | 140 +++- source/dnode/mnode/impl/src/mndVgroup.c | 166 +++-- source/dnode/mnode/impl/test/func/func.cpp | 6 + source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 - source/dnode/vnode/src/tsdb/tsdbMemTable.c | 55 +- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 93 +++ source/dnode/vnode/src/tsdb/tsdbRead.c | 4 + source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 1 + source/dnode/vnode/src/tsdb/tsdbSma.c | 15 + source/dnode/vnode/src/vnd/vnodeSvr.c | 76 +- source/libs/catalog/inc/catalogInt.h | 20 + source/libs/catalog/src/catalog.c | 271 ++++++++ source/libs/executor/inc/executorimpl.h | 21 +- source/libs/executor/inc/tsort.h | 15 +- source/libs/executor/src/executorimpl.c | 63 +- source/libs/executor/src/scanoperator.c | 82 ++- source/libs/executor/src/sortoperator.c | 59 +- source/libs/executor/src/tsort.c | 40 +- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/tudf.c | 4 +- source/libs/function/src/udfd.c | 25 +- source/libs/parser/src/parInsert.c | 52 +- source/libs/parser/src/parInsertData.c | 5 - source/libs/qcom/src/querymsg.c | 36 + source/libs/scalar/src/sclfunc.c | 6 + source/libs/scalar/src/sclvector.c | 6 +- source/libs/transport/src/trans.c | 14 +- source/libs/transport/src/transSrv.c | 5 +- source/os/src/osDir.c | 2 +- source/os/src/osSocket.c | 42 ++ tests/requirements.txt | 5 + tests/script/tsim/query/scalarFunction.sim | 5 + tests/script/tsim/query/udf.sim | 21 + .../1-insert/insertWithMoreVgroup.py | 291 ++++++++ tests/system-test/2-query/join.py | 238 +++++-- tests/system-test/2-query/log.py | 14 +- tests/system-test/2-query/pow.py | 652 ++++++++++++++++++ tests/system-test/2-query/smaTest.py | 131 ++++ tests/system-test/2-query/sqrt.py | 551 +++++++++++++++ tests/system-test/fulltest.sh | 4 +- tools/shell/src/shellEngine.c | 3 + tools/shell/src/shellNettest.c | 5 +- 70 files changed, 3588 insertions(+), 699 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbMemTable2.c create mode 100644 tests/requirements.txt create mode 100644 tests/system-test/1-insert/insertWithMoreVgroup.py create mode 100644 tests/system-test/2-query/pow.py create mode 100644 tests/system-test/2-query/smaTest.py create mode 100644 tests/system-test/2-query/sqrt.py diff --git a/Jenkinsfile2 b/Jenkinsfile2 index c9c9c3a7ca..49e13b7831 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -8,135 +8,202 @@ def skipbuild=0 def win_stop=0 def abortPreviousBuilds() { - def currentJobName = env.JOB_NAME - def currentBuildNumber = env.BUILD_NUMBER.toInteger() - def jobs = Jenkins.instance.getItemByFullName(currentJobName) - def builds = jobs.getBuilds() + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() - for (build in builds) { - if (!build.isBuilding()) { - continue; + for (build in builds) { + if (!build.isBuilding()) { + continue; + } + + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } + + build.doKill() //doTerm(),doKill(),doTerm() } - - if (currentBuildNumber == build.getNumber().toInteger()) { - continue; - } - - build.doKill() //doTerm(),doKill(),doTerm() - } } // abort previous build abortPreviousBuilds() def abort_previous(){ - def buildNumber = env.BUILD_NUMBER as int - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) } def pre_test(){ - sh'hostname' + sh 'hostname' sh ''' - date - sudo rmtaos || echo "taosd has not installed" + date + sudo rmtaos || echo "taosd has not installed" ''' sh ''' - killall -9 taosd ||echo "no taosd running" - killall -9 gdb || echo "no gdb running" - killall -9 python3.8 || echo "no python program running" - cd ${WKC} + killall -9 taosd ||echo "no taosd running" + killall -9 gdb || echo "no gdb running" + killall -9 python3.8 || echo "no python program running" + cd ${WKC} ''' script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WKC} - git checkout master - ''' + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + sh ''' + cd ${WKC} + git checkout 3.0 + [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. + ''' + } else { + sh ''' + cd ${WKC} + git checkout develop + ''' } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WKC} - git checkout 2.0 - ''' - } - else if(env.CHANGE_TARGET == '3.0'){ - sh ''' - cd ${WKC} - git checkout 3.0 - [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. - ''' - } - else{ - sh ''' - cd ${WKC} - git checkout develop - ''' - } } - sh''' - cd ${WKC} - git pull >/dev/null - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD - git submodule update --init --recursive + sh ''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git submodule update --init --recursive ''' - sh''' - cd ${WKC} - export TZ=Asia/Harbin - date - rm -rf debug - mkdir debug - cd debug - cmake .. > /dev/null - make -j4> /dev/null + sh ''' + cd ${WKC} + export TZ=Asia/Harbin + date + rm -rf debug + mkdir debug + cd debug + cmake .. > /dev/null + make -j4> /dev/null ''' - sh''' - cd ${WKPY} - git reset --hard - git pull - pip3 install . + sh ''' + cd ${WKPY} + git reset --hard + git pull + pip3 install . + ''' + return 1 +} +def pre_test_win(){ + bat ''' + hostname + date /t + time /t + taskkill /f /t /im python.exe + taskkill /f /t /im bash.exe + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine\\debug + exit 0 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git reset --hard + git fetch || git fetch + git checkout -f + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 3.0 + ''' + } else { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout develop + ''' + } + } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git branch + git pull || git pull + git fetch origin +refs/pull/%CHANGE_ID%/merge + git checkout -qf FETCH_HEAD + ''' +} +def pre_test_build_win() { + bat ''' + echo "building ..." + time /t + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + mkdir debug + cd debug + call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 + set CL=/MP8 + cmake .. -G "NMake Makefiles JOM" + jom -j 4 || exit 8 + time /t ''' return 1 } pipeline { - agent none - options { skipDefaultCheckout() } - environment{ - WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' - } - stages { - stage('pre_build'){ - agent{label " slave3_0 || slave15 || slave16 || slave17 "} - options { skipDefaultCheckout() } - when { - changeRequest() - } - steps { - script{ - abort_previous() - abortPreviousBuilds() + agent none + options { skipDefaultCheckout() } + environment{ + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDengine' + WKPY= '/var/lib/jenkins/workspace/taos-connector-python' + } + stages { + stage('run test') { + parallel { + stage('windows test') { + agent {label " windows11 "} + steps { + pre_test_win() + pre_test_build_win() + } + } + stage('linux test') { + agent{label " slave3_0 || slave15 || slave16 || slave17 "} + options { skipDefaultCheckout() } + when { + changeRequest() + } + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + cd ${WKC}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + sh ''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' + } + } + } } - timeout(time: 45, unit: 'MINUTES'){ - pre_test() - sh''' - cd ${WKC}/debug - ctest -VV - ''' - sh''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - sh''' - cd ${WKC}/tests - ./test-all.sh b1fq - ''' - } - } - } - } - post { + } + } + post { success { emailext ( subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", diff --git a/cmake/cmake.define b/cmake/cmake.define index 1655154506..d1d9266bca 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.16) set(CMAKE_VERBOSE_MAKEFILE OFF) +SET(BUILD_SHARED_LIBS "OFF") + #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 35b57ba117..74d11a5ffb 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -372,10 +372,9 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche // ----------------- Data column structure // SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { - int8_t type; // column type - uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint8_t bitmapMode : 1; // default is 0(2 bits), otherwise 1(1 bit) - uint8_t reserve : 6; + int8_t type; // column type + uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows + uint8_t reserve : 7; int16_t colId; // column ID int32_t bytes; // column data bytes defined int32_t offset; // data offset in a SDataRow (including the header size) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f6160cbbf5..3889784f2f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -73,7 +73,8 @@ typedef uint16_t tmsg_t; enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX }; enum { - HEARTBEAT_KEY_DBINFO = 1, + HEARTBEAT_KEY_USER_AUTHINFO = 1, + HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_MQ_TMP, }; @@ -426,7 +427,9 @@ int32_t tDeserializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* typedef struct { char user[TSDB_USER_LEN]; + int32_t version; int8_t superAuth; + SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; } SGetUserAuthRsp; @@ -669,10 +672,20 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; + int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserAuthRsp +} SUserAuthBatchRsp; + +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); + + typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -2578,6 +2591,28 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } +#define TD_AUTO_CREATE_TABLE 0x1 +typedef struct { + int64_t suid; + int64_t uid; + int32_t sver; + uint64_t nData; + const void* pData; + SVCreateTbReq cTbReq; +} SVSubmitBlk; + +typedef struct { + int32_t flags; + int32_t nBlocks; + union { + SArray* pArray; + SVSubmitBlk* pBlocks; + }; +} SVSubmitReq; + +int32_t tEncodeSVSubmitReq(SCoder* pCoder, const SVSubmitReq* pReq); +int32_t tDecodeSVSubmitReq(SCoder* pCoder, SVSubmitReq* pReq); + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/trow.h b/include/common/trow.h index fe1cf0d967..0d34c6e49f 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -122,8 +122,6 @@ typedef struct { typedef struct { /// timestamp TSKEY ts; - /// row version - uint64_t ver; union { /// union field for encode and decode uint32_t info; @@ -176,7 +174,7 @@ typedef struct { #define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_KEY(r) ((r)->ts) -#define TD_ROW_VER(r) ((r)->ver) +// #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 30d1bd0a51..04a24c4f32 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -40,6 +40,11 @@ enum { CTG_DBG_STB_RENT_NUM, }; +typedef enum { + AUTH_TYPE_READ = 1, + AUTH_TYPE_WRITE, + AUTH_TYPE_OTHER, +} AUTH_TYPE; typedef struct SCatalogReq { SArray *pTableName; // element is SNAME @@ -57,6 +62,7 @@ typedef struct SMetaData { typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; + uint32_t maxUserCacheNum; uint32_t dbRentSec; uint32_t stbRentSec; } SCatalogCfg; @@ -77,6 +83,11 @@ typedef struct SDbVgVersion { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; +typedef struct SUserAuthVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserAuthVersion; + typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; @@ -219,12 +230,18 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stable int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num); + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); + /** * Destroy catalog and relase all resources diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 22314e4bd6..d59a7c23f7 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -29,7 +29,11 @@ extern "C" { #endif #define UDF_LISTEN_PIPE_NAME_LEN 32 -#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." +#ifdef _WIN32 +#define UDF_LISTEN_PIPE_NAME_PREFIX "\\\\?\\pipe\\udfd.sock" +#else +#define UDF_LISTEN_PIPE_NAME_PREFIX ".udfd.sock." +#endif #define UDF_DNODE_ID_ENV_NAME "DNODE_ID" //====================================================================================== @@ -129,8 +133,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); // begin API to UDF writer. // dynamic lib init and destroy -typedef int32_t (*TUdfSetupFunc)(); -typedef int32_t (*TUdfTeardownFunc)(); +typedef int32_t (*TUdfInitFunc)(); +typedef int32_t (*TUdfDestroyFunc)(); //TODO: add API to check function arguments type, number etc. @@ -144,7 +148,7 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t ne return TSDB_CODE_SUCCESS; } - int allocCapacity = MAX(data->rowsAlloc, 8); + int allocCapacity = TMAX(data->rowsAlloc, 8); while (allocCapacity < newCapacity) { allocCapacity *= UDF_MEMORY_EXP_GROWTH; } @@ -238,11 +242,10 @@ static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRo data->varLenCol.payloadLen += dataLen; } } - data->numOfRows = MAX(currentRow + 1, data->numOfRows); + data->numOfRows = TMAX(currentRow + 1, data->numOfRows); return 0; } -typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 0c7db45c4b..555274599a 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -91,6 +91,8 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + #ifdef __cplusplus } #endif diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 0e7d486eab..a7d1522d12 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -68,6 +68,7 @@ typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, cha typedef bool (*RpcRfp)(int32_t code); typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; uint16_t localPort; // local port char * label; // for debug purpose int numOfThreads; // number of threads to handle connections diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 62c3771669..213a6930ee 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -161,6 +161,7 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +bool taosValidIpAndPort(uint32_t ip, uint16_t port); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b11a49fa1a..fc39e80c1e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,6 +28,27 @@ static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SUserAuthBatchRsp batchRsp = {0}; + if (tDeserializeSUserAuthBatchRsp(value, valueLen, &batchRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i); + tscDebug("hb user auth rsp, user:%s, version:%d", rsp->user, rsp->version); + + catalogUpdateUserAuthInfo(pCatalog, rsp); + } + + tFreeSUserAuthBatchRsp(&batchRsp); + return TSDB_CODE_SUCCESS; +} + static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -148,6 +169,24 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pRsp->info, i); switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } case HEARTBEAT_KEY_DBINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value); @@ -327,6 +366,39 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SUserAuthVersion *users = NULL; + uint32_t userNum = 0; + int32_t code = 0; + + code = catalogGetExpiredUsers(pCatalog, &users, &userNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (userNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthVersion *user = &users[i]; + user->version = htonl(user->version); + } + + SKv kv = { + .key = HEARTBEAT_KEY_USER_AUTHINFO, + .valueLen = sizeof(SUserAuthVersion) * userNum, + .value = users, + }; + + tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -407,6 +479,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); + code = hbGetExpiredUserInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = hbGetExpiredDBInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 2874220121..fc5781cb4d 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -587,15 +587,34 @@ TEST(testCase, projection_query_tables) { } taos_free_result(pRes); + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } taos_free_result(pRes); - for(int32_t i = 0; i < 10000; ++i) { - char sql[512] = {0}; - sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + pRes = taos_query(pConn, "create table tu2 using st2 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -604,24 +623,44 @@ TEST(testCase, projection_query_tables) { taos_free_result(p); } - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); + printf("start to insert next table\n"); + + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); } - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +// pRes = taos_query(pConn, "select * from tu"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } - taos_free_result(pRes); +// taos_free_result(pRes); taos_close(pConn); } @@ -659,7 +698,7 @@ TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use db"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); if (taos_errno(pRes) != 0) { printf("failed to use db, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4f4b8b196f..a4b36dcb67 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -363,9 +363,9 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { int32_t mapIndex = i; - if (pIndexMap) { - mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); - } +// if (pIndexMap) { +// mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); +// } SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); @@ -493,12 +493,12 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { bool isNull = false; if (pBlock->pBlockAgg == NULL) { - isNull = colDataIsNull(pColData, pBlock->info.rows, j, NULL); + isNull = colDataIsNull_s(pColData, pBlock->info.rows); } else { isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); } - char* p = colDataGetData(pColData, j); + char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a91f5f8223..6278b52a04 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1164,31 +1164,47 @@ int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq * return 0; } -int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - SCoder encoder = {0}; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->superAuth) < 0) return -1; +int32_t tSerializeSGetUserAuthRspImpl(SCoder *pEncoder, SGetUserAuthRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->superAuth) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); - if (tEncodeI32(&encoder, numOfReadDbs) < 0) return -1; - if (tEncodeI32(&encoder, numOfWriteDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1; - char *db = taosHashIterate(pRsp->readDbs, NULL); + char *db = taosHashIterate(pRsp->createdDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; + db = taosHashIterate(pRsp->createdDbs, db); + } + + db = taosHashIterate(pRsp->readDbs, NULL); + while (db != NULL) { + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->readDbs, db); } db = taosHashIterate(pRsp->writeDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->writeDbs, db); } + return 0; +} + +int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tSerializeSGetUserAuthRspImpl(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1196,39 +1212,58 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR return tlen; } -int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) { + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { return -1; } - SCoder decoder = {0}; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = 0; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - if (tDecodeI32(&decoder, &numOfReadDbs) < 0) return -1; - if (tDecodeI32(&decoder, &numOfWriteDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) return -1; + + for (int32_t i = 0; i < numOfCreatedDbs; ++i) { + char db[TSDB_DB_FNAME_LEN] = {0}; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + int32_t len = strlen(db) + 1; + taosHashPut(pRsp->createdDbs, db, len, db, len); + } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->writeDbs, db, len, db, len); } + return 0; +} + + +int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDeserializeSGetUserAuthRspImpl(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); tCoderClear(&decoder); @@ -1236,6 +1271,7 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * } void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { + taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->writeDbs); } @@ -2055,6 +2091,62 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSGetUserAuthRspImpl(&encoder, pUserAuthRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserAuthRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp rsp = {0}; + if (tDeserializeSGetUserAuthRspImpl(&decoder, &rsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + tFreeSGetUserAuthRsp(pUserAuthRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3860,3 +3952,65 @@ int32_t tDecodeSVDropStbReq(SCoder *pCoder, SVDropStbReq *pReq) { tEndDecode(pCoder); return 0; } + +static int32_t tEncodeSVSubmitBlk(SCoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; + if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; + if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVSubmitBlk(SCoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; + if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; + if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; + if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVSubmitReq(SCoder *pCoder, const SVSubmitReq *pReq) { + int32_t nBlocks = taosArrayGetSize(pReq->pArray); + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; + if (tEncodeI32v(pCoder, nBlocks) < 0) return -1; + for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { + if (tEncodeSVSubmitBlk(pCoder, (SVSubmitBlk *)taosArrayGet(pReq->pArray, iBlock), pReq->flags) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVSubmitReq(SCoder *pCoder, SVSubmitReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->nBlocks) < 0) return -1; + pReq->pBlocks = tCoderMalloc(pCoder, sizeof(SVSubmitBlk) * pReq->nBlocks); + if (pReq->pBlocks == NULL) return -1; + for (int32_t iBlock = 0; iBlock < pReq->nBlocks; iBlock++) { + if (tDecodeSVSubmitBlk(pCoder, pReq->pBlocks + iBlock, pReq->flags) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} \ No newline at end of file diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 590e6097c4..4c063b205f 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -684,6 +684,7 @@ _err: tdFreeDataCols(pTarget); return -1; } + static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, bool isMerge) { for (int i = 0; i < src->numOfCols; ++i) { ASSERT(target->cols[i].type == src->cols[i].type); diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 446894556e..114d7b6dfc 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "dmImp.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dmGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { @@ -130,10 +130,10 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; if (msgType == TDMT_DND_SERVER_STATUS) { @@ -517,7 +517,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); + void * pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -547,6 +547,8 @@ static int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; + + strncpy(rpcInit.localFqdn, pDnode->data.localFqdn, strlen(pDnode->data.localFqdn)); rpcInit.localPort = pDnode->data.serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = tsNumOfRpcThreads; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4a0df80358..89638524ae 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -255,8 +255,10 @@ typedef struct { int64_t updateTime; int8_t superUser; int32_t acctId; + int32_t authVersion; SHashObj* readDbs; SHashObj* writeDbs; + SRWLatch lock; } SUserObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 91c6e24e28..4a80f619d3 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,7 +26,6 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index b3eb7f2f95..2140d0fa67 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -29,6 +29,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index f42829eddf..8aecc22454 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -33,6 +33,7 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dfad99bdfb..af7efb5543 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -44,16 +44,17 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); int32_t mndInitDb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_DB, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndDbActionEncode, - .decodeFp = (SdbDecodeFp)mndDbActionDecode, - .insertFp = (SdbInsertFp)mndDbActionInsert, - .updateFp = (SdbUpdateFp)mndDbActionUpdate, - .deleteFp = (SdbDeleteFp)mndDbActionDelete}; + SSdbTable table = { + .sdbType = SDB_DB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndDbActionEncode, + .decodeFp = (SdbDecodeFp)mndDbActionDecode, + .insertFp = (SdbInsertFp)mndDbActionInsert, + .updateFp = (SdbUpdateFp)mndDbActionUpdate, + .deleteFp = (SdbDeleteFp)mndDbActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq); @@ -61,7 +62,6 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); - mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); @@ -194,6 +194,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pDb->lock); terrno = 0; @@ -222,17 +223,29 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { mTrace("db:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); - SArray *pOldRetensions = pOld->cfg.pRetensions; pOld->updateTime = pNew->updateTime; pOld->cfgVersion = pNew->cfgVersion; pOld->vgVersion = pNew->vgVersion; - memcpy(&pOld->cfg, &pNew->cfg, sizeof(SDbCfg)); - pNew->cfg.pRetensions = pOldRetensions; + pOld->cfg.buffer = pNew->cfg.buffer; + pOld->cfg.pages = pNew->cfg.pages; + pOld->cfg.pageSize = pNew->cfg.pageSize; + pOld->cfg.daysPerFile = pNew->cfg.daysPerFile; + pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; + pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; + pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; + pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod; + pOld->cfg.walLevel = pNew->cfg.walLevel; + pOld->cfg.strict = pNew->cfg.strict; + pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow; + pOld->cfg.replications = pNew->cfg.replications; taosWUnLockLatch(&pOld->lock); return 0; } -static int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { return sdbGetTableVer(pMnode->pSdb, SDB_VGROUP); } +static inline int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + return sdbGetTableVer(pSdb, SDB_VGROUP); +} SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; @@ -638,69 +651,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { - SAlterVnodeReq alterReq = {0}; - alterReq.vgVersion = pVgroup->version; - alterReq.buffer = pDb->cfg.buffer; - alterReq.pages = pDb->cfg.pages; - alterReq.pageSize = pDb->cfg.pageSize; - alterReq.daysPerFile = pDb->cfg.daysPerFile; - alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; - alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; - alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; - alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; - alterReq.walLevel = pDb->cfg.walLevel; - alterReq.strict = pDb->cfg.strict; - alterReq.cacheLastRow = pDb->cfg.cacheLastRow; - alterReq.replica = pVgroup->replica; - alterReq.selfIndex = -1; - - for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; - SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; - SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pVgidDnode == NULL) { - return NULL; - } - - pReplica->id = pVgidDnode->id; - pReplica->port = pVgidDnode->port; - memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); - mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } - } - - if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; - return NULL; - } - - int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); - if (contLen < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - contLen += +sizeof(SMsgHead); - - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); - *pContLen = contLen; - return pReq; -} - -static int32_t mndBuilAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -736,7 +687,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuilAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -752,19 +703,19 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndAlterDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); - if (pTrans == NULL) goto UPDATE_DB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); mndTransSetDbInfo(pTrans, pOld); - if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -UPDATE_DB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -778,7 +729,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_DB_OVER; + goto _OVER; } mDebug("db:%s, start to alter", alterReq.db); @@ -786,24 +737,26 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_DB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto ALTER_DB_OVER; + goto _OVER; } if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { - goto ALTER_DB_OVER; + goto _OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); + dbObj.cfg.numOfRetensions = 0; + dbObj.cfg.pRetensions = NULL; code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - goto ALTER_DB_OVER; + goto _OVER; } dbObj.cfgVersion++; @@ -811,7 +764,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndAlterDb(pMnode, pReq, pDb, &dbObj); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_DB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } @@ -831,13 +784,13 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto GET_DB_CFG_OVER; + goto _OVER; } pDb = mndAcquireDb(pMnode, cfgReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto GET_DB_CFG_OVER; + goto _OVER; } cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; @@ -866,7 +819,7 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto GET_DB_CFG_OVER; + goto _OVER; } tSerializeSDbCfgRsp(pRsp, contLen, &cfgRsp); @@ -876,9 +829,9 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { code = 0; -GET_DB_CFG_OVER: +_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } @@ -1097,7 +1050,8 @@ _OVER: return code; } -void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { +static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) { + int32_t numOfTables = 0; int32_t vindex = 0; SSdb *pSdb = pMnode->pSdb; @@ -1108,8 +1062,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - *num += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; - + numOfTables += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; vindex++; } @@ -1117,6 +1070,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { } sdbCancelFetch(pSdb, pIter); + return numOfTables; } static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { @@ -1170,8 +1124,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs return -1; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); @@ -1195,7 +1148,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto USE_DB_OVER; + goto _OVER; } char *p = strchr(usedbReq.db, '.'); @@ -1206,12 +1159,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); if (usedbRsp.pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; + goto _OVER; } mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; - } else { usedbRsp.vgVersion = usedbReq.vgVersion; } @@ -1232,15 +1184,15 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { } else { pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto USE_DB_OVER; + goto _OVER; } if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; + goto _OVER; } if (mndExtractDbInfo(pMnode, pDb, &usedbRsp, &usedbReq) < 0) { - goto USE_DB_OVER; + goto _OVER; } code = 0; @@ -1252,7 +1204,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto USE_DB_OVER; + goto _OVER; } tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); @@ -1260,7 +1212,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { pReq->pRsp = pRsp; pReq->rspLen = contLen; -USE_DB_OVER: +_OVER: if (code != 0) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } @@ -1298,8 +1250,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, continue; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName); @@ -1514,9 +1465,6 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)b, false); } - - // pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); - // *(int8_t *)pWrite = pDb->cfg.update; } static void setInformationSchemaDbCfg(SDbObj *pDbObj) { @@ -1544,7 +1492,6 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - *numOfTables += pVgroup->numOfTables; return true; } @@ -1594,49 +1541,3 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } - -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { - SUserIndexReq indexReq = {0}; - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SUserIndexRsp rsp = {0}; - bool exist = false; - - if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - code = mndProcessGetSmaReq(pMnode, &indexReq, &rsp, &exist); - if (code) { - goto _OVER; - } - - if (!exist) { - // TODO GET INDEX FROM FULLTEXT - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; - } else { - int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto _OVER; - } - - tSerializeSUserIndexRsp(pRsp, contLen, &rsp); - - pReq->pRsp = pRsp; - pReq->rspLen = contLen; - - code = 0; - } - -_OVER: - if (code != 0) { - mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); - } - - return code; -} diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b9331e6e03..3ac2951b6f 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -314,7 +314,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { + if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 59d07fd4aa..2de337537f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -403,6 +403,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SKv *kv = pIter; switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + void * rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserAuthInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserAuthVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_AUTHINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } case HEARTBEAT_KEY_DBINFO: { void * rspMsg = NULL; int32_t rspLen = 0; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0e3e8b68cc..8619df978b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -40,6 +40,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq); static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); @@ -56,6 +57,7 @@ int32_t mndInitSma(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); @@ -686,7 +688,7 @@ _OVER: return code; } -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { +static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { int32_t code = -1; SSmaObj *pSma = NULL; @@ -715,6 +717,51 @@ int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexR } mndReleaseSma(pMnode, pSma); + return code; +} + +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq) { + SUserIndexReq indexReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserIndexRsp rsp = {0}; + bool exist = false; + + if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + code = mndGetSma(pMnode, &indexReq, &rsp, &exist); + if (code) { + goto _OVER; + } + + if (!exist) { + // TODO GET INDEX FROM FULLTEXT + code = -1; + terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + } else { + int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto _OVER; + } + + tSerializeSUserIndexRsp(pRsp, contLen, &rsp); + + pReq->pRsp = pRsp; + pReq->rspLen = contLen; + + code = 0; + } + +_OVER: + if (code != 0) { + mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); + } return code; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f3438bd0dc..43e2735984 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1050,7 +1050,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since %s", terrstr()); } return code; @@ -1058,7 +1058,7 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5e15bdeb43..8b5fa06faf 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -186,6 +186,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pUser->lock); terrno = 0; @@ -228,11 +229,12 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { mTrace("user:%s, perform update action, old row:%p new row:%p", pOld->user, pOld, pNew); - memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); + taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; - + memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); + taosWUnLockLatch(&pOld->lock); return 0; } @@ -426,8 +428,12 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } memcpy(&newUser, pUser, sizeof(SUserObj)); + + taosRLockLatch(&pUser->lock); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { goto _OVER; } @@ -451,13 +457,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { taosHashClear(newUser.readDbs); + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; @@ -467,13 +476,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { taosHashClear(newUser.writeDbs); + newUser.authVersion++; } else { terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; goto _OVER; @@ -576,6 +588,38 @@ _OVER: return code; } +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { + memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); + pRsp->superAuth = pUser->superUser; + pRsp->version = pUser->authVersion; + taosRLockLatch(&pUser->lock); + pRsp->readDbs = mndDupDbHash(pUser->readDbs); + pRsp->writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pRsp->createdDbs) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SDbObj *pDb = NULL; + pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); + if (pIter == NULL) break; + + if (strcmp(pDb->createUser, pUser->user) == 0) { + int32_t len = strlen(pDb->name) + 1; + taosHashPut(pRsp->createdDbs, pDb->name, len, pDb->name, len); + } + + sdbRelease(pSdb, pDb); + } + + return 0; +} + static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; @@ -596,25 +640,9 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { goto _OVER; } - memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); - authRsp.superAuth = pUser->superUser; - authRsp.readDbs = mndDupDbHash(pUser->readDbs); - authRsp.writeDbs = mndDupDbHash(pUser->writeDbs); - - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDbObj *pDb = NULL; - pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); - if (pIter == NULL) break; - - if (strcmp(pDb->createUser, pUser->user) == 0) { - int32_t len = strlen(pDb->name) + 1; - taosHashPut(authRsp.readDbs, pDb->name, len, pDb->name, len); - taosHashPut(authRsp.writeDbs, pDb->name, len, pDb->name, len); - } - - sdbRelease(pSdb, pDb); + code = mndSetUserAuthRsp(pMnode, pUser, &authRsp); + if (code) { + goto _OVER; } int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); @@ -631,6 +659,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -681,3 +710,72 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { + SUserAuthBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); + if (batchRsp.pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t code = 0; + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to auth user since %s", pUsers[i].user, terrstr()); + continue; + } + + if (pUser->authVersion <= pUsers[i].version) { + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserAuthRsp rsp = {0}; + code = mndSetUserAuthRsp(pMnode, pUser, &rsp); + if (code) { + mndReleaseUser(pMnode, pUser); + tFreeSGetUserAuthRsp(&rsp); + goto _OVER; + } + + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + } + + int32_t rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUserAuthBatchRsp(&batchRsp); + return -1; + } + tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + +_OVER: + + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return code; +} + + + diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31e4a8ea7d..d1e4be1161 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,8 +21,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUMBER 1 -#define TSDB_VGROUP_RESERVE_SIZE 64 +#define VGROUP_VER_NUMBER 1 +#define VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); @@ -34,19 +34,21 @@ static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_VGROUP, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, - .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, - .insertFp = (SdbInsertFp)mndVgroupActionInsert, - .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, - .deleteFp = (SdbDeleteFp)mndVgroupActionDelete}; + SSdbTable table = { + .sdbType = SDB_VGROUP, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, + .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, + .insertFp = (SdbInsertFp)mndVgroupActionInsert, + .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); @@ -66,29 +68,29 @@ void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); - if (pRaw == NULL) goto VG_ENCODE_OVER; + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, VGROUP_VER_NUMBER, sizeof(SVgObj) + VGROUP_RESERVE_SIZE); + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->version, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, VG_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, VG_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, VG_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -VG_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to encode to raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -103,41 +105,41 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto VG_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_VGROUP_VER_NUMBER) { + if (sver != VGROUP_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto VG_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); - if (pRow == NULL) goto VG_DECODE_OVER; + if (pRow == NULL) goto _OVER; SVgObj *pVgroup = sdbGetRowObj(pRow); - if (pVgroup == NULL) goto VG_DECODE_OVER; + if (pVgroup == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, VG_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, VG_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, _OVER) if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) terrno = 0; -VG_DECODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -254,6 +256,68 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SAlterVnodeReq alterReq = {0}; + alterReq.vgVersion = pVgroup->version; + alterReq.buffer = pDb->cfg.buffer; + alterReq.pages = pDb->cfg.pages; + alterReq.pageSize = pDb->cfg.pageSize; + alterReq.daysPerFile = pDb->cfg.daysPerFile; + alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; + alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; + alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; + alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + alterReq.walLevel = pDb->cfg.walLevel; + alterReq.strict = pDb->cfg.strict; + alterReq.cacheLastRow = pDb->cfg.cacheLastRow; + alterReq.replica = pVgroup->replica; + alterReq.selfIndex = -1; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SReplica *pReplica = &alterReq.replicas[v]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pVgidDnode == NULL) { + return NULL; + } + + pReplica->id = pVgidDnode->id; + pReplica->port = pVgidDnode->port; + memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); + mndReleaseDnode(pMnode, pVgidDnode); + + if (pDnode->id == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + + if (alterReq.selfIndex == -1) { + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += +sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); + *pContLen = contLen; + return pReq; +} + void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SDropVnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; @@ -372,12 +436,12 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALLOC_VGROUP_OVER; + goto _OVER; } pArray = mndBuildDnodesArray(pMnode); if (pArray == NULL) { - goto ALLOC_VGROUP_OVER; + goto _OVER; } mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), @@ -410,7 +474,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { if (mndGetAvailableDnode(pMnode, pVgroup, pArray) != 0) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - goto ALLOC_VGROUP_OVER; + goto _OVER; } allocedVgroups++; @@ -421,7 +485,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); -ALLOC_VGROUP_OVER: +_OVER: if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; @@ -492,7 +556,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -533,14 +597,13 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* // default 3 replica for (int32_t i = 0; i < 3; ++i) { - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); char buf1[20] = {0}; const char *role = syncStr(pVgroup->vnodeGid[i].role); - STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf1, false); @@ -597,13 +660,12 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; -// int32_t dnodeId = pShow->replica; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 0473fa375e..c8f832160b 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -24,6 +24,7 @@ class MndTestFunc : public ::testing::Test { void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); + void SetBufSize(SCreateFuncReq* pReq, int32_t size); }; Testbase MndTestFunc::test; @@ -40,6 +41,10 @@ void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { strcpy(pReq->pComment, pComment); } +void MndTestFunc::SetBufSize(SCreateFuncReq* pReq, int32_t size) { + pReq->bufSize = size; +} + TEST_F(MndTestFunc, 01_Show_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -96,6 +101,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { strcpy(createReq.name, "f1"); SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); + SetBufSize(&createReq, -1); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 85d0f76268..58e00ee34a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -30,6 +30,7 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" + "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbSma.c" diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ffa6f81c0d..d84c1b1d07 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -40,7 +40,6 @@ typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5538a63b4b..3724a98a5e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -100,6 +100,7 @@ int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); +int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, int32_t* pAffectedRows); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, @@ -125,7 +126,6 @@ int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); -int32_t tsdbProcessSubmitReq(STsdb* pTsdb, int64_t version, void* pReq); typedef struct { int8_t streamType; // sma or other diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 754023c49d..b255622c60 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -69,10 +69,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static void tsdbResetCommitFile(SCommitH *pCommith); static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -// static int tsdbCommitMeta(STsdbRepo *pRepo); -// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); -// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); -// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); static int tsdbCommitToTable(SCommitH *pCommith, int tid); static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); static int tsdbComparKeyBlock(const void *arg1, const void *arg2); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ebe5ce9a8f..6520f371f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -85,6 +85,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey STSchema *pSchema = NULL; TSKEY rowKey = 0; TSKEY fKey = 0; + TSKEY lastKey = TSKEY_INITIAL_VAL; bool isRowDel = false; int filterIter = 0; STSRow *row = NULL; @@ -111,7 +112,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } else { fKey = tdGetKey(filterKeys[filterIter]); } - + // 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols + // 2. rowKey - would dup since Multi-Version supported while (true) { if (fKey == INT64_MAX && rowKey == INT64_MAX) break; @@ -125,12 +127,14 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } else { fKey = tdGetKey(filterKeys[filterIter]); } +#if 0 } else if (fKey > rowKey) { if (isRowDel) { pMergeInfo->rowsDeleteFailed++; } else { if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsInserted++; pMergeInfo->nOperations++; pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); @@ -178,6 +182,45 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey isRowDel = TD_ROW_IS_DELETED(row); } + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } +#endif + } else { // fkey >= rowKey + if (isRowDel) { + ASSERT(!keepDup); + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsDeleteSucceed++; + pMergeInfo->nOperations++; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } else { + if (keepDup) { + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } else { + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); + } + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || TD_ROW_KEY(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = TD_ROW_KEY(row); + isRowDel = TD_ROW_IS_DELETED(row); + } + filterIter++; if (filterIter >= nFilterKeys) { fKey = INT64_MAX; @@ -191,9 +234,6 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows) { - // STsdbMeta *pMeta = pRepo->tsdbMeta; - // int32_t points = 0; - // STable *pTable = NULL; SSubmitBlkIter blkIter = {0}; STsdbMemTable *pMemTable = pTsdb->mem; void *tptr; @@ -221,8 +261,9 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo } // copy data to buffer pool - pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pMsgIter->dataLen + sizeof(*pBlock)); - memcpy(pBlkCopy, pBlock, pMsgIter->dataLen + sizeof(*pBlock)); + int32_t tlen = pMsgIter->dataLen + pMsgIter->schemaLen + sizeof(*pBlock); + pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, tlen); + memcpy(pBlkCopy, pBlock, tlen); tInitSubmitBlkIter(pMsgIter, pBlkCopy, &blkIter); if (blkIter.row == NULL) return 0; @@ -241,7 +282,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - (*pAffectedRows) += pMsgIter->numOfRows; + (*pAffectedRows) = pMsgIter->numOfRows; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c new file mode 100644 index 0000000000..ae0d4630d1 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListCfg SMemSkipListCfg; + +struct SMemTable { + STsdb *pTsdb; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + int32_t nHash; + int32_t nBucket; + SMemData **pBuckets; +}; + +struct SMemSkipListCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; +}; + +struct SMemSkipList { + int8_t level; + uint32_t seed; +}; + +struct SMemData { + SMemData *pHashNext; + tb_uid_t suid; + tb_uid_t uid; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + SMemSkipList sl; +}; + +// SMemTable +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { + SMemTable *pMemTb = NULL; + + pMemTb = taosMemoryCalloc(1, sizeof(*pMemTb)); + if (pMemTb == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemTb->pTsdb = pTsdb; + pMemTb->minKey = TSKEY_MAX; + pMemTb->maxKey = TSKEY_MIN; + pMemTb->minVer = -1; + pMemTb->maxVer = -1; + pMemTb->nRows = 0; + pMemTb->nHash = 0; + pMemTb->nBucket = 1024; + pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); + if (pMemTb->pBuckets == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + *ppMemTb = pMemTb; + return 0; +} + +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { + // TODO + return 0; +} + +// SMemData + +// SMemSkipList \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4f27e4616a..54b3eea988 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -884,9 +884,11 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); +#if 0 // TODO: skiplist refactor if (TD_ROW_VER(rmem) > maxVer) { rmem = NULL; } +#endif } } @@ -894,9 +896,11 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); +#if 0 // TODO: skiplist refactor if (TD_ROW_VER(rimem) > maxVer) { rimem = NULL; } +#endif } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 8f68883fde..49d02adc3a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -294,6 +294,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { return 0; } +// TODO: filter by Multi-Version int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds, bool mergeBitmap) { ASSERT(pBlock->numOfSubBlocks > 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 83adb545ba..105bc330d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1965,6 +1965,21 @@ int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { return TSDB_CODE_SUCCESS; } +static int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if (!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 878dd5fca4..8460400b59 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -339,7 +339,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs); + rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); if (rsp.pArray == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -450,7 +450,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in SCoder coder = {0}; int ret; - pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; pRsp->pCont = NULL; pRsp->contLen = 0; pRsp->code = TSDB_CODE_SUCCESS; @@ -473,9 +473,13 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in /* code */ ret = metaDropTable(pVnode->pMeta, version, pDropTbReq); if (ret < 0) { - dropTbRsp.code = TSDB_CODE_SUCCESS; + if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } else { + dropTbRsp.code = terrno; + } } else { - dropTbRsp.code = terrno; + dropTbRsp.code = TSDB_CODE_SUCCESS; } taosArrayPush(rsp.pArray, &dropTbRsp); @@ -488,20 +492,55 @@ _exit: } static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp rsp = {0}; + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock; + SSubmitRsp rsp = {0}; + SVCreateTbReq createTbReq = {0}; + SCoder coder = {0}; + int32_t nRows; pRsp->code = 0; // handle the request - if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { - pRsp->code = terrno; - return -1; + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + goto _exit; } - // pRsp->msgType = TDMT_VND_SUBMIT_RSP; - // vnodeProcessSubmitReq(pVnode, ptr, pRsp); + for (;;) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + // create table for auto create table mode + if (msgIter.schemaLen > 0) { + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBlock->data, msgIter.schemaLen, TD_DECODER); + if (tDecodeSVCreateTbReq(&coder, &createTbReq) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + tCoderClear(&coder); + goto _exit; + } + + if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) { + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + pRsp->code = terrno; + tCoderClear(&coder); + goto _exit; + } + } + + tCoderClear(&coder); + } + + if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &nRows) < 0) { + pRsp->code = terrno; + goto _exit; + } + + rsp.numOfRows += nRows; + } + +_exit: // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); @@ -511,18 +550,3 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in return 0; } - -int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if (!pReq) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 918892b786..3e8528e3d9 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -54,6 +54,7 @@ enum { CTG_ACT_REMOVE_DB, CTG_ACT_REMOVE_STB, CTG_ACT_REMOVE_TBL, + CTG_ACT_UPDATE_USER, CTG_ACT_MAX }; @@ -95,8 +96,18 @@ typedef struct SCtgRentMgmt { SCtgRentSlot *slots; } SCtgRentMgmt; +typedef struct SCtgUserAuth { + int32_t version; + SRWLatch lock; + bool superUser; + SHashObj *createdDbs; + SHashObj *readDbs; + SHashObj *writeDbs; +} SCtgUserAuth; + typedef struct SCatalog { uint64_t clusterId; + SHashObj *userCache; //key:user, value:SCtgUserAuth SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -124,6 +135,8 @@ typedef struct SCtgCacheStat { uint64_t vgMissNum; uint64_t tblHitNum; uint64_t tblMissNum; + uint64_t userHitNum; + uint64_t userMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -169,6 +182,11 @@ typedef struct SCtgRemoveTblMsg { uint64_t dbId; } SCtgRemoveTblMsg; +typedef struct SCtgUpdateUserMsg { + SCatalog* pCtg; + SGetUserAuthRsp userAuth; +} SCtgUpdateUserMsg; + typedef struct SCtgMetaAction { int32_t act; @@ -234,6 +252,8 @@ typedef struct SCtgAction { #define CTG_FLAG_SYS_DB 0x8 #define CTG_FLAG_FORCE_UPDATE 0x10 +#define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v)) + #define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) #define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) #define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6f1f34a57b..f485f85809 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,6 +24,7 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action); int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +int32_t ctgActUpdateUser(SCtgMetaAction *action); extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; @@ -51,6 +52,11 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl + }, + { + CTG_ACT_UPDATE_USER, + "update user", + ctgActUpdateUser } }; @@ -357,6 +363,31 @@ _return: CTG_RET(code); } +int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->userAuth = *pAuth; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + tFreeSGetUserAuthRsp(pAuth); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); @@ -687,6 +718,43 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp return TSDB_CODE_SUCCESS; } +int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get user auth from mnode, user:%s", user); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen); + if (code) { + ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_USER_AUTH, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user); + CTG_ERR_RET(code); + } + + ctgDebug("Got user auth from mnode, user:%s", user); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -859,6 +927,55 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char return TSDB_CODE_SUCCESS; } +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { + if (NULL == pCtg->userCache) { + ctgDebug("empty user auth cache, user:%s", user); + goto _return; + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + if (NULL == pUser) { + ctgDebug("user not in cache, user:%s", user); + goto _return; + } + + *inCache = true; + + ctgDebug("Got user from cache, user:%s", user); + CTG_CACHE_STAT_ADD(userHitNum, 1); + + if (pUser->superUser) { + *pass = true; + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pUser->lock); + if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + CTG_UNLOCK(CTG_READ, &pUser->lock); + return TSDB_CODE_SUCCESS; + } + + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + + CTG_UNLOCK(CTG_READ, &pUser->lock); + + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(userMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; @@ -1952,6 +2069,45 @@ _return: CTG_RET(code); } +int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + bool inCache = false; + int32_t code = 0; + + *pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + + if (inCache) { + return TSDB_CODE_SUCCESS; + } + + SGetUserAuthRsp authRsp = {0}; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); + + if (authRsp.superAuth) { + *pass = true; + goto _return; + } + + if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + goto _return; + } + + if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + +_return: + + ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false); + + return TSDB_CODE_SUCCESS; +} int32_t ctgActUpdateVg(SCtgMetaAction *action) { @@ -2121,6 +2277,67 @@ _return: CTG_RET(code); } +int32_t ctgActUpdateUser(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateUserMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + if (NULL == pCtg->userCache) { + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pCtg->userCache) { + ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); + if (NULL == pUser) { + SCtgUserAuth userAuth = {0}; + + userAuth.version = msg->userAuth.version; + userAuth.superUser = msg->userAuth.superAuth; + userAuth.createdDbs = msg->userAuth.createdDbs; + userAuth.readDbs = msg->userAuth.readDbs; + userAuth.writeDbs = msg->userAuth.writeDbs; + + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + pUser->version = msg->userAuth.version; + + CTG_LOCK(CTG_WRITE, &pUser->lock); + + taosHashCleanup(pUser->createdDbs); + pUser->createdDbs = msg->userAuth.createdDbs; + msg->userAuth.createdDbs = NULL; + + taosHashCleanup(pUser->readDbs); + pUser->readDbs = msg->userAuth.readDbs; + msg->userAuth.readDbs = NULL; + + taosHashCleanup(pUser->writeDbs); + pUser->writeDbs = msg->userAuth.writeDbs; + msg->userAuth.writeDbs = NULL; + + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + +_return: + + + taosHashCleanup(msg->userAuth.createdDbs); + taosHashCleanup(msg->userAuth.readDbs); + taosHashCleanup(msg->userAuth.writeDbs); + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); @@ -2836,6 +3053,35 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == users || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = taosHashGetSize(pCtg->userCache); + if (*num > 0) { + *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); + if (NULL == *users) { + ctgError("calloc %d userAuthVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + } + + uint32_t i = 0; + SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); + while (pAuth != NULL) { + void *key = taosHashGetKey(pAuth, NULL); + strncpy((*users)[i].user, key, sizeof((*users)[i].user)); + (*users)[i].version = pAuth->version; + pAuth = taosHashIterate(pCtg->userCache, pAuth); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { CTG_API_ENTER(); @@ -2880,6 +3126,31 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); + +_return: + + CTG_API_LEAVE(code); +} + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pAuth) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bc6139c304..3b5d0c209f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -87,9 +87,7 @@ typedef struct SResultInfo { // TODO refactor typedef struct STableQueryInfo { TSKEY lastKey; // last check ts, todo remove it later SResultRowPosition pos; // current active time window -// int32_t groupIndex; // group id in table list // SVariant tag; -// SResultRowInfo resInfo; // result info } STableQueryInfo; typedef enum { @@ -363,11 +361,12 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo *pCols; - SSDataBlock *pRes; - int32_t totalTables; - int32_t curPos; - void *pReader; + SColumnInfo *pCols; + SSDataBlock *pRes; + SArray *pColMatchInfo; + int32_t curPos; + SReadHandle readHandle; + STableGroupInfo *pTableGroups; } STagScanInfo; typedef struct SStreamBlockScanInfo { @@ -579,9 +578,8 @@ typedef struct SSortOperatorInfo { uint32_t sortBufSize; // max buffer size for in-memory sort SArray* pSortInfo; SSortHandle* pSortHandle; - SArray* inputSlotMap; // for index map from table scan output + SArray* pColMatchInfo; // for index map from table scan output int32_t bufPageSize; -// int32_t numOfRowsInRes; // TODO extact struct int64_t startTs; // sort start time @@ -646,7 +644,7 @@ void cleanupAggSup(SAggSupporter* pAggSup); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity); +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo); SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); @@ -704,7 +702,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); @@ -717,7 +715,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); -void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win); diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 2072707b30..d74628a72f 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -117,18 +117,25 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex); +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex); +void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); + +/** + * + * @param pSortHandle + * @return + */ +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a237eb0e7d..943d4b2783 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3520,7 +3520,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSortedMergeOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL); } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; @@ -4701,7 +4701,7 @@ static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols); -static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); +static SArray* createSortInfo(SNodeList* pNodeList); static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); @@ -4739,7 +4739,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { return NULL; @@ -4783,6 +4782,25 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { + STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); + + int32_t code = + doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); + + int32_t numOfOutputCols = 0; + SArray* colList = + extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); + return pOperator; } else { ASSERT(0); } @@ -4852,16 +4870,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SArray* info = createSortInfo(pSortPhyNode->pSortKeys, pSortPhyNode->pTargets); - SArray* slotMap = createIndexMap(pSortPhyNode->pTargets); + SArray* info = createSortInfo(pSortPhyNode->pSortKeys); int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; - if (pSortPhyNode->pExprs != NULL) { - pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); - } + SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); - pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, slotMap, pTaskInfo); + int32_t numOfOutputCols = 0; + SArray* pColList = + extractColMatchInfo(pSortPhyNode->pTargets, pSortPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -5019,7 +5037,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { return pList; } -SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { +SArray* createSortInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { @@ -5034,22 +5052,7 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST); SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr; - - bool found = false; - for (int32_t j = 0; j < LIST_LENGTH(pNodeListTarget); ++j) { - STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j); - - SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr; - if (pColNode->slotId == pColNodeT->slotId) { // to find slotId in PhysiSort OutputDataBlockDesc - bi.slotId = pTarget->slotId; - found = true; - break; - } - } - - if (!found) { - qError("sort slot id does not found"); - } + bi.slotId = pColNode->slotId; taosArrayPush(pList, &bi); } @@ -5088,7 +5091,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchInfo c = {0}; c.output = true; - c.colId = pColNode->colId; + c.colId = pColNode->colId; c.targetSlotId = pNode->slotId; taosArrayPush(pList, &c); } @@ -5166,9 +5169,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* if (code != TSDB_CODE_SUCCESS) { goto _error; } -#if 0 - return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId); -#endif + return tsdbQueryTables(pHandle->vnode, &cond, pTableGroupInfo, queryId, taskId); _error: diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8c9fdfe4e6..b728daa3bb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ -#include "ttime.h" +#include #include "filter.h" #include "function.h" #include "functionMgt.h" #include "os.h" #include "querynodes.h" +#include "systable.h" #include "tglobal.h" #include "tname.h" -#include "systable.h" +#include "ttime.h" #include "tdatablock.h" #include "tmsg.h" @@ -1159,16 +1160,17 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe } static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { -#if 0 if (pOperator->status == OP_EXEC_DONE) { return NULL; } + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + +#if 0 int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; STagScanInfo *pInfo = pOperator->info; SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; int32_t count = 0; SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); @@ -1237,55 +1239,54 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo +#endif - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; + STagScanInfo* pInfo = pOperator->info; + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + SSDataBlock* pRes = pInfo->pRes; - STableQueryInfo* item = taosArrayGetP(pa, i); + SArray* pa = taosArrayGetP(pInfo->pTableGroups->pGroupList, 0); - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfExprs; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { - continue; - } + char str[512] = {0}; + int32_t count = 0; + SMetaReader mr = {0}; - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resSchema.type; - bytes = pExprInfo[j].base.resSchema.bytes; + while (pInfo->curPos < pInfo->pTableGroups->numOfTables && count < pOperator->resultInfo.capacity) { + STableKeyInfo* item = taosArrayGet(pa, pInfo->curPos); - if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); - } + for (int32_t j = 0; j < pOperator->numOfExprs; ++j) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); - dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; - doSetTagValueToResultBuf(dst, data, type, bytes); + // refactor later + if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&mr, item->uid); + + STR_TO_VARSTR(str, mr.me.name); + metaReaderClear(&mr); + + colDataAppend(pDst, count, str, false); + + // data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); + // dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + // doSetTagValueToResultBuf(dst, data, type, bytes); } count += 1; } - if (pInfo->curPos >= pInfo->totalTables) { + if (++pInfo->curPos >= pInfo->pTableGroups->numOfTables) { pOperator->status = OP_EXEC_DONE; } - - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } + // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); + setTaskStatus(pTaskInfo, TASK_COMPLETED); } pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; - -#endif - return TSDB_CODE_SUCCESS; + return (pRes->info.rows == 0) ? NULL : pInfo->pRes; } static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { @@ -1293,14 +1294,18 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, + SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pReader = readHandle; + pInfo->pTableGroups = pTableGroupInfo; + pInfo->pColMatchInfo = pColMatchInfo; + pInfo->pRes = pResBlock; + pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -1308,9 +1313,12 @@ SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExpr; - pOperator->numOfExprs = numOfOutput; + pOperator->numOfExprs = numOfOutput; pOperator->pTaskInfo = pTaskInfo; + initResultSizeInfo(pOperator, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0f973b0cf0..619651f11f 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -5,7 +5,7 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, - SArray* pIndexMap, SExecTaskInfo* pTaskInfo) { + SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); int32_t rowSize = pResBlock->info.rowSize; @@ -20,17 +20,19 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header - pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer pInfo->pSortInfo = pSortInfo; - pInfo->inputSlotMap = pIndexMap; + pInfo->pColMatchInfo= pColMatchColInfo; pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; + // lazy evaluation for the following parameter since the input datablock is not known till now. +// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header +// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, NULL); @@ -45,14 +47,12 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR return NULL; } -// TODO merge aggregate super table void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { - colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + colDataAppendNULL(pColInfo, pBlock->info.rows); } else { char* pData = tsortGetValue(pTupleHandle, i); colDataAppend(pColInfo, pBlock->info.rows, pData, false); @@ -62,11 +62,12 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { pBlock->info.rows += 1; } -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) { +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) { blockDataCleanup(pDataBlock); - blockDataEnsureCapacity(pDataBlock, capacity); + ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); - blockDataEnsureCapacity(pDataBlock, capacity); + SSDataBlock* p = tsortGetSortedDataBlock(pHandle); + blockDataEnsureCapacity(p, capacity); while (1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); @@ -74,12 +75,32 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i break; } - appendOneRowToDataBlock(pDataBlock, pTupleHandle); - if (pDataBlock->info.rows >= capacity) { + appendOneRowToDataBlock(p, pTupleHandle); + if (p->info.rows >= capacity) { return pDataBlock; } } + if (p->info.rows > 0) { + int32_t numOfCols = taosArrayGetSize(pColMatchInfo); + for(int32_t i = 0; i < numOfCols; ++i) { + SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); + + for(int32_t j = 0; j < p->info.numOfCols; ++j) { + SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, j); + if (pSrc->info.colId == pmInfo->colId) { + SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); + colDataAssign(pDst, pSrc, p->info.rows); + break; + } + } + } + + pDataBlock->info.rows = p->info.rows; + pDataBlock->info.capacity = p->info.rows; + } + + blockDataDestroy(p); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } @@ -106,16 +127,16 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { SSortOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); } - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT, - pInfo->bufPageSize, numOfBufPage, pInfo->binfo.pRes, pTaskInfo->id.str); +// pInfo->binfo.pRes is not equalled to the input datablock. +// int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, + -1, -1, NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[0]; tsortAddSource(pInfo->pSortHandle, ps); @@ -127,7 +148,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { } pOperator->status = OP_RES_TO_RETURN; - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); } void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { @@ -135,5 +156,5 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); taosArrayDestroy(pInfo->pSortInfo); - taosArrayDestroy(pInfo->inputSlotMap); + taosArrayDestroy(pInfo->pColMatchInfo); } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 50aa4cfc01..040ee8c7f5 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -64,25 +64,8 @@ struct SSortHandle { static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); -static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - - colInfo.info.type = pSchema[i].type; - colInfo.info.bytes = pSchema[i].bytes; - colInfo.info.colId = pSchema[i].colId; - taosArrayPush(pBlock->pDataBlock, &colInfo); - - if (IS_VAR_DATA_TYPE(colInfo.info.type)) { - pBlock->info.hasVarCol = true; - } - } - - return pBlock; +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) { + return createOneDataBlock(pSortHandle->pDataBlock, false); } /** @@ -98,7 +81,10 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, SArray* pIndexMap, int32_t pSortHandle->numOfPages = numOfPages; pSortHandle->pSortInfo = pSortInfo; pSortHandle->pIndexMap = pIndexMap; - pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + + if (pBlock != NULL) { + pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + } pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); pSortHandle->cmpParam.orderInfo = pSortInfo; @@ -530,6 +516,17 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { if (pHandle->pDataBlock == NULL) { pHandle->pDataBlock = createOneDataBlock(pBlock, false); + + // calculate the buffer pages according to the total available buffers. + int32_t rowSize = blockDataGetRowSize(pBlock); + if (rowSize * 4 > 4096) { + pHandle->pageSize = rowSize * 4; + } else { + pHandle->pageSize = 4096; + } + // todo!! + pHandle->numOfPages = 1024; + sortBufSize = pHandle->numOfPages * pHandle->pageSize; } // perform the scalar function calculation before apply the sort @@ -538,7 +535,6 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { } // todo relocate the columns - int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock, pHandle->pIndexMap); if (code != 0) { return code; @@ -689,7 +685,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex); - return colDataIsNull(pColInfoSrc, 0, pVHandle->rowIndex, NULL); + return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex); } void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 38922833f9..eac11558cb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -917,7 +917,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = qTbnameFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5b1573c88f..0a99ef61ce 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -594,7 +594,9 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS //TODO: free the array output->pDataBlock output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - taosArrayPush(output->pDataBlock, input->columnData); + for (int32_t i = 0; i < numOfCols; ++i) { + taosArrayPush(output->pDataBlock, (input + i)->columnData); + } return 0; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ba9fca2969..f5e4a9c6e6 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -81,6 +81,9 @@ typedef struct SUdf { TUdfAggStartFunc aggStartFunc; TUdfAggProcessFunc aggProcFunc; TUdfAggFinishFunc aggFinishFunc; + + TUdfInitFunc initFunc; + TUdfDestroyFunc destroyFunc; } SUdf; // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix @@ -101,7 +104,19 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); return UDFC_CODE_LOAD_UDF_FAILURE; } - //TODO: init and destroy function + + char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *initSuffix = "_init"; + strcpy(initFuncName, udfName); + strncat(initFuncName, initSuffix, strlen(initSuffix)); + uv_dlsym(&udf->lib, initFuncName, (void**)(&udf->initFunc)); + + char destroyFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *destroySuffix = "_destroy"; + strcpy(destroyFuncName, udfName); + strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + uv_dlsym(&udf->lib, destroyFuncName, (void**)(&udf->destroyFunc)); + if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); @@ -159,6 +174,9 @@ void udfdProcessRequest(uv_work_t *req) { if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; udfdLoadUdf(setup->udfName, udf); + if (udf->initFunc) { + udf->initFunc(); + } udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -170,7 +188,6 @@ void udfdProcessRequest(uv_work_t *req) { } SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); handle->udf = udf; - // TODO: allocate private structure and call init function and set it to handle SUdfResponse rsp; rsp.seqNum = request.seqNum; rsp.type = request.type; @@ -275,10 +292,12 @@ void udfdProcessRequest(uv_work_t *req) { if (unloadUdf) { uv_cond_destroy(&udf->condReady); uv_mutex_destroy(&udf->lock); + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } uv_dlclose(&udf->lib); taosMemoryFree(udf); } - // TODO: call destroy and free udf private taosMemoryFree(handle); SUdfResponse response; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 152aa6a01e..e82873b923 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -22,7 +22,6 @@ #include "ttime.h" #include "ttypes.h" -// clang-format off #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -248,12 +247,11 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool } else { CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); + SVgroupInfo vg; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); } - SVgroupInfo vg; - CHECK_CODE( - catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - return TSDB_CODE_SUCCESS; } @@ -828,12 +826,21 @@ static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } -static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, STableMeta* pMeta) { +static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* pTableName, const char* pName, + int32_t len, STableMeta* pMeta) { + SVgroupInfo vg; + SParseContext* pBasicCtx = pCxt->pComCxt; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pTableName, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); + + pMeta->uid = tGenIdPI64(); + pMeta->vgId = vg.vgId; + STableMeta* pBackup = NULL; if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pBackup->uid = tGenIdPI64(); return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); } @@ -856,7 +863,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } - CHECK_CODE(storeTableMeta(pCxt->pSubTableHashObj, tbFName, len, pCxt->pTableMeta)); + CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, &name, tbFName, len, pCxt->pTableMeta)); SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); @@ -1262,9 +1269,10 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -1312,11 +1320,10 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *p return TSDB_CODE_SUCCESS; } - -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1391,10 +1398,11 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in return TSDB_CODE_SUCCESS; } -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1459,7 +1467,7 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBu } #ifdef TD_DEBUG_PRINT_ROW - if(rowEnd) { + if (rowEnd) { STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae05d2293e..4a96301d95 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -498,14 +498,9 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); } - int32_t len = pBlocks->numOfRows * - (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); - // erase the empty space reserved for binary data int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, isRawPayload); - assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 286e7d3d44..822c214fe5 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -181,6 +181,25 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SGetUserAuthReq req = {0}; + strncpy(req.user, input, sizeof(req.user)); + + int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSGetUserAuthReq(pBuf, bufLen, &req); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + + int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; SUseDbRsp usedbRsp = {0}; @@ -419,6 +438,20 @@ int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessGetUserAuthRsp(void *output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSGetUserAuthRsp(msg, msgSize, (SGetUserAuthRsp *)output) != 0) { + qError("tDeserializeSGetUserAuthRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + return TSDB_CODE_SUCCESS; +} + + void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -427,6 +460,8 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryBuildGetUserAuthMsg; + queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -435,6 +470,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryProcessGetUserAuthRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 51ac48adaf..77caf67a2b 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1513,3 +1513,9 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t*) colDataGetData(pInput->columnData, 4)); return TSDB_CODE_SUCCESS; } + +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + ASSERT(inputNum == 1); + colDataAppend(pOutput->columnData, pOutput->numOfRows, colDataGetData(pInput->columnData, 0), false); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 84aa5559d0..1e37533f2c 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1023,8 +1023,7 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, 0); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0); } } } @@ -1050,8 +1049,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index f776fb3764..846cf6f967 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -46,9 +46,21 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + ip = taosGetIpv4FromFqdn(pInit->localFqdn); + if (ip == 0xFFFFFFFF) { + tError("invalid fqdn: %s", pInit->localFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; + taosMemoryFree(pRpc); + return NULL; + } + } + pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; - pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); if (pRpc->tcphandle == NULL) { taosMemoryFree(pRpc); return NULL; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 7378ca3241..ad3f520210 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -817,7 +817,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); - uv_os_sock_t fds[2]; if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; @@ -841,6 +840,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + tError("failed to bind, reason: %s", terrstr()); + goto End; + } if (false == addHandleToAcceptloop(srv)) { goto End; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 6d90773d1b..19e4defafc 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -44,7 +44,7 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) { return -1; } - printf("parse relative path:%s to abs path:%s\n", words, pwordexp->wordPos); + // printf("parse relative path:%s to abs path:%s\n", words, pwordexp->wordPos); return 0; } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 6aa8520082..8cac660039 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -638,6 +638,48 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { return 0; } +bool taosValidIpAndPort(uint32_t ip, uint16_t port) { + struct sockaddr_in serverAdd; + SocketFd fd; + int32_t reuse; + + // printf("open tcp server socket:0x%x:%hu", ip, port); + + bzero((char *)&serverAdd, sizeof(serverAdd)); + serverAdd.sin_family = AF_INET; + serverAdd.sin_addr.s_addr = ip; + serverAdd.sin_port = (uint16_t)htons(port); + + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return false; + } + + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return false; + } + pSocket->refId = 0; + pSocket->fd = fd; + + /* set REUSEADDR option, so the portnumber can be re-used */ + reuse = 1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; + } + /* bind socket to server address */ + if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return false; + } + taosCloseSocket(&pSocket); + return true; +} TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; SocketFd fd; diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000000..ce459414c4 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,5 @@ +taospy +numpy +fabric2 +psutil +pandas diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim index 80d2dafa84..97d49ae901 100644 --- a/tests/script/tsim/query/scalarFunction.sim +++ b/tests/script/tsim/query/scalarFunction.sim @@ -105,17 +105,22 @@ print ====> $data90 $data91 $data92 $data93 $data94 $data95 if $rows != $rowNum then return -1 endi + print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb +print ====> rows = $rows and totalRows = $totalRows for stb if $rows != $totalRows then return -1 endi + print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb +print ====> rows = $rows and rowNum = $rowNum for ntb if $rows != $rowNum then return -1 endi + print ====> log sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 8acd07cfe4..c76569b40f 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -43,6 +43,27 @@ if $data00 != 2.236067977 then return -1 endi +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); +sql select udf1(f1, f2) from t2; +if $rows != 2 then + return -1 +endi +if $data00 != 88 then + return -1 +endi +if $data10 != 88 then + return -1 +endi + +sql select udf2(f1, f2) from t2; +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi + #sql drop function udf1; #sql drop function udf2; system sh/exec.sh -n dnode1 -s stop -x SIGKILL diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py new file mode 100644 index 0000000000..a7d17bc41e --- /dev/null +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -0,0 +1,291 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import threading +import multiprocessing as mp +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import datetime as dt +import time +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: offset return valid + ''' + return + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + # tdSql.prepare() + # self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + + # test base case + self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + + # test advance case + # self.test_case2() + # tdLog.debug(" LIMIT test_case2 ............ [OK]") + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # --------------- case ------------------- + + # create tables + def create_tables(self,dbname,stbname,count): + tdSql.execute("use %s" %dbname) + tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + pre_create = "create table" + sql = pre_create + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + for i in range(count): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%3000 == 0: + tdSql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + def newcur(self,host,cfg): + user = "root" + password = "taosdata" + port =6030 + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop): + host = "chenhaoran02" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + tsql.execute("use %s" %dbname) + tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + + pre_create = "create table" + sql = pre_create + tcountStop=int(tcountStop) + tcountStart=int(tcountStart) + count=tcountStop-tcountStart + + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + # print(type(tcountStop),type(tcountStart)) + for i in range(tcountStart,tcountStop): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%20000 == 0: + # print(sql) + tsql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + # print(sql) + tsql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + + + # insert data + def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount): + tdSql.execute("use %s" %dbname) + pre_insert = "insert into " + sql = pre_insert + tcount=tcountStop-tcountStart + allRows=tcount*rowCount + tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows)) + exeStartTime=time.time() + for i in range(tcountStart,tcountStop): + sql += " %s_%d values "%(stbname,i) + for j in range(rowCount): + sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) + if j >0 and j%5000 == 0: + # print(sql) + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbname,i) + # end sql + if sql != pre_insert: + # print(sql) + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedInsert=allRows/spendTime + # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + + # test case1 base + def test_case1(self): + tdLog.debug("-----create database and tables test------- ") + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + # tdSql.execute("create database db11 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + # tdSql.execute("create stable stb1(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)") + + # tdSql.execute("create database db12 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + + # t1 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(1,)) + # t2 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(2,)) + # t1 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", 0,count/2,)) + # t2 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", count/2,count,)) + + count=50000 + vgroups=1 + threads = [] + threadNumbers=2 + for i in range(threadNumbers): + threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,))) + start_time = time.time() + for tr in threads: + tr.start() + for tr in threads: + tr.join() + end_time = time.time() + spendTime=end_time-start_time + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + # self.new_create_tables("db1", "stb1", 15*10000) + # self.new_create_tables("db1", "stb1", 15*10000) + + # tdSql.execute("create database db4 vgroups 4") + # self.create_tables("db4", "stb4", 30*10000) + + # tdSql.execute("create database db6 vgroups 6") + # self.create_tables("db6", "stb6", 30*10000) + + # tdSql.execute("create database db8 vgroups 8") + # self.create_tables("db8", "stb8", 30*10000) + + # tdSql.execute("create database db12 vgroups 12") + # self.create_tables("db12", "stb12", 30*10000) + + # tdSql.execute("create database db16 vgroups 16") + # self.create_tables("db16", "stb16", 30*10000) + return + + # test case2 base:insert data + def test_case2(self): + + tdLog.debug("-----insert data test------- ") + # drop database + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + tdSql.execute("create database db1 vgroups 1") + self.create_tables("db1", "stb1", 1*100) + self.insert_data("db1", "stb1", self.ts, 1*50,1*10000) + + + tdSql.execute("create database db4 vgroups 4") + self.create_tables("db4", "stb4", 1*100) + self.insert_data("db4", "stb4", self.ts, 1*100,1*10000) + + tdSql.execute("create database db6 vgroups 6") + self.create_tables("db6", "stb6", 1*100) + self.insert_data("db6", "stb6", self.ts, 1*100,1*10000) + + tdSql.execute("create database db8 vgroups 8") + self.create_tables("db8", "stb8", 1*100) + self.insert_data("db8", "stb8", self.ts, 1*100,1*10000) + + tdSql.execute("create database db12 vgroups 12") + self.create_tables("db12", "stb12", 1*100) + self.insert_data("db12", "stb12", self.ts, 1*100,1*10000) + + tdSql.execute("create database db16 vgroups 16") + self.create_tables("db16", "stb16", 1*100) + self.insert_data("db16", "stb16", self.ts, 1*100,1*10000) + + return + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index b4878e42c9..a39bc21946 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -5,6 +5,7 @@ from util.sql import * from util.cases import * from util.dnodes import * +PRIMARY_COL = "ts" INT_COL = "c1" BINT_COL = "c2" @@ -18,9 +19,10 @@ BINARY_COL = "c8" NCHAR_COL = "c9" TS_COL = "c10" -UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] -TS_TYPE_COL = [TS_COL] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] class TDTestCase: @@ -28,88 +30,171 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - def __length_condition(self): - length_condition = [] + def __query_condition(self,tbname): + query_condition = [] for char_col in CHAR_COL: - length_condition.extend( + query_condition.extend( ( - char_col, - f"upper( {char_col} )", + f"{tbname}.{char_col}", + f"upper( {tbname}.{char_col} )", ) ) - length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) - length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) - length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL ) + for num_col in NUM_COL: + query_condition.extend( + ( + f"{tbname}.{num_col}", + f"sin( {tbname}.{num_col} )" + ) + ) + query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL ) - length_condition.append('''"test1234!@#$%^&*():'> 0 " - return "" + def __join_condition(self, tb_list, filter=PRIMARY_COL): + # sourcery skip: flip-comparison + if 1 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[0]}.{filter} " + elif 2 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + else: + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + for i in range(1, len(tb_list)-1 ): + join_filter += f"and {tb_list[i]}.{filter} = {tb_list[i+1]}.{filter}" - def __group_condition(self, col, having = ""): + return join_filter + + def __where_condition(self, col, tbname): + if col in NUM_COL: + return f" abs( {tbname}.{col} ) >= 0" + elif col in CHAR_COL: + return f" lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' " + elif col in BOOLEAN_COL: + return f" {tbname}.{col} in (false, true) " + elif col in TS_TYPE_COL or col in PRIMARY_COL: + return f" cast( {tbname}.{col} as binary(16) ) is not null " + else: + return "" + + def __group_condition(self, tbname, col, having = ""): return f" group by {col} having {having}" if having else f" group by {col} " - def __length_current_check(self, tbname): - length_condition = self.__length_condition() - for condition in length_condition: - where_condition = self.__where_condition(condition) - group_having = self.__group_condition(condition, having=f"{condition} is not null " ) - group_no_having= self.__group_condition(condition ) + def __join_check(self, tblist, checkrows, join_flag=True): + query_conditions = self.__query_condition(tblist[0]) + join_condition = self.__join_condition(tb_list=tblist) if join_flag else " " + for condition in query_conditions: + where_condition = self.__where_condition(col=condition, tbname=tblist[0]) + group_having = self.__group_condition(tbname=tblist[0], col=condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(tbname=tblist[0], col=condition ) groups = ["", group_having, group_no_having] - for group_condition in groups: - tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") - datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] - length_data = [ len(str(data)) if data else None for data in datas ] - tdSql.query(f"select length( {condition} ) from {tbname} {where_condition} {group_condition}") - for i in range(len(length_data)): - tdSql.checkData(i, 0, length_data[i] ) if length_data[i] else tdSql.checkData(i, 0, None) + if where_condition: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition} " + else: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} {group_condition} " - def __length_err_check(self,tbname): - sqls = [] + if not join_flag : + tdSql.error(sql=sql) + if len(tblist) == 2: + if "ct1" in tblist or "t1" in tblist: + self.__join_current(sql, checkrows) + elif where_condition or "not null" in group_condition: + self.__join_current(sql, checkrows + 2 ) + elif group_condition: + self.__join_current(sql, checkrows + 3 ) + else: + self.__join_current(sql, checkrows + 5 ) + if len(tblist) > 2 or len(tblist) < 1: + tdSql.error(sql=sql) - for un_char_col in UN_CHAR_COL: - sqls.extend( - ( - f"select length( {un_char_col} ) from {tbname} ", - f"select length(ceil( {un_char_col} )) from {tbname} ", - f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", - ) - ) + # def __join_err_check(self,tbname): + # sqls = [] - sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) - sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # for un_char_col in NUM_COL: + # sqls.extend( + # ( + # f"select length( {un_char_col} ) from {tbname} ", + # f"select length(ceil( {un_char_col} )) from {tbname} ", + # f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", + # ) + # ) - sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) - sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) - sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) - sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) - sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) - sqls.extend( - ( - f"select length() from {tbname} ", - f"select length(*) from {tbname} ", - f"select length(ccccccc) from {tbname} ", - f"select length(111) from {tbname} ", - f"select length(c8, 11) from {tbname} ", - ) - ) + # sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) + # sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + + # sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) + # sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) + # sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + # sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + # sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + # sqls.extend( + # ( + # f"select length() from {tbname} ", + # f"select length(*) from {tbname} ", + # f"select length(ccccccc) from {tbname} ", + # f"select length(111) from {tbname} ", + # f"select length(c8, 11) from {tbname} ", + # ) + # ) + + # return sqls + + def __join_current(self, sql, checkrows): + tdSql.query(sql=sql) + tdSql.checkRows(checkrows) - return sqls def __test_current(self): + # sourcery skip: extract-duplicate-method, inline-immediately-returned-variable tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") - tbname = ["ct1", "ct2", "ct4", "t1"] - for tb in tbname: - self.__length_current_check(tb) - tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + tblist_1 = ["ct1", "ct2"] + self.__join_check(tblist_1, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========") + tblist_2 = ["ct2", "ct4"] + self.__join_check(tblist_2, self.rows) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========") + tblist_3 = ["t1", "ct4"] + self.__join_check(tblist_3, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_3} over==========") + tblist_4 = ["t1", "ct1"] + self.__join_check(tblist_4, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_4} over==========") def __test_error(self): + # sourcery skip: extract-duplicate-method, move-assign-in-block tdLog.printNoPrefix("==========err sql condition check , must return error==========") + err_list_1 = ["ct1","ct2", "ct4"] + err_list_2 = ["ct1","ct2", "t1"] + err_list_3 = ["ct1","ct4", "t1"] + err_list_4 = ["ct2","ct4", "t1"] + err_list_5 = ["ct1", "ct2","ct4", "t1"] + self.__join_check(err_list_1, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_1} over==========") + self.__join_check(err_list_2, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_2} over==========") + self.__join_check(err_list_3, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_3} over==========") + self.__join_check(err_list_4, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========") + self.__join_check(err_list_5, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========") + self.__join_check(["ct2", "ct4"], -1, join_flag=False) + tdLog.printNoPrefix("==========err sql condition check in has no join condition over==========") + + tdSql.error( f"select c1, c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{INT_COL}=ct4.{INT_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{TS_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct1.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and c1 is not null " ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and ct1.c1 is not null " ) + + tbname = ["ct1", "ct2", "ct4", "t1"] for tb in tbname: @@ -150,33 +235,33 @@ class TDTestCase: now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( f'''insert into ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) tdSql.execute( f'''insert into ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} ) ( { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} ) ''' ) @@ -184,15 +269,15 @@ class TDTestCase: tdSql.execute( f'''insert into ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) @@ -200,7 +285,7 @@ class TDTestCase: for i in range(rows): insert_data = f'''insert into t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( @@ -210,12 +295,12 @@ class TDTestCase: ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) @@ -228,7 +313,8 @@ class TDTestCase: self.__create_tb() tdLog.printNoPrefix("==========step2:insert data") - self.__insert_data(10) + self.rows = 10 + self.__insert_data(self.rows) tdLog.printNoPrefix("==========step3:all check") self.all_test() diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index e6762b2d61..6e4c292183 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -191,13 +191,13 @@ class TDTestCase: def support_types(self): type_error_sql_lists = [ "select log(ts ,2 ) from t1" , - "select log(c7,2 ) from t1", - "select log(c8,2 ) from t1", - "select log(c9,2 ) from t1", - "select log(ts,2 ) from ct1" , - "select log(c7,2 ) from ct1", - "select log(c8,2 ) from ct1", - "select log(c9,2 ) from ct1", + "select log(c7,c2 ) from t1", + "select log(c8,c1 ) from t1", + "select log(c9,c2 ) from t1", + "select log(ts,c7 ) from ct1" , + "select log(c7,c9 ) from ct1", + "select log(c8,c2 ) from ct1", + "select log(c9,c1 ) from ct1", "select log(ts,2 ) from ct3" , "select log(c7,2 ) from ct3", "select log(c8,2 ) from ct3", diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py new file mode 100644 index 0000000000..8b0137b411 --- /dev/null +++ b/tests/system-test/2-query/pow.py @@ -0,0 +1,652 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_pow2(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.pow(elem,2) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow1(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else : + elem = pow(elem ,1) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow__10(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem == 0: + elem = None + else: + elem = pow(elem ,-10) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select pow from t1", + # "select pow(-+--+c1 ,2) from t1", + # "select +-pow(c1,2) from t1", + # "select ++-pow(c1,2) from t1", + # "select ++--pow(c1,2) from t1", + # "select - -pow(c1,2)*0 from t1", + # "select pow(tbname+1,2) from t1 ", + "select pow(123--123,2)==1 from t1", + "select pow(c1,2) as 'd1' from t1", + "select pow(c1 ,c2 ,2) from t1", + "select pow(c1 ,NULL ,2) from t1", + "select pow(, 2) from t1;", + "select pow(pow(c1, 2) ab from t1)", + "select pow(c1 ,2 ) as int from t1", + "select pow from stb1", + # "select pow(-+--+c1) from stb1", + # "select +-pow(c1) from stb1", + # "select ++-pow(c1) from stb1", + # "select ++--pow(c1) from stb1", + # "select - -pow(c1)*0 from stb1", + # "select pow(tbname+1) from stb1 ", + "select pow(123--123 ,2)==1 from stb1", + "select pow(c1 ,2) as 'd1' from stb1", + "select pow(c1 ,c2 ,2 ) from stb1", + "select pow(c1 ,NULL,2) from stb1", + "select pow(,) from stb1;", + "select pow(pow(c1 , 2) ab from stb1)", + "select pow(c1 , 2) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select pow(ts ,2 ) from t1" , + "select pow(c7,c1 ) from t1", + "select pow(c8,c2) from t1", + "select pow(c9,c3 ) from t1", + "select pow(ts,c4 ) from ct1" , + "select pow(c7,c5 ) from ct1", + "select pow(c8,c6 ) from ct1", + "select pow(c9,c8 ) from ct1", + "select pow(ts,2 ) from ct3" , + "select pow(c7,2 ) from ct3", + "select pow(c8,2 ) from ct3", + "select pow(c9,2 ) from ct3", + "select pow(ts,2 ) from ct4" , + "select pow(c7,2 ) from ct4", + "select pow(c8,2 ) from ct4", + "select pow(c9,2 ) from ct4", + "select pow(ts,2 ) from stb1" , + "select pow(c7,2 ) from stb1", + "select pow(c8,2 ) from stb1", + "select pow(c9,2 ) from stb1" , + + "select pow(ts,2 ) from stbbb1" , + "select pow(c7,2 ) from stbbb1", + + "select pow(ts,2 ) from tbname", + "select pow(c9,2 ) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select pow(c1,2 ) from t1", + "select pow(c2,2 ) from t1", + "select pow(c3,2 ) from t1", + "select pow(c4,2 ) from t1", + "select pow(c5,2 ) from t1", + "select pow(c6,2 ) from t1", + + "select pow(c1,2 ) from ct1", + "select pow(c2,2 ) from ct1", + "select pow(c3,2 ) from ct1", + "select pow(c4,2 ) from ct1", + "select pow(c5,2 ) from ct1", + "select pow(c6,2 ) from ct1", + + "select pow(c1,2 ) from ct3", + "select pow(c2,2 ) from ct3", + "select pow(c3,2 ) from ct3", + "select pow(c4,2 ) from ct3", + "select pow(c5,2 ) from ct3", + "select pow(c6,2 ) from ct3", + + "select pow(c1,2 ) from stb1", + "select pow(c2,2 ) from stb1", + "select pow(c3,2 ) from stb1", + "select pow(c4,2 ) from stb1", + "select pow(c5,2 ) from stb1", + "select pow(c6,2 ) from stb1", + + "select pow(c6,2) as alisb from stb1", + "select pow(c6,2) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_pow_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select pow(c1 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c2 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c3 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c4 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c5 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c6 ,2) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select pow(c1 ,2) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 9.000000000) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,2), pow(c2 ,2) ,pow(c3, 2), pow(c4 ,2), pow(c5 ,2) from t1") + self.check_result_auto_pow1( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,1), pow(c2 ,1) ,pow(c3, 1), pow(c4 ,1), pow(c5 ,1) from t1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,-10), pow(c2 ,-10) ,pow(c3, -10), pow(c4 ,-10), pow(c5 ,-10) from t1") + + # used for sub table + tdSql.query("select c1 ,pow(c1 ,2) from ct1") + tdSql.checkData(0, 1, 64.000000000) + tdSql.checkData(1 , 1, 49.000000000) + tdSql.checkData(3 , 1, 25.000000000) + tdSql.checkData(4 , 1, 0) + + # # test bug fix for pow(c1,c2) + + tdSql.query("select c1, c5 ,pow(c1,c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 104577724.506799981) + tdSql.checkData(2 , 2, 3684781.623933245) + tdSql.checkData(3 , 2, 152225.429759376) + tdSql.checkData(4 , 2, 7573.273783071) + + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,2), pow(c2,2) ,pow(c3,2), pow(c4,2), pow(c5,2) from ct1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,-10), pow(c2,-10) ,pow(c3,-10), pow(c4,-10), pow(c5,-10) from ct1") + + # nest query for pow functions + tdSql.query("select c1 , pow(c1,2) ,pow(pow(c1,2),2) , pow(pow(pow(c1,2),2),2) from ct1;") + tdSql.checkData(0 , 0 , 8) + tdSql.checkData(0 , 1 , 64.000000000) + tdSql.checkData(0 , 2 , 4096.000000000) + tdSql.checkData(0 , 3 , 16777216.000000000) + + tdSql.checkData(1 , 0 , 7) + tdSql.checkData(1 , 1 , 49.000000000) + tdSql.checkData(1 , 2 , 2401.000000000) + tdSql.checkData(1 , 3 , 5764801.000000000) + + tdSql.checkData(4 , 0 , 0) + tdSql.checkData(4 , 1 , 0.000000000) + tdSql.checkData(4 , 2 , 0.000000000) + tdSql.checkData(4 , 3 , 0.000000000) + + # # used for stable table + + tdSql.query("select pow(c1, 2) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select pow(c1, 2) from stbbb1") + tdSql.error("select pow(c1, 2) from tbname") + tdSql.error("select pow(c1, 2) from ct5") + + # mix with common col + tdSql.query("select c1, pow(c1 ,2) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,64.000000000) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c1, pow(c1,2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,25.000000000) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, pow(c1 ,2),pow(c1,2), log(pow(c1,2) ,2) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,36.000000000) + tdSql.checkData(3 , 2 ,36.000000000) + tdSql.checkData(3 , 3 ,5.169925001) + + tdSql.query("select c1, pow(c1,1),c5, floor(c5 ) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from stb1 ") + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from ct1 ") + tdSql.error("select pow(c1 ,2), count(c5) from stb1 ") + tdSql.error("select pow(c1 ,2), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, pow(c1 ,2) -0 ,pow(c1-4 ,2)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 16.000000000) + + tdSql.query(" select c1, pow(c1 ,2) -0 ,pow(c1-0.1 ,2)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 62.310000000) + + tdSql.query("select c1, pow(c1, -10), c2, pow(c2, -10), c3, pow(c3, -10) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, pow(c1, 100000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + + tdSql.query("select c1, pow(c1, 10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, pow(c1, 2) from ct1") + tdSql.checkData(0, 1,64.000000000) + tdSql.query("select c1, pow(c1, 2.0) from ct1") + tdSql.checkData(0, 1, 64.000000000) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start pow(x,y) + tdSql.query("select c1,c2, pow(c1,c2) from ct1") + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1,c2, pow(c2,c1) from ct1") + tdSql.checkData(0, 2, 3897131646727578700481513520437089271808.000000000) + tdSql.checkData(1, 2, 17217033054561120738612297152331776.000000000) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,64.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c10 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + def insert_data1(self, tbname, ts_start, count): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) + if i >0 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + # test case1 base + # def test_case1(self): + # # + # # limit base function + # # + # # base no where + # sql = "select * from t1 limit 10" + # tdSql.waitedQuery(sql, 10, WAITS) + + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py new file mode 100644 index 0000000000..28e869e044 --- /dev/null +++ b/tests/system-test/2-query/sqrt.py @@ -0,0 +1,551 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_sqrt(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem < 0: + elem = None + else: + elem = math.sqrt(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("sqrt function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("sqrt value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select sqrt from t1", + # "select sqrt(-+--+c1 ) from t1", + # "select +-sqrt(c1) from t1", + # "select ++-sqrt(c1) from t1", + # "select ++--sqrt(c1) from t1", + # "select - -sqrt(c1)*0 from t1", + # "select sqrt(tbname+1) from t1 ", + "select sqrt(123--123)==1 from t1", + "select sqrt(c1) as 'd1' from t1", + "select sqrt(c1 ,c2) from t1", + "select sqrt(c1 ,NULL ) from t1", + "select sqrt(,) from t1;", + "select sqrt(sqrt(c1) ab from t1)", + "select sqrt(c1 ) as int from t1", + "select sqrt from stb1", + # "select sqrt(-+--+c1) from stb1", + # "select +-sqrt(c1) from stb1", + # "select ++-sqrt(c1) from stb1", + # "select ++--sqrt(c1) from stb1", + # "select - -sqrt(c1)*0 from stb1", + # "select sqrt(tbname+1) from stb1 ", + "select sqrt(123--123)==1 from stb1", + "select sqrt(c1) as 'd1' from stb1", + "select sqrt(c1 ,c2 ) from stb1", + "select sqrt(c1 ,NULL) from stb1", + "select sqrt(,) from stb1;", + "select sqrt(sqrt(c1) ab from stb1)", + "select sqrt(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select sqrt(ts) from t1" , + "select sqrt(c7) from t1", + "select sqrt(c8) from t1", + "select sqrt(c9) from t1", + "select sqrt(ts) from ct1" , + "select sqrt(c7) from ct1", + "select sqrt(c8) from ct1", + "select sqrt(c9) from ct1", + "select sqrt(ts) from ct3" , + "select sqrt(c7) from ct3", + "select sqrt(c8) from ct3", + "select sqrt(c9) from ct3", + "select sqrt(ts) from ct4" , + "select sqrt(c7) from ct4", + "select sqrt(c8) from ct4", + "select sqrt(c9) from ct4", + "select sqrt(ts) from stb1" , + "select sqrt(c7) from stb1", + "select sqrt(c8) from stb1", + "select sqrt(c9) from stb1" , + + "select sqrt(ts) from stbbb1" , + "select sqrt(c7) from stbbb1", + + "select sqrt(ts) from tbname", + "select sqrt(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select sqrt(c1) from t1", + "select sqrt(c2) from t1", + "select sqrt(c3) from t1", + "select sqrt(c4) from t1", + "select sqrt(c5) from t1", + "select sqrt(c6) from t1", + + "select sqrt(c1) from ct1", + "select sqrt(c2) from ct1", + "select sqrt(c3) from ct1", + "select sqrt(c4) from ct1", + "select sqrt(c5) from ct1", + "select sqrt(c6) from ct1", + + "select sqrt(c1) from ct3", + "select sqrt(c2) from ct3", + "select sqrt(c3) from ct3", + "select sqrt(c4) from ct3", + "select sqrt(c5) from ct3", + "select sqrt(c6) from ct3", + + "select sqrt(c1) from stb1", + "select sqrt(c2) from stb1", + "select sqrt(c3) from stb1", + "select sqrt(c4) from stb1", + "select sqrt(c5) from stb1", + "select sqrt(c6) from stb1", + + "select sqrt(c6) as alisb from stb1", + "select sqrt(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_sqrt_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select sqrt(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select sqrt(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 1.732050808) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,sqrt(c2) from ct1") + tdSql.checkData(0, 1, 298.140906284) + tdSql.checkData(1 , 1, 278.885281074) + tdSql.checkData(3 , 1, 235.701081881) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,sqrt(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 2.979932904) + tdSql.checkData(2 , 2, 2.787471970) + tdSql.checkData(3 , 2, 2.580697551) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_sqrt( "select c1, c2, c3 , c4, c5 from ct1", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c4), sqrt(c5) from ct1") + + # nest query for sqrt functions + tdSql.query("select c4 , sqrt(c4) ,sqrt(sqrt(c4)) , sqrt(sqrt(sqrt(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 9.380831520) + tdSql.checkData(0 , 2 , 3.062814314) + tdSql.checkData(0 , 3 , 1.750089802) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 8.774964387) + tdSql.checkData(1 , 2 , 2.962256638) + tdSql.checkData(1 , 3 , 1.721120750) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select sqrt(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select sqrt(c1) from stbbb1") + tdSql.error("select sqrt(c1) from tbname") + tdSql.error("select sqrt(c1) from ct5") + + # mix with common col + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,2.828427125) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c2, sqrt(c2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,55555) + tdSql.checkData(4 , 1 ,235.701081881) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, sqrt(c1),sqrt(c1), sqrt(sqrt(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,2.449489743) + tdSql.checkData(3 , 2 ,2.449489743) + tdSql.checkData(3 , 3 ,1.565084580) + + tdSql.query("select c1, sqrt(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, sqrt(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, sqrt(c1),c5, count(c5) from ct1 ") + tdSql.error("select sqrt(c1), count(c5) from stb1 ") + tdSql.error("select sqrt(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, sqrt(c1) -0 ,sqrt(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.000000000) + + tdSql.query(" select c1, sqrt(c1) -0 ,sqrt(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.710693865) + + tdSql.query("select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, sqrt(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 10000.000000000) + + + tdSql.query("select c1, sqrt(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 3162277.660168380) + + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, 3162277660171.025390625) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, 100000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, 100000000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0, 1,2.828427125) + tdSql.checkRows(13) + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, sqrt(1) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start sqrt(x,y) + tdSql.query("select c1,c2, sqrt(c2) from ct1") + tdSql.checkData(0, 2, 298.140906284) + tdSql.checkData(1, 2, 278.885281074) + tdSql.checkData(4, 2, 0.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=sqrt(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,11111) + tdSql.checkData(0,2,1.000000000) + tdSql.checkData(0,3,1.000000000) + tdSql.checkData(0,4,0.900000000) + tdSql.checkData(0,5,1.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from sub1_bound") + + self.check_result_auto_sqrt( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c3), sqrt(c2) ,sqrt(c1) from sub1_bound") + + self.check_result_auto_sqrt("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select sqrt(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select sqrt(abs(c1)) ,sqrt(abs(c2)) , sqrt(abs(c3)) , sqrt(abs(c4)), sqrt(abs(c5)), sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483647)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767)) + tdSql.checkData(0,3,math.sqrt(127)) + tdSql.checkData(0,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.sqrt(2147483647)) + tdSql.checkData(1,1,math.sqrt(9223372036854775807)) + tdSql.checkData(1,2,math.sqrt(32767)) + tdSql.checkData(1,3,math.sqrt(127)) + tdSql.checkData(1,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.sqrt(2147483646)) + tdSql.checkData(3,1,math.sqrt(9223372036854775806)) + tdSql.checkData(3,2,math.sqrt(32766)) + tdSql.checkData(3,3,math.sqrt(126)) + tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483648.000000000)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767.000000000)) + tdSql.checkData(0,3,math.sqrt(63.500000000)) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: sqrt basic query ============") + + self.basic_sqrt_function() + + tdLog.printNoPrefix("==========step5: big number sqrt query ============") + + self.test_big_number() + + tdLog.printNoPrefix("==========step6: base number for sqrt query ============") + + self.pow_base_test() + + tdLog.printNoPrefix("==========step7: sqrt boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step8: sqrt filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b7a71bbe4..51f1649cc8 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -22,4 +22,6 @@ python3 ./test.py -f 2-query/abs.py python3 ./test.py -f 2-query/ceil.py python3 ./test.py -f 2-query/floor.py python3 ./test.py -f 2-query/round.py -python3 ./test.py -f 2-query/log.py \ No newline at end of file +python3 ./test.py -f 2-query/log.py +python3 ./test.py -f 2-query/pow.py +python3 ./test.py -f 2-query/sqrt.py diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index befcfc0731..08fe2cbc32 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -141,6 +141,9 @@ int32_t shellRunCommand(char *command) { *p++ = '\\'; } break; + default: + *p++ = '\\'; + break; } *p++ = c; esc = false; diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index c8ec31c48b..345b85d896 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -21,7 +21,7 @@ static void shellWorkAsClient() { SRpcInit rpcInit = {0}; SEpSet epSet = {.inUse = 0, .numOfEps = 1}; SRpcMsg rpcRsp = {0}; - void *clientRpc = NULL; + void * clientRpc = NULL; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass); @@ -116,6 +116,7 @@ static void shellWorkAsServer() { } SRpcInit rpcInit = {0}; + memcpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); rpcInit.localPort = pArgs->port; rpcInit.label = "CHK"; rpcInit.numOfThreads = tsNumOfRpcThreads; @@ -126,7 +127,7 @@ static void shellWorkAsServer() { void *serverRpc = rpcOpen(&rpcInit); if (serverRpc == NULL) { - printf("failed to init net test server since %s", terrstr()); + printf("failed to init net test server since %s\n", terrstr()); } else { printf("network test server is initialized, port:%u\n", pArgs->port); taosSetSignal(SIGTERM, shellNettestHandler); From 8e7cfb7f0257bd769d47ec7f25d3bcca60d76c84 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 6 May 2022 19:45:01 +0800 Subject: [PATCH 42/63] fix: some problems of parser and planner --- tests/script/tsim/table/basic1.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index be3b718fae..913ced74aa 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -91,9 +91,9 @@ print =============== create normal table sql create database ndb sql use ndb sql create table nt0 (ts timestamp, i int) -sql create table if not exists nt0 (ts timestamp, i int) +# sql create table if not exists nt0 (ts timestamp, i int) sql create table nt1 (ts timestamp, i int) -sql create table if not exists nt1 (ts timestamp, i int) +# sql create table if not exists nt1 (ts timestamp, i int) sql create table if not exists nt3 (ts timestamp, i int) sql show tables From aed53ac9e168a5efcb38571f10bf6dba87eefcfb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 13:06:10 +0000 Subject: [PATCH 43/63] feat: create table if not exists --- include/common/tmsg.h | 4 ++-- source/common/src/tmsg.c | 10 +++++----- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3889784f2f..763ff61e80 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -672,7 +672,6 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; - int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); @@ -685,7 +684,6 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); - typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -1554,7 +1552,9 @@ typedef struct SVDropStbReq { int32_t tEncodeSVDropStbReq(SCoder* pCoder, const SVDropStbReq* pReq); int32_t tDecodeSVDropStbReq(SCoder* pCoder, SVDropStbReq* pReq); +#define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { + int32_t flags; tb_uid_t uid; int64_t ctime; const char* name; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6278b52a04..85845dd5c4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1255,7 +1255,6 @@ int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) return 0; } - int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2091,7 +2090,7 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } -int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tSerializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2110,7 +2109,7 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp return tlen; } -int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tDeserializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2136,7 +2135,7 @@ int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchR return 0; } -void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); @@ -2146,7 +2145,6 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ taosArrayDestroy(pRsp->pArray); } - int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3746,6 +3744,7 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; if (tEncodeI64(pCoder, pReq->uid) < 0) return -1; if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1; @@ -3771,6 +3770,7 @@ int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1; if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8460400b59..811c5c10e4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -360,7 +360,11 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, // do create table if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) { - cRsp.code = terrno; + if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + cRsp.code = TSDB_CODE_SUCCESS; + } else { + cRsp.code = terrno; + } } else { cRsp.code = TSDB_CODE_SUCCESS; tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); From 8240b10a3ee28286d4a44dd6e1faf5554aaf7692 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 May 2022 21:36:39 +0800 Subject: [PATCH 44/63] enh(index): support more data type --- source/libs/index/inc/indexCache.h | 1 + source/libs/index/inc/indexComm.h | 2 +- source/libs/index/src/indexCache.c | 119 +++++++++++++++++++++++++++-- source/libs/index/src/indexComm.c | 27 +++++++ source/libs/index/src/indexTfile.c | 14 ++-- 5 files changed, 149 insertions(+), 14 deletions(-) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 0daca26ec9..d474d87409 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -48,6 +48,7 @@ typedef struct IndexCache { } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) + typedef struct CacheTerm { // key char* colVal; diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 0d8418ba65..3b07429089 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -24,7 +24,7 @@ extern char JSON_COLUMN[]; extern char JSON_VALUE_DELIM; char* indexPackJsonData(SIndexTerm* itm); - +char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index ad82dd0748..b3ae7b7dbe 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -86,9 +86,18 @@ static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = getComparFunc(type, 0); return tDoCommpare(func, QUERY_LESS_THAN, a, b); } -static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } -static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } -static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); +} +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_THAN, a, b); +} +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); +} static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual}; @@ -109,10 +118,12 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr } MemTable* mem = cache; IndexCache* pCache = mem->pCache; - CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; - CacheTerm* pCt = &ct; - char* key = indexCacheTermGet(&ct); + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* key = indexCacheTermGet(pCt); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { @@ -133,6 +144,8 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr break; } } + + taosMemoryFree(pCt); tSkipListDestroyIter(iter); return 0; } @@ -153,6 +166,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes if (cache == NULL) { return 0; } + _cache_range_compare cmpFn = rangeCompare[type]; MemTable* mem = cache; @@ -204,6 +218,48 @@ static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempRe } static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + if (cache == NULL) { + return 0; + } + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* exBuf = NULL; + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + exBuf = indexPackJsonData(term); + pCt->colVal = exBuf; + } + char* key = indexCacheTermGet(pCt); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + if (0 == strcmp(c->colVal, pCt->colVal)) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else { + break; + } + } + + taosMemoryFree(pCt); + taosMemoryFree(exBuf); + tSkipListDestroyIter(iter); + return 0; + return TSDB_CODE_SUCCESS; } static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { @@ -233,6 +289,56 @@ static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResu static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, RangeType type) { + if (cache == NULL) { + return 0; + } + _cache_range_compare cmpFn = rangeCompare[type]; + + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + int8_t dType = INDEX_TYPE_GET_TYPE(term->colType); + int skip = 0; + char* exBuf = NULL; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + exBuf = indexPackJsonDataPrefix(term, &skip); + pCt->colVal = exBuf; + } + char* key = indexCacheTermGet(pCt); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + + TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType); + if (cond == MATCH) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else if (cond == CONTINUE) { + continue; + } else if (cond == BREAK) { + break; + } + } + + taosMemoryFree(pCt); + taosMemoryFree(exBuf); + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; } static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { @@ -408,6 +514,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); + cache->mem->pCache = cache; cache->occupiedMem = 0; // sched to merge // unref cache in bgwork diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 4dea5fa011..cdd7b35675 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -46,3 +46,30 @@ char* indexPackJsonData(SIndexTerm* itm) { return buf; } +char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { + /* + * |<-----colname---->|<-----dataType---->|<--------colVal---------->| + * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| + */ + uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + + int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; + char* buf = (char*)taosMemoryCalloc(1, sz); + char* p = buf; + + memcpy(p, itm->colName, itm->nColName); + p += itm->nColName; + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, &ty, sizeof(ty)); + p += sizeof(ty); + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + *skip = p - buf; + + return buf; +} diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 9edd868272..b5551e825f 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -308,20 +308,20 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) } static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); - int ret = 0; - char* p = tem->colVal; - uint64_t sz = tem->nColVal; + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + int skip = 0; + if (hasJson) { - p = indexPackJsonData(tem); - sz = strlen(p); + p = indexPackJsonDataPrefix(tem, &skip); } SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); - FstSlice h = fstSliceCreate((uint8_t*)p, sz); + FstSlice h = fstSliceCreate((uint8_t*)p, skip); fstStreamBuilderSetRange(sb, &h, type); fstSliceDestroy(&h); From ea3e1a202e51c769286d434f9da3fdd698aee70e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 May 2022 22:12:35 +0800 Subject: [PATCH 45/63] fix(query): add some check for null pointer. --- source/libs/executor/src/sortoperator.c | 4 ++++ source/libs/executor/src/tsort.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 619651f11f..b34889ec46 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -67,6 +67,10 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); SSDataBlock* p = tsortGetSortedDataBlock(pHandle); + if (p == NULL) { + return NULL; + } + blockDataEnsureCapacity(p, capacity); while (1) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 040ee8c7f5..d585988e5e 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -553,7 +553,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { } } - if (pHandle->pDataBlock->info.rows > 0) { + if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { size_t size = blockDataGetSize(pHandle->pDataBlock); // Perform the in-memory sort and then flush data in the buffer into disk. From 0886d23e02d1c91ca323bf397034dbf3746feaae Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:17:56 +0800 Subject: [PATCH 46/63] fix: alter db replications --- source/dnode/mnode/impl/inc/mndVgroup.h | 5 +- source/dnode/mnode/impl/src/mndDb.c | 35 ++++--- source/dnode/mnode/impl/src/mndVgroup.c | 94 ++++++++++++++++--- source/dnode/mnode/impl/test/dnode/mdnode.cpp | 92 ++++++++++++++++++ 4 files changed, 199 insertions(+), 27 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 87be15a4fd..1e95859157 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -31,8 +31,9 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist); -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist); +SArray *mndBuildDnodesArray(SMnode *pMnode); +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cf9dd664e3..c3e4629f2a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -399,6 +399,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; + if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; @@ -720,7 +721,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -732,27 +733,30 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj newVgroup = {0}; memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); if (newVgroup.replica < pDb->cfg.replications) { - SVnodeGid new1 = {0}; - SVnodeGid new2 = {0}; - SVnodeGid exist = {0}; - if (mndAddVnodeToVgroup(pMnode, &newVgroup, &new1, &new2, &exist) != 0) { + mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, + pVgroup->vnodeGid[0].dnodeId); + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) { mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); return -1; } - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new1, true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new2, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; } else { + mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); + SVnodeGid del1 = {0}; SVnodeGid del2 = {0}; - SVnodeGid exist = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, &del1, &del2, &exist) != 0) { + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1, &del2) != 0) { mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); return -1; } + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; } SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); @@ -765,8 +769,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj } static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SArray *pArray = mndBuildDnodesArray(pMnode); while (1) { SVgObj *pVgroup = NULL; @@ -774,9 +779,10 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); + taosArrayDestroy(pArray); return -1; } } @@ -784,6 +790,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * sdbRelease(pSdb, pVgroup); } + taosArrayDestroy(pArray); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ca60a3b9ed..48a475ec63 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -370,7 +370,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 return true; } -static SArray *mndBuildDnodesArray(SMnode *pMnode) { +SArray *mndBuildDnodesArray(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); @@ -421,7 +421,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } - mDebug("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); pDnode->numOfVnodes++; } @@ -440,12 +440,10 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { } pArray = mndBuildDnodesArray(pMnode); - if (pArray == NULL) { - goto _OVER; - } + if (pArray == NULL) goto _OVER; - mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), - pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); + mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), + pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); int32_t allocedVgroups = 0; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); @@ -483,7 +481,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { *ppVgroups = pVgroups; code = 0; - mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); + mInfo("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); _OVER: if (code != 0) taosMemoryFree(pVgroups); @@ -491,11 +489,85 @@ _OVER: return code; } -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist) { - return 0; +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + int32_t maxPos = 1; + for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + bool used = false; + for (int32_t vn = 0; vn < maxPos; ++vn) { + if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { + used = true; + break; + } + } + if (used) continue; + + if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + + SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; + pVgid->dnodeId = pDnode->id; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pDnode->numOfVnodes++; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); + maxPos++; + if (maxPos == 3) return 0; + } + + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; } -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist) { +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { + int32_t removedNum = 0; + + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + for (int32_t vn = 0; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId == pDnode->id) { + if (removedNum == 0) *del1 = *pVgid; + if (removedNum == 1) *del2 = *pVgid; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + memset(pVgid, 0, sizeof(SVnodeGid)); + removedNum++; + pDnode->numOfVnodes--; + + if (removedNum == 2) goto _OVER; + } + } + } + +_OVER: + if (removedNum != 2) return -1; + + for (int32_t vn = 1; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId != 0) { + memcpy(&pVgroup->vnodeGid[0], pVgid, sizeof(SVnodeGid)); + memset(pVgid, 0, sizeof(SVnodeGid)); + } + } + + mInfo("db:%s, vgId:%d, dnode:%d is keeped", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); return 0; } diff --git a/source/dnode/mnode/impl/test/dnode/mdnode.cpp b/source/dnode/mnode/impl/test/dnode/mdnode.cpp index a4cbc201a9..15ade8166f 100644 --- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp @@ -266,4 +266,96 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { taosMsleep(1300); test.SendShowReq(TSDB_MGMT_TABLE_DNODE, "dnodes", ""); EXPECT_EQ(test.GetShowRows(), 4); + + // alter replica +#if 0 + { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; + createReq.daysPerFile = 1000; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.strict = 1; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + createReq.numOfStables = 0; + createReq.numOfRetensions = 0; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + EXPECT_EQ(test.GetShowRows(), 3); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 3; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + +#endif } From 384fed49f986d9af907109648b76708090cc745b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:32:22 +0800 Subject: [PATCH 47/63] enh: add authVersion for user privilege --- source/dnode/mnode/impl/src/mndUser.c | 21 ++++++++++----------- tools/taos-tools | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 8b5fa06faf..81ded4ef8b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -106,6 +106,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) + SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) @@ -161,6 +162,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -588,7 +590,7 @@ _OVER: return code; } -static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = pUser->superUser; pRsp->version = pUser->authVersion; @@ -601,7 +603,7 @@ static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUser terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -659,7 +661,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: - + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -711,7 +713,8 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen) { SUserAuthBatchRsp batchRsp = {0}; batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); if (batchRsp.pArray == NULL) { @@ -731,7 +734,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ mndReleaseUser(pMnode, pUser); continue; } - + SGetUserAuthRsp rsp = {0}; code = mndSetUserAuthRsp(pMnode, pUser, &rsp); if (code) { @@ -740,7 +743,6 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ goto _OVER; } - taosArrayPush(batchRsp.pArray, &rsp); mndReleaseUser(pMnode, pUser); } @@ -748,7 +750,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ if (taosArrayGetSize(batchRsp.pArray) <= 0) { *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return 0; } @@ -772,10 +774,7 @@ _OVER: *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return code; } - - - diff --git a/tools/taos-tools b/tools/taos-tools index 59e0ebaf49..2f3dfddd4d 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20 +Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c From 99efc7334aa59741cd1784574a8449f0505f5cdf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:53:49 +0800 Subject: [PATCH 48/63] enh: add authVersion for user privilege --- source/dnode/mnode/impl/src/mndDb.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c3e4629f2a..70bdda5855 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1070,6 +1070,17 @@ static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; + SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser); + if (pUser != NULL) { + pUser->authVersion++; + SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + goto _OVER; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + } + int32_t rspLen = 0; void *pRsp = NULL; if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto _OVER; From 612bdec1f746eebd57b12db9519c6924f18fb041 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:58:02 +0800 Subject: [PATCH 49/63] fix: invalid header file --- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8619df978b..0ec4332d37 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -21,7 +21,7 @@ #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndShow.h" -#include "mndStb.c" +#include "mndStb.h" #include "mndStream.h" #include "mndTrans.h" #include "mndUser.h" diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8ae0d5d19c..c32008cb5c 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndStb.h" #include "mndAuth.h" #include "mndDb.h" From b38f18020d73bf43d68387660195f8891fb1d401 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 23:04:25 +0800 Subject: [PATCH 50/63] enh: check kill trans auth --- source/dnode/mnode/impl/inc/mndAuth.h | 1 + source/dnode/mnode/impl/src/mndAuth.c | 24 +++++++++--------------- source/dnode/mnode/impl/src/mndTrans.c | 3 +-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index ae7663a775..890879912f 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -31,6 +31,7 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser); int32_t mndCheckNodeAuth(SUserObj *pOperUser); int32_t mndCheckFuncAuth(SUserObj *pOperUser); +int32_t mndCheckTransAuth(SUserObj *pOperUser); int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 696d850266..8e5ec40c47 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -74,10 +74,7 @@ static int32_t mndProcessAuthReq(SNodeMsg *pReq) { } int32_t mndCheckCreateUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } @@ -118,28 +115,25 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, } int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckNodeAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckFuncAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } + if (pOperUser->superUser) return 0; + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} +int32_t mndCheckTransAuth(SUserObj *pOperUser) { + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 43e2735984..e57e9a0461 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1336,8 +1336,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { goto _OVER; } - if (!pUser->superUser) { - terrno = TSDB_CODE_MND_NO_RIGHTS; + if (mndCheckTransAuth(pUser) != 0) { goto _OVER; } From 2b803d8b930a0e63c6d43d80ec975f1627435d4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 May 2022 23:40:26 +0800 Subject: [PATCH 51/63] enh(query): set uid for the result datablock of stream scanner when handling the submit block. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqRead.c | 7 +++++- source/libs/executor/src/scanoperator.c | 31 ++++++++++++++----------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 62deb200c9..488edc4f25 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -126,7 +126,7 @@ int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); -int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows, +int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t* pUid, int32_t *pNumOfRows, int16_t *pNumOfCols); // need to reposition diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 261b4aa8ae..ffd76af5e2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -161,7 +161,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows, + if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, &block.info.numOfCols) < 0) { ASSERT(0); } @@ -540,7 +540,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { tqReadHandleSetMsg(pReader, pCont, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows, + if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, &block.info.numOfCols) < 0) { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 511d57ed58..5ec3ab0b47 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -84,10 +84,12 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { return false; } -int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows, +int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, int32_t* pNumOfRows, int16_t* pNumOfCols) { /*int32_t sversion = pHandle->pBlock->sversion;*/ // TODO set to real sversion + *pUid = 0; + int32_t sversion = 0; if (pHandle->sver != sversion) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); @@ -169,7 +171,10 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterInit(&iter, pTschema); STSRow* row; int32_t curRow = 0; + tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); + *pUid = pHandle->pBlock->uid; // set the uid of table for submit block + while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b728daa3bb..b28a65d1d2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -539,10 +539,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { while (tqNextDataBlock(pInfo->readerHandle)) { SArray* pCols = NULL; - uint64_t groupId; - int32_t numOfRows; - int16_t outputCol; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows, &outputCol); + uint64_t groupId = 0; + uint64_t uid = 0; + int32_t numOfRows = 0; + int16_t outputCol = 0; + + int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &uid, &numOfRows, &outputCol); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; @@ -551,6 +553,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = groupId; pInfo->pRes->info.rows = numOfRows; + pInfo->pRes->info.uid = uid; int32_t numOfCols = pInfo->pRes->info.numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { @@ -606,10 +609,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; + goto _error; } int32_t numOfOutput = taosArrayGetSize(pColList); @@ -626,16 +627,13 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColIds); int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); if (code != 0) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; + goto _error; } pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); if (pInfo->pBlockLists == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _error; } pInfo->readerHandle = streamReadHandle; @@ -647,7 +645,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->numOfExprs = pResBlock->info.numOfCols; + pOperator->numOfExprs = pResBlock->info.numOfCols; pOperator->fpSet._openFn = operatorDummyOpenFn; pOperator->fpSet.getNextFn = doStreamBlockScan; pOperator->fpSet.closeFn = operatorDummyCloseFn; @@ -656,6 +654,11 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL); return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } static void destroySysScanOperator(void* param, int32_t numOfOutput) { From d75b3568a70dce7e39c22fc982f69c852878be73 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 6 May 2022 23:40:34 +0800 Subject: [PATCH 52/63] enh(sync): add trace log --- source/libs/sync/src/syncRaftLog.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index ae153251c3..031722ab3c 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,7 +57,13 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - assert(code == 0); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(code == 0); walFsync(pWal, true); return code; @@ -69,7 +75,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - assert(walReadWithHandle(pWalHandle, index) == 0); + int32_t code = walReadWithHandle(pWalHandle, index); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walReadWithHandle error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); assert(pEntry != NULL); From 27a74a56dfe52a29eaab8cdea615a0778d99a619 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 00:13:25 +0800 Subject: [PATCH 53/63] feat: merge rows in mem/file during commit for update --- source/common/src/trow.c | 12 +++- source/dnode/vnode/src/tsdb/tsdbCommit.c | 52 ++++++++------ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 83 ++++++++++++++++++---- tests/script/tsim/query/scalarFunction.sim | 1 + 4 files changed, 110 insertions(+), 38 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 4c063b205f..261b6e1d67 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -466,7 +466,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; } - } else { + } else if (!tdValTypeIsNone(valType)) { if (IS_VAR_DATA_TYPE(pCol->type)) { // keep the last offset // discard the last var data @@ -483,7 +483,9 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int } #ifdef TD_SUPPORT_BITMAP - tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + if (!isMerge || !tdValTypeIsNone(valType)) { + tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + } #endif return 0; } @@ -533,7 +535,9 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols ++dcol; } } +#if 0 ++pCols->numOfRows; +#endif return TSDB_CODE_SUCCESS; } @@ -584,7 +588,9 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols ++dcol; } } +#if 0 ++pCols->numOfRows; +#endif return TSDB_CODE_SUCCESS; } @@ -698,7 +704,7 @@ static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, b tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode, isMerge); } else { - // Keep the origi value for None + // Keep the origin value for None } } else { tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b255622c60..a008c4a0e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1330,13 +1330,15 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt TSKEY maxKey, int maxRows, int8_t update) { TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; + TSKEY lastKey = TSKEY_INITIAL_VAL; STSchema *pSchema = NULL; ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); tdResetDataCols(pTarget); pTarget->bitmapMode = pDataCols->bitmapMode; - + // TODO: filter Multi-Version + // TODO: support delete function while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); STSRow *row = tsdbNextIterRow(pCommitIter->pIter); @@ -1349,6 +1351,9 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt if (key1 == INT64_MAX && key2 == INT64_MAX) break; if (key1 < key2) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } for (int i = 0; i < pDataCols->numOfCols; ++i) { // TODO: dataColAppendVal may fail SCellVal sVal = {0}; @@ -1359,7 +1364,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt pTarget->bitmapMode, false); } - ++pTarget->numOfRows; + lastKey = key1; ++(*iter); } else if (key1 > key2) { if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { @@ -1367,7 +1372,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); + if (key2 == lastKey) { + if (TD_SUPPORT_UPDATE(update)) { + tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); + } + } else { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); + lastKey = key2; + } tSkipListIterNext(pCommitIter->pIter); } else { @@ -1397,6 +1412,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ++(*iter); tSkipListIterNext(pCommitIter->pIter); #endif + + if(lastKey != key1) { + lastKey = key1; + ++pTarget->numOfRows; + } + // copy disk data for (int i = 0; i < pDataCols->numOfCols; ++i) { SCellVal sVal = {0}; @@ -1416,26 +1437,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt } // TODO: merge with Multi-Version - STSRow *curRow = row; - - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); - STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter); - - if (key2 < TD_ROW_KEY(nextRow)) { - tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); - } else { - tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); - } - // TODO: merge with Multi-Version - } else { - ++pTarget->numOfRows; - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); } + ++(*iter); + tSkipListIterNext(pCommitIter->pIter); } - if (pTarget->numOfRows >= maxRows) break; + if (pTarget->numOfRows >= (maxRows - 1)) break; + } + + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 6520f371f6..48d32c0dd3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -20,7 +20,7 @@ static void tsdbFreeTbData(STbData *pTbData); static char *tsdbGetTsTupleKey(const void *data); static int tsdbTbDataComp(const void *arg1, const void *arg2); static char *tsdbTbDataGetUid(const void *arg); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge); int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) { STsdbMemTable *pMemTable; @@ -82,15 +82,19 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; - STSchema *pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; + STSchema *pSchema = NULL; + TSKEY rowKey = 0; + TSKEY fKey = 0; + // only fetch lastKey from mem data as file data not used in this function actually TSKEY lastKey = TSKEY_INITIAL_VAL; bool isRowDel = false; int filterIter = 0; STSRow *row = NULL; SMergeInfo mInfo; + // TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by + // query handle) + if (pMergeInfo == NULL) pMergeInfo = &mInfo; memset(pMergeInfo, 0, sizeof(*pMergeInfo)); @@ -190,21 +194,66 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } } #endif - } else { // fkey >= rowKey +#if 1 + } else if (fKey > rowKey) { if (isRowDel) { + // TODO: support delete function + pMergeInfo->rowsDeleteFailed++; + } else { + if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + + if (lastKey != rowKey) { + pMergeInfo->rowsInserted++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + if (pCols) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pCols->numOfRows; + } + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); + } + lastKey = rowKey; + } else { + if (keepDup) { + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true); + } else { + // discard + } + } + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || TD_ROW_KEY(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = TD_ROW_KEY(row); + isRowDel = TD_ROW_IS_DELETED(row); + } + } else { // fkey == rowKey + if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?) ASSERT(!keepDup); if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsDeleteSucceed++; pMergeInfo->nOperations++; - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); } else { if (keepDup) { if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsUpdated++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + if (lastKey != rowKey) { + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + lastKey = rowKey; + ++pCols->numOfRows; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); + } else { + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true); + } } else { pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); @@ -228,6 +277,10 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey fKey = tdGetKey(filterKeys[filterIter]); } } +#endif + } + if (lastKey != TSKEY_INITIAL_VAL) { + ++pCols->numOfRows; } return 0; @@ -301,8 +354,8 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); #endif - pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, - tsdbGetTsTupleKey); + pTbData->pData = + tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, tsdbGetTsTupleKey); if (pTbData->pData == NULL) { taosMemoryFree(pTbData); return NULL; @@ -337,7 +390,7 @@ static char *tsdbTbDataGetUid(const void *arg) { STbData *pTbData = (STbData *)arg; return (char *)(&(pTbData->uid)); } -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row) { +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge) { if (pCols) { if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) { *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, TD_ROW_SVER(row)); @@ -347,7 +400,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } } - tdAppendSTSRowToDataCol(row, *ppSchema, pCols, false); + tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge); } return 0; diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim index 97d49ae901..2946a89ff6 100644 --- a/tests/script/tsim/query/scalarFunction.sim +++ b/tests/script/tsim/query/scalarFunction.sim @@ -102,6 +102,7 @@ print ====> $data60 $data61 $data62 $data63 $data64 $data65 print ====> $data70 $data71 $data72 $data73 $data74 $data75 print ====> $data80 $data81 $data82 $data83 $data84 $data85 print ====> $data90 $data91 $data92 $data93 $data94 $data95 +print ====> rows = $rows and rowNum = $rowNum for ct1 if $rows != $rowNum then return -1 endi From 6b18948884f00184aaa22d743423cdcd51d362e8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 7 May 2022 00:14:52 +0800 Subject: [PATCH 54/63] enh(wal): set error code --- source/libs/sync/src/syncRaftLog.c | 1 + source/libs/wal/src/walWrite.c | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index ae153251c3..5fa77c9964 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,6 +57,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + perror("wal write error: "); assert(code == 0); walFsync(pWal, true); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index dc31086e9f..da1c36dcc4 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -198,12 +198,14 @@ int walRoll(SWal *pWal) { if (pWal->pWriteIdxTFile != NULL) { code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } if (pWal->pWriteLogTFile != NULL) { code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } @@ -263,14 +265,19 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog if (index == pWal->vers.lastVer + 1) { if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->vers.firstVer = index; - code = walRoll(pWal); - ASSERT(code == 0); + if (walRoll(pWal) < 0) { + return -1; + } } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { - walRoll(pWal); + if (walRoll(pWal) < 0) { + return -1; + } } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { - walRoll(pWal); + if (walRoll(pWal) < 0) { + return -1; + } } } } else { From 1fa906cde0e12f2af29490d817423fe5a2cef923 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 00:46:09 +0800 Subject: [PATCH 55/63] fix: fill null val --- include/common/tdataformat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 74d11a5ffb..e94be797b9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,7 +27,7 @@ extern "C" { // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#undef TD_SUPPORT_READ2 +#define TD_SUPPORT_READ2 #define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) From b1a429f7df17f1687d4d87134a83b13915fe4838 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 00:56:07 +0800 Subject: [PATCH 56/63] feat: code optimization for update --- source/common/src/trow.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 261b6e1d67..50e51a40dd 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -670,7 +670,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * } } } - if (rowsToMerge > 0) { + if (lastKey != TSKEY_INITIAL_VAL) { ++target->numOfRows; } (*pOffset) += rowsToMerge; @@ -778,7 +778,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i ASSERT(target->numOfRows <= target->maxPoints - 1); } - if (nRows > 0) { + if (lastKey != TSKEY_INITIAL_VAL) { ++target->numOfRows; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 54b3eea988..aa305dd71c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -816,9 +816,10 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); - if (TD_ROW_KEY(rmem) > maxVer) { - rmem = NULL; - } + // TODO: filter max version + // if (TD_ROW_VER(rmem) > maxVer) { + // rmem = NULL; + // } } } @@ -826,9 +827,10 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); - if (TD_ROW_KEY(rimem) > maxVer) { - rimem = NULL; - } + // TODO: filter max version + // if (TD_ROW_VER(rimem) > maxVer) { + // rimem = NULL; + // } } } From 3ae79a77c75b0a3a4443f7d1017342438743e005 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 7 May 2022 01:47:45 +0800 Subject: [PATCH 57/63] enh(stream): fix create stb in stream --- source/common/src/tdatablock.c | 29 +++--- source/dnode/mnode/impl/inc/mndStb.h | 5 ++ source/dnode/mnode/impl/src/mndStb.c | 112 +++++++++++++----------- source/dnode/mnode/impl/src/mndStream.c | 76 +++++++++++++++- source/libs/sync/src/syncRaftLog.c | 2 +- 5 files changed, 159 insertions(+), 65 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a4b36dcb67..2c17f2c2fc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1478,11 +1478,11 @@ void blockDebugShowData(const SArray* dataBlocks) { * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in * SDataBlock->info.uid * @param suid // TODO: check with Liao whether suid response is reasonable - * + * * TODO: colId should be set */ -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, - tb_uid_t suid) { +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, + tb_uid_t uid, tb_uid_t suid) { int32_t sz = taosArrayGetSize(pDataBlocks); int32_t bufSize = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; ++i) { @@ -1494,16 +1494,16 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks ASSERT(bufSize < 3 * 1024 * 1024); *pReq = taosMemoryCalloc(1, bufSize); - if(!(*pReq)) { + if (!(*pReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } void* pDataBuf = *pReq; - int32_t msgLen = sizeof(SSubmitReq); + int32_t msgLen = sizeof(SSubmitReq); int32_t numOfBlks = 0; SRowBuilder rb = {0}; - tdSRowInit(&rb, 0); // TODO: use the latest version + tdSRowInit(&rb, 0); // TODO: use the latest version for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); @@ -1511,8 +1511,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t rows = pDataBlock->info.rows; int32_t rowSize = pDataBlock->info.rowSize; int64_t groupId = pDataBlock->info.groupId; - - if(rb.nCols != colNum) { + + if (rb.nCols != colNum) { tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); } @@ -1525,10 +1525,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks msgLen += sizeof(SSubmitBlk); int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row - tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf + for (int32_t j = 0; j < rows; ++j) { // iterate by row + tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf printf("|"); - bool isStartKey = false; + bool isStartKey = false; for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); @@ -1536,7 +1536,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks case TSDB_DATA_TYPE_TIMESTAMP: if (!isStartKey) { isStartKey = true; - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, + 0, 0); } else { tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k); break; @@ -1629,14 +1630,14 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) { blkHead->uid = htobe64(pDataBlock->info.uid); int32_t rows = pDataBlock->info.rows; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ /*blkHead->dataLen = htonl(rows * maxLen);*/ blkHead->dataLen = 0; void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); STSRow* rowData = blockData; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { + for (int32_t j = 0; j < rows; j++) { SRowBuilder rb = {0}; tdSRowInit(&rb, pTSchema->version); tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index a415d39434..28d3215b98 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -31,6 +31,11 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t n int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); +int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate); +SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName); +int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb); +int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8ae0d5d19c..cd76c7c8bb 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -342,7 +342,7 @@ void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) { sdbRelease(pSdb, pStb); } -static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { +SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { SName name = {0}; tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -463,7 +463,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, return pHead; } -static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { +int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { if (pCreate->igExists < 0 || pCreate->igExists > 1) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -634,91 +634,96 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { return NULL; } -static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { - SStbObj stbObj = {0}; - memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); - memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); - stbObj.createdTime = taosGetTimestampMs(); - stbObj.updateTime = stbObj.createdTime; - stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); - stbObj.dbUid = pDb->uid; - stbObj.version = 1; - stbObj.nextColId = 1; - stbObj.xFilesFactor = pCreate->xFilesFactor; - stbObj.delay = pCreate->delay; - stbObj.ttl = pCreate->ttl; - stbObj.numOfColumns = pCreate->numOfColumns; - stbObj.numOfTags = pCreate->numOfTags; - stbObj.commentLen = pCreate->commentLen; - if (stbObj.commentLen > 0) { - stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); - if (stbObj.comment == NULL) { +int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) { + memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN); + memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); + pDst->createdTime = taosGetTimestampMs(); + pDst->updateTime = pDst->createdTime; + pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + pDst->dbUid = pDb->uid; + pDst->version = 1; + pDst->nextColId = 1; + pDst->xFilesFactor = pCreate->xFilesFactor; + pDst->delay = pCreate->delay; + pDst->ttl = pCreate->ttl; + pDst->numOfColumns = pCreate->numOfColumns; + pDst->numOfTags = pCreate->numOfTags; + pDst->commentLen = pCreate->commentLen; + if (pDst->commentLen > 0) { + pDst->comment = taosMemoryCalloc(pDst->commentLen, 1); + if (pDst->comment == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); + memcpy(pDst->comment, pCreate->comment, pDst->commentLen); } - stbObj.ast1Len = pCreate->ast1Len; - if (stbObj.ast1Len > 0) { - stbObj.pAst1 = taosMemoryCalloc(stbObj.ast1Len, 1); - if (stbObj.pAst1 == NULL) { + pDst->ast1Len = pCreate->ast1Len; + if (pDst->ast1Len > 0) { + pDst->pAst1 = taosMemoryCalloc(pDst->ast1Len, 1); + if (pDst->pAst1 == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.pAst1, pCreate->pAst1, stbObj.ast1Len); + memcpy(pDst->pAst1, pCreate->pAst1, pDst->ast1Len); } - stbObj.ast2Len = pCreate->ast2Len; - if (stbObj.ast2Len > 0) { - stbObj.pAst2 = taosMemoryCalloc(stbObj.ast2Len, 1); - if (stbObj.pAst2 == NULL) { + pDst->ast2Len = pCreate->ast2Len; + if (pDst->ast2Len > 0) { + pDst->pAst2 = taosMemoryCalloc(pDst->ast2Len, 1); + if (pDst->pAst2 == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.pAst2, pCreate->pAst2, stbObj.ast2Len); + memcpy(pDst->pAst2, pCreate->pAst2, pDst->ast2Len); } - stbObj.pColumns = taosMemoryCalloc(1, stbObj.numOfColumns * sizeof(SSchema)); - stbObj.pTags = taosMemoryCalloc(1, stbObj.numOfTags * sizeof(SSchema)); - if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { + pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema)); + pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema)); + if (pDst->pColumns == NULL || pDst->pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < stbObj.numOfColumns; ++i) { + for (int32_t i = 0; i < pDst->numOfColumns; ++i) { SField *pField = taosArrayGet(pCreate->pColumns, i); - SSchema *pSchema = &stbObj.pColumns[i]; + SSchema *pSchema = &pDst->pColumns[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; pSchema->flags = pField->flags; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); - pSchema->colId = stbObj.nextColId; - stbObj.nextColId++; + pSchema->colId = pDst->nextColId; + pDst->nextColId++; } - for (int32_t i = 0; i < stbObj.numOfTags; ++i) { + for (int32_t i = 0; i < pDst->numOfTags; ++i) { SField *pField = taosArrayGet(pCreate->pTags, i); - SSchema *pSchema = &stbObj.pTags[i]; + SSchema *pSchema = &pDst->pTags[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); - pSchema->colId = stbObj.nextColId; - stbObj.nextColId++; + pSchema->colId = pDst->nextColId; + pDst->nextColId++; } + return 0; +} + +static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { + SStbObj stbObj = {0}; int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); - mndTransSetDbInfo(pTrans, pDb); - if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) { + goto _OVER; + } + + if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -727,6 +732,15 @@ _OVER: mndTransDrop(pTrans); return code; } +int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + mndTransSetDbInfo(pTrans, pDb); + if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; + return 0; +} static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5c6e2ce771..1404b1cd94 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -290,6 +290,72 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast return 0; } +static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { + SStbObj *pStb = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + + SMCreateStbReq createReq = {0}; + tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); + createReq.numOfColumns = pStream->outputSchema.nCols; + createReq.numOfTags = 1; // group id + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + // build fields + // build tags + + if (mndCheckCreateStbReq(&createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.name); + if (pStb != NULL) { + terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; + goto _OVER; + } + + pDb = mndAcquireDbByStb(pMnode, createReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + int32_t numOfStbs = -1; + mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs); + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { + terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; + goto _OVER; + } + + SStbObj stbObj = {0}; + + if (mndBuildStbFromReq(pMnode, &stbObj, &createReq, pDb) != 0) { + goto _OVER; + } + + if (mndBuildStbFromReq(pMnode, pStb, &createReq, pDb) != 0) { + goto _OVER; + } + + if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; + + return pStb; +_OVER: + mndReleaseStb(pMnode, pStb); + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + return NULL; +} + static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; @@ -311,13 +377,21 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.trigger = pCreate->triggerType; streamObj.waterMark = pCreate->watermark; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); +#if 0 + if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } +#endif + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 5fa77c9964..6bf17b8a82 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,7 +57,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - perror("wal write error: "); + if (code < 0) perror("wal write error: "); assert(code == 0); walFsync(pWal, true); From 6b05911e45f5c34bace8ab9721ff58e9277ad49d Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 7 May 2022 06:49:06 +0800 Subject: [PATCH 58/63] remove windows test --- Jenkinsfile2 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 49e13b7831..d827c6640a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -168,13 +168,6 @@ pipeline { stages { stage('run test') { parallel { - stage('windows test') { - agent {label " windows11 "} - steps { - pre_test_win() - pre_test_build_win() - } - } stage('linux test') { agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } From ce0841e398972eab05044a2f29a727b6e4ee8316 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 01:58:01 +0000 Subject: [PATCH 59/63] fix: auto-create table --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/parser/src/parInsert.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 397e074061..ffe176f85e 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -196,7 +196,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int8_t type; int64_t ctime; tb_uid_t suid; - int c, ret; + int c = 0, ret; // search & delete the name idx tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index e82873b923..8396a33122 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -279,6 +279,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { int32_t numOfBlocks = blocks->numOfTables; while (numOfBlocks--) { int32_t dataLen = blk->dataLen; + int32_t schemaLen = blk->schemaLen; blk->uid = htobe64(blk->uid); blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); @@ -286,7 +287,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { blk->dataLen = htonl(blk->dataLen); blk->schemaLen = htonl(blk->schemaLen); blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); + blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); } } From 861fde25087bec9594cf7c6f324e07754ed9a34c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 7 May 2022 10:21:51 +0800 Subject: [PATCH 60/63] fix(stream): create stb in stream --- example/src/tstream.c | 8 +++--- source/dnode/mnode/impl/inc/mndDef.h | 3 +-- source/dnode/mnode/impl/src/mndSma.c | 1 - source/dnode/mnode/impl/src/mndStream.c | 34 +++++++++++++++---------- source/libs/wal/src/walRead.c | 5 +++- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/example/src/tstream.c b/example/src/tstream.c index 5bd833213d..65fd005954 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -81,10 +81,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = - taos_query(pConn, - "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k " - "from tu1 interval(10m)"); + pRes = taos_query( + pConn, + "create stream stream1 trigger window_close into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k " + "from tu1 interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 89638524ae..9ae7922f6b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -72,7 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, - + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, @@ -601,7 +601,6 @@ typedef struct { int32_t triggerParam; int64_t waterMark; char* sql; - char* logicalPlan; char* physicalPlan; SArray* tasks; // SArray> SSchemaWrapper outputSchema; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8619df978b..91a2bd4740 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -419,7 +419,6 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre streamObj.fixedSinkVgId = smaObj.dstVgId; streamObj.smaId = smaObj.uid; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 1404b1cd94..aa7f16e10d 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -290,7 +290,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast return 0; } -static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { +static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -301,7 +301,22 @@ static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStr createReq.numOfTags = 1; // group id createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); // build fields + taosArraySetSize(createReq.pColumns, createReq.numOfColumns); + for (int32_t i = 0; i < createReq.numOfColumns; i++) { + SField *pField = taosArrayGet(createReq.pColumns, i); + tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN); + pField->flags = pStream->outputSchema.pSchema[i].flags; + pField->type = pStream->outputSchema.pSchema[i].type; + pField->bytes = pStream->outputSchema.pSchema[i].bytes; + } + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, 1); // build tags + SField *pField = taosArrayGet(createReq.pTags, 0); + strcpy(pField->name, "group_id"); + pField->type = TSDB_DATA_TYPE_UBIGINT; + pField->flags = 0; + pField->bytes = 8; if (mndCheckCreateStbReq(&createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -342,18 +357,14 @@ static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStr goto _OVER; } - if (mndBuildStbFromReq(pMnode, pStb, &createReq, pDb) != 0) { - goto _OVER; - } - if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; - return pStb; + return 0; _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); - return NULL; + return -1; } static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { @@ -373,7 +384,6 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.fixedSinkVgId = 0; streamObj.smaId = 0; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; streamObj.trigger = pCreate->triggerType; streamObj.waterMark = pCreate->watermark; @@ -384,16 +394,14 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); -#if 0 - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } -#endif - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { - mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); + if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 70a3559cd9..4fe07029f1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -49,7 +49,10 @@ void walCloseReadHandle(SWalReadHandle *pRead) { taosMemoryFree(pRead); } -int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { + // TODO + return 0; +} static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; From f942f144e9b3958a72189a2d568c7c7496f77b68 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 11:12:46 +0800 Subject: [PATCH 61/63] feat: bitmap operation optimization --- source/dnode/vnode/src/inc/tsdb.h | 5 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 ++++---- source/dnode/vnode/src/tsdb/tsdbRead.c | 29 +++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 29 ++++------------------ 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d84c1b1d07..58003a97d7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -325,9 +325,8 @@ typedef struct { typedef struct { int16_t colId; uint16_t type : 6; - uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) - uint32_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint32_t len : 31; // data length + bitmap length + uint16_t blen : 10; // 0 no bitmap if all rows are NORM, > 0 bitmap length + uint32_t len; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a008c4a0e5..f0b1baf1da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -943,16 +943,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF &(pAggrBlkCol->numOfNull)); if (pAggrBlkCol->numOfNull == 0) { - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsOfBlockSma; } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) { // check if all rows normal - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsNotAllNull; @@ -985,7 +985,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; int32_t tBitmapsLen = 0; - if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((ncol != 0) && (pBlockCol->blen > 0)) { tBitmaps = isSuper ? sBitmaps : nBitmaps; } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index aa305dd71c..0b9f21dbcf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1619,7 +1619,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa SCellVal sVal = {0}; TSKEY rowKey = TSKEY_INITIAL_VAL; int32_t nResult = 0; - bool isMerge = true; + int32_t mergeOption = 0; // 0 discard 1 overwrite 2 merge // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; @@ -1715,12 +1715,18 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { - isMerge = false; + mergeOption = 1; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } ++nResult; + } else if (update){ + mergeOption = 2; + } else { + mergeOption = 0; + break; } + *lastRowKey = rowKey; } } else { @@ -1730,11 +1736,16 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { - isMerge = false; + mergeOption = 1; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } ++nResult; + } else if(update) { + mergeOption = 2; + } else { + mergeOption = 0; + break; } *lastRowKey = rowKey; } else { @@ -1754,7 +1765,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa colDataAppend(pColInfo, *curRow, NULL, true); } else if (tdValTypeIsNone(sVal.valType)) { // TODO: Set null if nothing append for this row - if (!isMerge) { + if (mergeOption == 1) { colDataAppend(pColInfo, *curRow, NULL, true); } } else { @@ -1769,14 +1780,14 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa ++k; } } else { - if (!isMerge) { + if (mergeOption == 1) { colDataAppend(pColInfo, *curRow, NULL, true); } ++i; } } - if (*lastRowKey != rowKey) { + if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); colDataAppend(pColInfo, *curRow, NULL, true); @@ -2008,7 +2019,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); // numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2065,7 +2076,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); // ++numOfRows; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2747,7 +2758,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int rv = TD_ROW_SVER(row); } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, - NULL, true, &lastRowKey); + NULL, pCfg->update, &lastRowKey); if (numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 49d02adc3a..bebdfb3b63 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -616,7 +616,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tcolId = pBlockCol->colId; toffset = tsdbGetBlockColOffset(pBlockCol); tlen = pBlockCol->len; - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; } else { ASSERT(pDataCol->colId == tcolId); TD_SET_COL_ROWS_NORM(pDataCol); @@ -624,17 +624,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((dcol != 0) && (pBlockCol->blen > 0)) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } if (tcolId == pDataCol->colId) { @@ -784,8 +775,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { // load the key row blockCol.colId = colId; - TD_SET_COL_ROWS_NORM(&blockCol); // default is NORM for the primary key column - blockCol.blen = 0; + blockCol.blen = 0; // default is NORM for the primary key column blockCol.len = pBlock->keyLen; blockCol.type = pDataCol->type; blockCol.offset = TSDB_KEY_COL_OFFSET; @@ -815,7 +805,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * ASSERT(pBlockCol->colId == pDataCol->colId); } // set the bitmap - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1; } @@ -833,17 +823,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if (!TD_COL_ROWS_NORM(pBlockCol)) { + if (pBlockCol->blen) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; From fdd901ae5db522a6d190778d654ea00f5650e9fd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 03:28:15 +0000 Subject: [PATCH 62/63] fix auto-create table problem --- source/dnode/vnode/src/vnd/vnodeSvr.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 811c5c10e4..403c02b440 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -533,6 +533,13 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } } + msgIter.uid = createTbReq.uid; + if (createTbReq.type == TSDB_CHILD_TABLE) { + msgIter.suid = createTbReq.ctb.suid; + } else { + msgIter.suid = 0; + } + tCoderClear(&coder); } From 3b73dc295f8d34341ff6ff820dca9161600b2a0d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 7 May 2022 11:53:08 +0800 Subject: [PATCH 63/63] set table uid --- source/dnode/vnode/src/tq/tqRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5ec3ab0b47..f531d3f5fb 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -173,7 +173,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t curRow = 0; tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - *pUid = pHandle->pBlock->uid; // set the uid of table for submit block + *pUid = pHandle->msgIter.uid; // set the uid of table for submit block while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row);