diff --git a/Jenkinsfile2 b/Jenkinsfile2
index 49e13b7831..d827c6640a 100644
--- a/Jenkinsfile2
+++ b/Jenkinsfile2
@@ -168,13 +168,6 @@ pipeline {
stages {
stage('run test') {
parallel {
- stage('windows test') {
- agent {label " windows11 "}
- steps {
- pre_test_win()
- pre_test_build_win()
- }
- }
stage('linux test') {
agent{label " slave3_0 || slave15 || slave16 || slave17 "}
options { skipDefaultCheckout() }
diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 11d0f34c45..e94be797b9 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -372,10 +372,9 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
// ----------------- Data column structure
// SDataCol arrangement: data => bitmap => dataOffset
typedef struct SDataCol {
- int8_t type; // column type
- uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
- uint8_t bitmapMode : 1; // default is 0(2 bits), otherwise 1(1 bit)
- uint8_t reserve : 6;
+ int8_t type; // column type
+ uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
+ uint8_t reserve : 7;
int16_t colId; // column ID
int32_t bytes; // column data bytes defined
int32_t offset; // data offset in a SDataRow (including the header size)
@@ -387,8 +386,6 @@ typedef struct SDataCol {
TSKEY ts; // only used in last NULL column
} SDataCol;
-
-
#define isAllRowsNull(pCol) ((pCol)->len == 0)
#define isAllRowsNone(pCol) ((pCol)->len == 0)
static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
@@ -482,7 +479,7 @@ void tdResetDataCols(SDataCols *pCols);
int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
SDataCols *tdFreeDataCols(SDataCols *pCols);
-int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer);
+int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer);
// ----------------- K-V data row structure
/* |<-------------------------------------- len -------------------------------------------->|
diff --git a/include/common/trow.h b/include/common/trow.h
index 8b1d612433..0d34c6e49f 100644
--- a/include/common/trow.h
+++ b/include/common/trow.h
@@ -42,7 +42,7 @@ extern "C" {
* @brief value type
* - for data from client input and STSRow in memory, 3 types of value none/null/norm available
*/
-#define TD_VTYPE_NORM 0x00U // normal val: not none, not null(no need assign value)
+#define TD_VTYPE_NORM 0x00U // normal val: not none, not null
#define TD_VTYPE_NULL 0x01U // null val
#define TD_VTYPE_NONE 0x02U // none or unknown/undefined
#define TD_VTYPE_MAX 0x03U //
@@ -140,8 +140,6 @@ typedef struct {
};
/// row total length
uint32_t len;
- /// row version
- uint64_t ver;
/// the inline data, maybe a tuple or a k-v tuple
char data[];
} STSRow;
@@ -176,7 +174,7 @@ typedef struct {
#define TD_ROW_DATA(r) ((r)->data)
#define TD_ROW_LEN(r) ((r)->len)
#define TD_ROW_KEY(r) ((r)->ts)
-#define TD_ROW_VER(r) ((r)->ver)
+// #define TD_ROW_VER(r) ((r)->ver)
#define TD_ROW_KEY_ADDR(r) (r)
// N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
@@ -241,13 +239,13 @@ static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colI
static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode);
bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode);
int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
- int8_t bitmapMode);
+ int8_t bitmapMode, bool isMerge);
static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset);
static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset,
col_id_t colId);
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols);
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge);
/**
* @brief
@@ -718,6 +716,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
+
return TSDB_CODE_SUCCESS;
}
diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h
index 0000972f5e..d59a7c23f7 100644
--- a/include/libs/function/tudf.h
+++ b/include/libs/function/tudf.h
@@ -29,7 +29,11 @@ extern "C" {
#endif
#define UDF_LISTEN_PIPE_NAME_LEN 32
-#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
+#ifdef _WIN32
+#define UDF_LISTEN_PIPE_NAME_PREFIX "\\\\?\\pipe\\udfd.sock"
+#else
+#define UDF_LISTEN_PIPE_NAME_PREFIX ".udfd.sock."
+#endif
#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
//======================================================================================
@@ -129,8 +133,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
// begin API to UDF writer.
// dynamic lib init and destroy
-typedef int32_t (*TUdfSetupFunc)();
-typedef int32_t (*TUdfTeardownFunc)();
+typedef int32_t (*TUdfInitFunc)();
+typedef int32_t (*TUdfDestroyFunc)();
//TODO: add API to check function arguments type, number etc.
@@ -242,7 +246,6 @@ static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRo
return 0;
}
-typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
diff --git a/include/libs/index/index.h b/include/libs/index/index.h
index c94d75338a..fa4cb1d2bd 100644
--- a/include/libs/index/index.h
+++ b/include/libs/index/index.h
@@ -56,7 +56,8 @@ typedef enum {
QUERY_LESS_EQUAL,
QUERY_GREATER_THAN,
QUERY_GREATER_EQUAL,
- QUERY_RANGE
+ QUERY_RANGE,
+ QUERY_MAX
} EIndexQueryType;
/*
@@ -178,8 +179,8 @@ void indexOptsDestroy(SIndexOpts* opts);
* @param:
*/
-SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType,
- const char* colName, int32_t nColName, const char* colVal, int32_t nColVal);
+SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName,
+ int32_t nColName, const char* colVal, int32_t nColVal);
void indexTermDestroy(SIndexTerm* p);
/*
diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h
index 0c7db45c4b..555274599a 100644
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@@ -91,6 +91,8 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
#ifdef __cplusplus
}
#endif
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index abc752955d..441e87eff7 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -621,6 +621,14 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639)
#define TSDB_CODE_PAR_INVALID_DROP_STABLE TAOS_DEF_ERROR_CODE(0, 0x263A)
#define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE TAOS_DEF_ERROR_CODE(0, 0x263B)
+#define TSDB_CODE_PAR_DUPLICATED_COLUMN TAOS_DEF_ERROR_CODE(0, 0x263C)
+#define TSDB_CODE_PAR_INVALID_TAGS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x263D)
+#define TSDB_CODE_PAR_INVALID_ROW_LENGTH TAOS_DEF_ERROR_CODE(0, 0x263E)
+#define TSDB_CODE_PAR_INVALID_COLUMNS_NUM TAOS_DEF_ERROR_CODE(0, 0x263F)
+#define TSDB_CODE_PAR_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x2640)
+#define TSDB_CODE_PAR_INVALID_FIRST_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2641)
+#define TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN TAOS_DEF_ERROR_CODE(0, 0x2642)
+#define TSDB_CODE_PAR_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x2643)
//planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
diff --git a/include/util/tdef.h b/include/util/tdef.h
index 316640797b..0ab277e0c3 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -29,6 +29,8 @@ extern "C" {
#define TSKEY_MAX (INT64_MAX - 1)
#define TSKEY_INITIAL_VAL TSKEY_MIN
+#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle
+
// Bytes for each type.
extern const int32_t TYPE_BYTES[15];
diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index 2874220121..fc5781cb4d 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -587,15 +587,34 @@ TEST(testCase, projection_query_tables) {
}
taos_free_result(pRes);
+ pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+ }
+ taos_free_result(pRes);
+
pRes = taos_query(pConn, "create table tu using st1 tags(1)");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
- for(int32_t i = 0; i < 10000; ++i) {
- char sql[512] = {0};
- sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
+ pRes = taos_query(pConn, "create table tu2 using st2 tags(1)");
+ if (taos_errno(pRes) != 0) {
+ printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+ }
+ taos_free_result(pRes);
+
+ for(int32_t i = 0; i < 10000000; i += 20) {
+ char sql[1024] = {0};
+ sprintf(sql,
+ "insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+ "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+ "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+ "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)",
+ i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7,
+ i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14,
+ i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19);
TAOS_RES* p = taos_query(pConn, sql);
if (taos_errno(p) != 0) {
printf("failed to insert data, reason:%s\n", taos_errstr(p));
@@ -604,24 +623,44 @@ TEST(testCase, projection_query_tables) {
taos_free_result(p);
}
- pRes = taos_query(pConn, "select * from tu");
- if (taos_errno(pRes) != 0) {
- printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
- taos_free_result(pRes);
- ASSERT_TRUE(false);
+ printf("start to insert next table\n");
+
+ for(int32_t i = 0; i < 10000000; i += 20) {
+ char sql[1024] = {0};
+ sprintf(sql,
+ "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+ "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+ "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
+ "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)",
+ i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7,
+ i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14,
+ i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19);
+ TAOS_RES* p = taos_query(pConn, sql);
+ if (taos_errno(p) != 0) {
+ printf("failed to insert data, reason:%s\n", taos_errstr(p));
+ }
+
+ taos_free_result(p);
}
- TAOS_ROW pRow = NULL;
- TAOS_FIELD* pFields = taos_fetch_fields(pRes);
- int32_t numOfFields = taos_num_fields(pRes);
+// pRes = taos_query(pConn, "select * from tu");
+// if (taos_errno(pRes) != 0) {
+// printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
+// taos_free_result(pRes);
+// ASSERT_TRUE(false);
+// }
- char str[512] = {0};
- while ((pRow = taos_fetch_row(pRes)) != NULL) {
- int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
- printf("%s\n", str);
- }
+// TAOS_ROW pRow = NULL;
+// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+// int32_t numOfFields = taos_num_fields(pRes);
+//
+// char str[512] = {0};
+// while ((pRow = taos_fetch_row(pRes)) != NULL) {
+// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+// printf("%s\n", str);
+// }
- taos_free_result(pRes);
+// taos_free_result(pRes);
taos_close(pConn);
}
@@ -659,7 +698,7 @@ TEST(testCase, agg_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
- TAOS_RES* pRes = taos_query(pConn, "use db");
+ TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 4f4b8b196f..2c17f2c2fc 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -363,9 +363,9 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd
for (int32_t i = 0; i < pDest->info.numOfCols; ++i) {
int32_t mapIndex = i;
- if (pIndexMap) {
- mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i);
- }
+// if (pIndexMap) {
+// mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i);
+// }
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex);
@@ -493,12 +493,12 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) {
bool isNull = false;
if (pBlock->pBlockAgg == NULL) {
- isNull = colDataIsNull(pColData, pBlock->info.rows, j, NULL);
+ isNull = colDataIsNull_s(pColData, pBlock->info.rows);
} else {
isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]);
}
- char* p = colDataGetData(pColData, j);
+ char* p = colDataGetData(pColData, j);
colDataAppend(pDstCol, j - startIndex, p, isNull);
}
}
@@ -1478,11 +1478,11 @@ void blockDebugShowData(const SArray* dataBlocks) {
* @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in
* SDataBlock->info.uid
* @param suid // TODO: check with Liao whether suid response is reasonable
- *
+ *
* TODO: colId should be set
*/
-int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid,
- tb_uid_t suid) {
+int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
+ tb_uid_t uid, tb_uid_t suid) {
int32_t sz = taosArrayGetSize(pDataBlocks);
int32_t bufSize = sizeof(SSubmitReq);
for (int32_t i = 0; i < sz; ++i) {
@@ -1494,16 +1494,16 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
ASSERT(bufSize < 3 * 1024 * 1024);
*pReq = taosMemoryCalloc(1, bufSize);
- if(!(*pReq)) {
+ if (!(*pReq)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
void* pDataBuf = *pReq;
- int32_t msgLen = sizeof(SSubmitReq);
+ int32_t msgLen = sizeof(SSubmitReq);
int32_t numOfBlks = 0;
SRowBuilder rb = {0};
- tdSRowInit(&rb, 0); // TODO: use the latest version
+ tdSRowInit(&rb, 0); // TODO: use the latest version
for (int32_t i = 0; i < sz; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
@@ -1511,8 +1511,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
int32_t rows = pDataBlock->info.rows;
int32_t rowSize = pDataBlock->info.rowSize;
int64_t groupId = pDataBlock->info.groupId;
-
- if(rb.nCols != colNum) {
+
+ if (rb.nCols != colNum) {
tdSRowSetTpInfo(&rb, colNum, pTSchema->flen);
}
@@ -1525,10 +1525,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
msgLen += sizeof(SSubmitBlk);
int32_t dataLen = 0;
- for (int32_t j = 0; j < rows; ++j) { // iterate by row
- tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf
+ for (int32_t j = 0; j < rows; ++j) { // iterate by row
+ tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf
printf("|");
- bool isStartKey = false;
+ bool isStartKey = false;
for (int32_t k = 0; k < colNum; ++k) { // iterate by column
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
@@ -1536,7 +1536,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
case TSDB_DATA_TYPE_TIMESTAMP:
if (!isStartKey) {
isStartKey = true;
- tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0);
+ tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true,
+ 0, 0);
} else {
tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k);
break;
@@ -1629,14 +1630,14 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) {
blkHead->uid = htobe64(pDataBlock->info.uid);
int32_t rows = pDataBlock->info.rows;
- int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+ /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/
/*blkHead->dataLen = htonl(rows * maxLen);*/
blkHead->dataLen = 0;
void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk));
STSRow* rowData = blockData;
- for (int32_t j = 0; j < pDataBlock->info.rows; j++) {
+ for (int32_t j = 0; j < rows; j++) {
SRowBuilder rb = {0};
tdSRowInit(&rb, pTSchema->version);
tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen);
diff --git a/source/common/src/trow.c b/source/common/src/trow.c
index 7157c1e0f0..50e51a40dd 100644
--- a/source/common/src/trow.c
+++ b/source/common/src/trow.c
@@ -24,7 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{
},
{
// 1 bit
- TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I,
+ TD_VTYPE_NORM_BYTE_I, // normal
+ TD_VTYPE_NULL_BYTE_I,
TD_VTYPE_NULL_BYTE_I, // padding
}
@@ -33,6 +34,24 @@ const uint8_t tdVTypeByte[2][3] = {{
// declaration
static uint8_t tdGetBitmapByte(uint8_t byte);
+// static void dataColSetNEleNull(SDataCol *pCol, int nEle);
+
+/**
+ * @brief src2 data has more priority than src1
+ *
+ * @param target
+ * @param src1
+ * @param iter1
+ * @param limit1
+ * @param src2
+ * @param iter2
+ * @param limit2
+ * @param tRows
+ * @param update
+ */
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+ int limit2, int tRows, bool update);
+
// implementation
/**
* @brief Compress bitmap bytes comprised of 2-bits to counterpart of 1-bit.
@@ -229,23 +248,23 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) {
void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) {
int32_t i = 0, j = 0;
int32_t nBytes = TD_BITMAP_BYTES(nBits);
- int32_t nStrictBytes = nBits / 4;
- int32_t nPartialBits = nBits - nStrictBytes * 4;
+ int32_t nRoundBytes = nBits / 4;
+ int32_t nRemainderBits = nBits - nRoundBytes * 4;
- switch (nPartialBits) {
+ switch (nRemainderBits) {
case 0:
// NOTHING TODO
break;
case 1: {
- void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+ void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes);
*(uint8_t *)lastByte &= 0xC0;
} break;
case 2: {
- void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+ void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes);
*(uint8_t *)lastByte &= 0xF0;
} break;
case 3: {
- void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes);
+ void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes);
*(uint8_t *)lastByte &= 0xFC;
} break;
default:
@@ -266,10 +285,6 @@ void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) {
}
}
-// static void dataColSetNEleNull(SDataCol *pCol, int nEle);
-static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
- int limit2, int tRows, bool forceSetNull);
-
static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->dataOff[index] = pCol->len;
@@ -329,7 +344,7 @@ bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode
if (*((uint8_t *)pBitmap) != vTypeByte) {
return false;
}
- pBitmap = POINTER_SHIFT(pBitmap, 1);
+ pBitmap = POINTER_SHIFT(pBitmap, i);
}
int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
@@ -410,11 +425,12 @@ STSRow *tdRowDup(STSRow *row) {
* @param val
* @param numOfRows
* @param maxPoints
- * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit)
+ * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit)
+ * @param isMerge merge to current row
* @return int
*/
int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints,
- int8_t bitmapMode) {
+ int8_t bitmapMode, bool isMerge) {
TASSERT(pCol != NULL);
// Assume that the columns not specified during insert/upsert mean None.
@@ -430,33 +446,58 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int
dataColSetNEleNone(pCol, numOfRows, bitmapMode);
}
}
- if (!tdValTypeIsNorm(valType)) {
+ const void *value = val;
+ if (!tdValTypeIsNorm(valType) || !val) {
// TODO:
// 1. back compatibility and easy to debug with codes of 2.0 to save NULL values.
// 2. later on, considering further optimization, don't save Null/None for VarType.
- val = getNullValue(pCol->type);
+ value = getNullValue(pCol->type);
}
- if (IS_VAR_DATA_TYPE(pCol->type)) {
- // set offset
- pCol->dataOff[numOfRows] = pCol->len;
- // Copy data
- memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val));
- // Update the length
- pCol->len += varDataTLen(val);
- } else {
- ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
- memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes);
- pCol->len += pCol->bytes;
+ if (!isMerge) {
+ if (IS_VAR_DATA_TYPE(pCol->type)) {
+ // set offset
+ pCol->dataOff[numOfRows] = pCol->len;
+ // Copy data
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+ // Update the length
+ pCol->len += varDataTLen(value);
+ } else {
+ ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
+ pCol->len += pCol->bytes;
+ }
+ } else if (!tdValTypeIsNone(valType)) {
+ if (IS_VAR_DATA_TYPE(pCol->type)) {
+ // keep the last offset
+ // discard the last var data
+ int32_t lastVarLen = varDataTLen(POINTER_SHIFT(pCol->pData, pCol->dataOff[numOfRows]));
+ pCol->len -= lastVarLen;
+ // Copy data
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
+ // Update the length
+ pCol->len += varDataTLen(value);
+ } else {
+ ASSERT(pCol->len - TYPE_BYTES[pCol->type] == TYPE_BYTES[pCol->type] * numOfRows);
+ memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), value, pCol->bytes);
+ }
}
+
#ifdef TD_SUPPORT_BITMAP
- tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode);
+ if (!isMerge || !tdValTypeIsNone(valType)) {
+ tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode);
+ }
#endif
return 0;
}
// internal
-static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) {
+#if 0
ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow));
+#endif
+
+ // Multi-Version rows with the same key and different versions supported
+ ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) <= TD_ROW_KEY(pRow));
int rcol = 1;
int dcol = 1;
@@ -464,12 +505,14 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
SDataCol *pDataCol = &(pCols->cols[0]);
ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
- tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
while (dcol < pCols->numOfCols) {
pDataCol = &(pCols->cols[dcol]);
if (rcol >= schemaNCols(pSchema)) {
- tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
++dcol;
continue;
}
@@ -480,22 +523,26 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) {
return terrno;
}
- tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
++dcol;
++rcol;
} else if (pRowCol->colId < pDataCol->colId) {
++rcol;
} else {
- tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
++dcol;
}
}
+#if 0
++pCols->numOfRows;
+#endif
return TSDB_CODE_SUCCESS;
}
// internal
-static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) {
ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow));
int rcol = 0;
@@ -506,12 +553,14 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
SDataCol *pDataCol = &(pCols->cols[0]);
ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
- tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
while (dcol < pCols->numOfCols) {
pDataCol = &(pCols->cols[dcol]);
if (rcol >= tRowCols || rcol >= tSchemaCols) {
- tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
++dcol;
continue;
}
@@ -527,17 +576,21 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) {
return terrno;
}
- tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
++dcol;
++rcol;
} else if (pIdx->colId < pDataCol->colId) {
++rcol;
} else {
- tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
+ tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode,
+ isMerge);
++dcol;
}
}
+#if 0
++pCols->numOfRows;
+#endif
return TSDB_CODE_SUCCESS;
}
@@ -548,20 +601,30 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
* @param pRow
* @param pSchema
* @param pCols
- * @param forceSetNull
*/
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) {
if (TD_IS_TP_ROW(pRow)) {
- return tdAppendTpRowToDataCol(pRow, pSchema, pCols);
+ return tdAppendTpRowToDataCol(pRow, pSchema, pCols, isMerge);
} else if (TD_IS_KV_ROW(pRow)) {
- return tdAppendKvRowToDataCol(pRow, pSchema, pCols);
+ return tdAppendKvRowToDataCol(pRow, pSchema, pCols, isMerge);
} else {
ASSERT(0);
}
return TSDB_CODE_SUCCESS;
}
-int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull,
+/**
+ * @brief source data has more priority than target
+ *
+ * @param target
+ * @param source
+ * @param rowsToMerge
+ * @param pOffset
+ * @param update
+ * @param maxVer
+ * @return int
+ */
+int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool update,
TDRowVerT maxVer) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
ASSERT(target->numOfCols == source->numOfCols);
@@ -576,17 +639,38 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *
if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
// TODO: filter the maxVer
- for (int i = 0; i < rowsToMerge; i++) {
+ TSKEY lastKey = TSKEY_INITIAL_VAL;
+ for (int i = 0; i < rowsToMerge; ++i) {
+ bool merge = false;
for (int j = 0; j < source->numOfCols; j++) {
if (source->cols[j].len > 0 || target->cols[j].len > 0) {
SCellVal sVal = {0};
if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) {
TASSERT(0);
}
+
+ if (j == 0) {
+ if (lastKey == *(TSKEY *)sVal.val) {
+ if (!update) {
+ break;
+ }
+ merge = true;
+ } else if (lastKey != TSKEY_INITIAL_VAL) {
+ ++target->numOfRows;
+ }
+
+ lastKey = *(TSKEY *)sVal.val;
+ }
+ if (i == 0) {
+ (target->cols + j)->bitmap = (source->cols + j)->bitmap;
+ }
+
tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
- target->bitmapMode);
+ target->bitmapMode, merge);
}
}
+ }
+ if (lastKey != TSKEY_INITIAL_VAL) {
++target->numOfRows;
}
(*pOffset) += rowsToMerge;
@@ -596,7 +680,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *
int iter1 = 0;
tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows,
- pTarget->numOfRows + rowsToMerge, forceSetNull);
+ pTarget->numOfRows + rowsToMerge, update);
}
tdFreeDataCols(pTarget);
@@ -607,67 +691,95 @@ _err:
return -1;
}
-// src2 data has more priority than src1
+static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, bool isMerge) {
+ for (int i = 0; i < src->numOfCols; ++i) {
+ ASSERT(target->cols[i].type == src->cols[i].type);
+ if (src->cols[i].len > 0 || target->cols[i].len > 0) {
+ SCellVal sVal = {0};
+ if (tdGetColDataOfRow(&sVal, src->cols + i, iter, src->bitmapMode) < 0) {
+ TASSERT(0);
+ }
+ if (isMerge) {
+ if (!tdValTypeIsNone(sVal.valType)) {
+ tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+ target->bitmapMode, isMerge);
+ } else {
+ // Keep the origin value for None
+ }
+ } else {
+ tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
+ target->bitmapMode, isMerge);
+ }
+ }
+ }
+}
+/**
+ * @brief src2 data has more priority than src1
+ *
+ * @param target
+ * @param src1
+ * @param iter1
+ * @param limit1
+ * @param src2
+ * @param iter2
+ * @param limit2
+ * @param tRows
+ * @param update
+ */
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
- int limit2, int tRows, bool forceSetNull) {
+ int limit2, int tRows, bool update) {
tdResetDataCols(target);
+ target->bitmapMode = src1->bitmapMode;
ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
+ int32_t nRows = 0;
- while (target->numOfRows < tRows) {
+ // TODO: filter the maxVer
+ // TODO: handle the delete function
+ TSKEY lastKey = TSKEY_INITIAL_VAL;
+ while (nRows < tRows) {
if (*iter1 >= limit1 && *iter2 >= limit2) break;
TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
- TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
+ // TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
// TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
- ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
- // TODO: filter the maxVer
- if (key1 < key2) {
- for (int i = 0; i < src1->numOfCols; ++i) {
- ASSERT(target->cols[i].type == src1->cols[i].type);
- if (src1->cols[i].len > 0 || target->cols[i].len > 0) {
- SCellVal sVal = {0};
- if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
- TASSERT(0);
- }
- tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
- target->bitmapMode);
- }
- }
+ // ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
- ++target->numOfRows;
+ if (key1 <= key2) {
+ // select key1 if not delete
+ if (update && (lastKey == key1)) {
+ tdAppendValToDataCols(target, src1, *iter1, true);
+ } else if (lastKey != key1) {
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++target->numOfRows;
+ }
+ tdAppendValToDataCols(target, src1, *iter1, false);
+ }
+ ++nRows;
++(*iter1);
- } else if (key1 >= key2) {
- // TODO: filter the maxVer
- if ((key1 > key2) || ((key1 == key2) && !TKEY_IS_DELETED(key2))) {
- for (int i = 0; i < src2->numOfCols; ++i) {
- SCellVal sVal = {0};
- ASSERT(target->cols[i].type == src2->cols[i].type);
- if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) {
- TASSERT(0);
- }
- if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) {
- tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
- target->bitmapMode);
- } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
- if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
- TASSERT(0);
- }
- tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints,
- target->bitmapMode);
- } else if (target->cols[i].len > 0) {
- dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode);
- }
+ lastKey = key1;
+ } else {
+ // use key2 if not deleted
+ // TODO: handle the delete function
+ if (update && (lastKey == key2)) {
+ tdAppendValToDataCols(target, src2, *iter2, true);
+ } else if (lastKey != key2) {
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++target->numOfRows;
}
- ++target->numOfRows;
+ tdAppendValToDataCols(target, src2, *iter2, false);
}
+ ++nRows;
++(*iter2);
- if (key1 == key2) ++(*iter1);
+ lastKey = key2;
}
- ASSERT(target->numOfRows <= target->maxPoints);
+ ASSERT(target->numOfRows <= target->maxPoints - 1);
+ }
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++target->numOfRows;
}
}
@@ -777,7 +889,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->sversion = pDataCols->sversion;
if (keepData) pRet->numOfRows = pDataCols->numOfRows;
- for (int i = 0; i < pDataCols->numOfCols; i++) {
+ for (int i = 0; i < pDataCols->numOfCols; ++i) {
pRet->cols[i].type = pDataCols->cols[i].type;
pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
pRet->cols[i].colId = pDataCols->cols[i].colId;
@@ -797,8 +909,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
}
if (!TD_COL_ROWS_NORM(pRet->cols + i)) {
- int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows);
- memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes);
+ memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, TD_BITMAP_BYTES(pDataCols->numOfRows));
}
}
}
diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c
index e3d67cd488..3fd2ef8e73 100644
--- a/source/common/src/ttypes.c
+++ b/source/common/src/ttypes.c
@@ -557,7 +557,7 @@ static const void *nullValues[] = {
};
const void *getNullValue(int32_t type) {
- assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT);
+ assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); // TODO: extend the types
return nullValues[type - 1];
}
diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h
index a415d39434..28d3215b98 100644
--- a/source/dnode/mnode/impl/inc/mndStb.h
+++ b/source/dnode/mnode/impl/inc/mndStb.h
@@ -31,6 +31,11 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t n
int32_t *pRspLen);
int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs);
+int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate);
+SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName);
+int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb);
+int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c
index b9331e6e03..3ac2951b6f 100644
--- a/source/dnode/mnode/impl/src/mndFunc.c
+++ b/source/dnode/mnode/impl/src/mndFunc.c
@@ -314,7 +314,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) {
goto _OVER;
}
- if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
+ if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
goto _OVER;
}
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 8ae0d5d19c..cd76c7c8bb 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -342,7 +342,7 @@ void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) {
sdbRelease(pSdb, pStb);
}
-static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) {
+SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) {
SName name = {0};
tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@@ -463,7 +463,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb,
return pHead;
}
-static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
+int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
if (pCreate->igExists < 0 || pCreate->igExists > 1) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
@@ -634,91 +634,96 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) {
return NULL;
}
-static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
- SStbObj stbObj = {0};
- memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
- memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN);
- stbObj.createdTime = taosGetTimestampMs();
- stbObj.updateTime = stbObj.createdTime;
- stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
- stbObj.dbUid = pDb->uid;
- stbObj.version = 1;
- stbObj.nextColId = 1;
- stbObj.xFilesFactor = pCreate->xFilesFactor;
- stbObj.delay = pCreate->delay;
- stbObj.ttl = pCreate->ttl;
- stbObj.numOfColumns = pCreate->numOfColumns;
- stbObj.numOfTags = pCreate->numOfTags;
- stbObj.commentLen = pCreate->commentLen;
- if (stbObj.commentLen > 0) {
- stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1);
- if (stbObj.comment == NULL) {
+int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) {
+ memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN);
+ memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
+ pDst->createdTime = taosGetTimestampMs();
+ pDst->updateTime = pDst->createdTime;
+ pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+ pDst->dbUid = pDb->uid;
+ pDst->version = 1;
+ pDst->nextColId = 1;
+ pDst->xFilesFactor = pCreate->xFilesFactor;
+ pDst->delay = pCreate->delay;
+ pDst->ttl = pCreate->ttl;
+ pDst->numOfColumns = pCreate->numOfColumns;
+ pDst->numOfTags = pCreate->numOfTags;
+ pDst->commentLen = pCreate->commentLen;
+ if (pDst->commentLen > 0) {
+ pDst->comment = taosMemoryCalloc(pDst->commentLen, 1);
+ if (pDst->comment == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
- memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen);
+ memcpy(pDst->comment, pCreate->comment, pDst->commentLen);
}
- stbObj.ast1Len = pCreate->ast1Len;
- if (stbObj.ast1Len > 0) {
- stbObj.pAst1 = taosMemoryCalloc(stbObj.ast1Len, 1);
- if (stbObj.pAst1 == NULL) {
+ pDst->ast1Len = pCreate->ast1Len;
+ if (pDst->ast1Len > 0) {
+ pDst->pAst1 = taosMemoryCalloc(pDst->ast1Len, 1);
+ if (pDst->pAst1 == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
- memcpy(stbObj.pAst1, pCreate->pAst1, stbObj.ast1Len);
+ memcpy(pDst->pAst1, pCreate->pAst1, pDst->ast1Len);
}
- stbObj.ast2Len = pCreate->ast2Len;
- if (stbObj.ast2Len > 0) {
- stbObj.pAst2 = taosMemoryCalloc(stbObj.ast2Len, 1);
- if (stbObj.pAst2 == NULL) {
+ pDst->ast2Len = pCreate->ast2Len;
+ if (pDst->ast2Len > 0) {
+ pDst->pAst2 = taosMemoryCalloc(pDst->ast2Len, 1);
+ if (pDst->pAst2 == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
- memcpy(stbObj.pAst2, pCreate->pAst2, stbObj.ast2Len);
+ memcpy(pDst->pAst2, pCreate->pAst2, pDst->ast2Len);
}
- stbObj.pColumns = taosMemoryCalloc(1, stbObj.numOfColumns * sizeof(SSchema));
- stbObj.pTags = taosMemoryCalloc(1, stbObj.numOfTags * sizeof(SSchema));
- if (stbObj.pColumns == NULL || stbObj.pTags == NULL) {
+ pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema));
+ pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema));
+ if (pDst->pColumns == NULL || pDst->pTags == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
- for (int32_t i = 0; i < stbObj.numOfColumns; ++i) {
+ for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
SField *pField = taosArrayGet(pCreate->pColumns, i);
- SSchema *pSchema = &stbObj.pColumns[i];
+ SSchema *pSchema = &pDst->pColumns[i];
pSchema->type = pField->type;
pSchema->bytes = pField->bytes;
pSchema->flags = pField->flags;
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
- pSchema->colId = stbObj.nextColId;
- stbObj.nextColId++;
+ pSchema->colId = pDst->nextColId;
+ pDst->nextColId++;
}
- for (int32_t i = 0; i < stbObj.numOfTags; ++i) {
+ for (int32_t i = 0; i < pDst->numOfTags; ++i) {
SField *pField = taosArrayGet(pCreate->pTags, i);
- SSchema *pSchema = &stbObj.pTags[i];
+ SSchema *pSchema = &pDst->pTags[i];
pSchema->type = pField->type;
pSchema->bytes = pField->bytes;
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
- pSchema->colId = stbObj.nextColId;
- stbObj.nextColId++;
+ pSchema->colId = pDst->nextColId;
+ pDst->nextColId++;
}
+ return 0;
+}
+
+static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
+ SStbObj stbObj = {0};
int32_t code = -1;
+
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg);
if (pTrans == NULL) goto _OVER;
mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
- mndTransSetDbInfo(pTrans, pDb);
- if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
- if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
- if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
- if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
- if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
+ if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) {
+ goto _OVER;
+ }
+
+ if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
+
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
@@ -727,6 +732,15 @@ _OVER:
mndTransDrop(pTrans);
return code;
}
+int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+ mndTransSetDbInfo(pTrans, pDb);
+ if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
+ if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
+ if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
+ if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
+ if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
+ return 0;
+}
static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) {
SMnode *pMnode = pReq->pNode;
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 5c6e2ce771..1404b1cd94 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -290,6 +290,72 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
return 0;
}
+static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) {
+ SStbObj *pStb = NULL;
+ SDbObj *pDb = NULL;
+ SUserObj *pUser = NULL;
+
+ SMCreateStbReq createReq = {0};
+ tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN);
+ createReq.numOfColumns = pStream->outputSchema.nCols;
+ createReq.numOfTags = 1; // group id
+ createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
+ // build fields
+ // build tags
+
+ if (mndCheckCreateStbReq(&createReq) != 0) {
+ terrno = TSDB_CODE_INVALID_MSG;
+ goto _OVER;
+ }
+
+ pStb = mndAcquireStb(pMnode, createReq.name);
+ if (pStb != NULL) {
+ terrno = TSDB_CODE_MND_STB_ALREADY_EXIST;
+ goto _OVER;
+ }
+
+ pDb = mndAcquireDbByStb(pMnode, createReq.name);
+ if (pDb == NULL) {
+ terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+ goto _OVER;
+ }
+
+ pUser = mndAcquireUser(pMnode, user);
+ if (pUser == NULL) {
+ goto _OVER;
+ }
+
+ if (mndCheckWriteAuth(pUser, pDb) != 0) {
+ goto _OVER;
+ }
+
+ int32_t numOfStbs = -1;
+ mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs);
+ if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) {
+ terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB;
+ goto _OVER;
+ }
+
+ SStbObj stbObj = {0};
+
+ if (mndBuildStbFromReq(pMnode, &stbObj, &createReq, pDb) != 0) {
+ goto _OVER;
+ }
+
+ if (mndBuildStbFromReq(pMnode, pStb, &createReq, pDb) != 0) {
+ goto _OVER;
+ }
+
+ if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
+
+ return pStb;
+_OVER:
+ mndReleaseStb(pMnode, pStb);
+ mndReleaseDb(pMnode, pDb);
+ mndReleaseUser(pMnode, pUser);
+ return NULL;
+}
+
static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) {
mDebug("stream:%s to create", pCreate->name);
SStreamObj streamObj = {0};
@@ -311,13 +377,21 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
streamObj.trigger = pCreate->triggerType;
streamObj.waterMark = pCreate->watermark;
- STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
+ STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
if (pTrans == NULL) {
mError("stream:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
+#if 0
+ if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) {
+ mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
+ mndTransDrop(pTrans);
+ return -1;
+ }
+#endif
+
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) {
mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp
index 0473fa375e..c8f832160b 100644
--- a/source/dnode/mnode/impl/test/func/func.cpp
+++ b/source/dnode/mnode/impl/test/func/func.cpp
@@ -24,6 +24,7 @@ class MndTestFunc : public ::testing::Test {
void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size);
void SetComment(SCreateFuncReq* pReq, const char* pComment);
+ void SetBufSize(SCreateFuncReq* pReq, int32_t size);
};
Testbase MndTestFunc::test;
@@ -40,6 +41,10 @@ void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) {
strcpy(pReq->pComment, pComment);
}
+void MndTestFunc::SetBufSize(SCreateFuncReq* pReq, int32_t size) {
+ pReq->bufSize = size;
+}
+
TEST_F(MndTestFunc, 01_Show_Func) {
test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", "");
EXPECT_EQ(test.GetShowRows(), 0);
@@ -96,6 +101,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
+ SetBufSize(&createReq, -1);
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 62deb200c9..488edc4f25 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -126,7 +126,7 @@ int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList
int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle);
-int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows,
+int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t* pUid, int32_t *pNumOfRows,
int16_t *pNumOfCols);
// need to reposition
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 2716692bd7..d84c1b1d07 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -273,7 +273,8 @@ typedef enum {
typedef struct {
uint8_t last : 1;
- uint8_t blkVer : 7;
+ uint8_t hasDupKey : 1; // 0: no dup TS key, 1: has dup TS key(since supporting Multi-Version)
+ uint8_t blkVer : 6;
uint8_t numOfSubBlocks;
col_id_t numOfCols; // not including timestamp column
uint32_t len; // data block length
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 261b4aa8ae..ffd76af5e2 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -161,7 +161,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
- if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
+ if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
&block.info.numOfCols) < 0) {
ASSERT(0);
}
@@ -540,7 +540,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
tqReadHandleSetMsg(pReader, pCont, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
- if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
+ if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
&block.info.numOfCols) < 0) {
ASSERT(0);
}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 511d57ed58..5ec3ab0b47 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -84,10 +84,12 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
return false;
}
-int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows,
+int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, int32_t* pNumOfRows,
int16_t* pNumOfCols) {
/*int32_t sversion = pHandle->pBlock->sversion;*/
// TODO set to real sversion
+ *pUid = 0;
+
int32_t sversion = 0;
if (pHandle->sver != sversion) {
pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
@@ -169,7 +171,10 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
int32_t curRow = 0;
+
tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
+ *pUid = pHandle->pBlock->uid; // set the uid of table for submit block
+
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
tdSTSRowIterReset(&iter, row);
// get all wanted col of that block
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 97b796c6a5..a008c4a0e5 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -1330,13 +1330,15 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
TSKEY maxKey, int maxRows, int8_t update) {
TSKEY key1 = INT64_MAX;
TSKEY key2 = INT64_MAX;
+ TSKEY lastKey = TSKEY_INITIAL_VAL;
STSchema *pSchema = NULL;
ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey);
tdResetDataCols(pTarget);
pTarget->bitmapMode = pDataCols->bitmapMode;
-
+ // TODO: filter Multi-Version
+ // TODO: support delete function
while (true) {
key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
STSRow *row = tsdbNextIterRow(pCommitIter->pIter);
@@ -1349,6 +1351,9 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
if (key1 == INT64_MAX && key2 == INT64_MAX) break;
if (key1 < key2) {
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++pTarget->numOfRows;
+ }
for (int i = 0; i < pDataCols->numOfCols; ++i) {
// TODO: dataColAppendVal may fail
SCellVal sVal = {0};
@@ -1356,10 +1361,10 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
TASSERT(0);
}
tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints,
- pTarget->bitmapMode);
+ pTarget->bitmapMode, false);
}
- ++pTarget->numOfRows;
+ lastKey = key1;
++(*iter);
} else if (key1 > key2) {
if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
@@ -1367,7 +1372,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
ASSERT(pSchema != NULL);
}
- tdAppendSTSRowToDataCol(row, pSchema, pTarget);
+ if (key2 == lastKey) {
+ if (TD_SUPPORT_UPDATE(update)) {
+ tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
+ }
+ } else {
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++pTarget->numOfRows;
+ }
+ tdAppendSTSRowToDataCol(row, pSchema, pTarget, false);
+ lastKey = key2;
+ }
tSkipListIterNext(pCommitIter->pIter);
} else {
@@ -1397,6 +1412,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
++(*iter);
tSkipListIterNext(pCommitIter->pIter);
#endif
+
+ if(lastKey != key1) {
+ lastKey = key1;
+ ++pTarget->numOfRows;
+ }
+
// copy disk data
for (int i = 0; i < pDataCols->numOfCols; ++i) {
SCellVal sVal = {0};
@@ -1405,7 +1426,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
}
// TODO: tdAppendValToDataCol may fail
tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints,
- pTarget->bitmapMode);
+ pTarget->bitmapMode, false);
}
if (TD_SUPPORT_UPDATE(update)) {
@@ -1416,26 +1437,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
}
// TODO: merge with Multi-Version
- STSRow *curRow = row;
-
- ++(*iter);
- tSkipListIterNext(pCommitIter->pIter);
- STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter);
-
- if (key2 < TD_ROW_KEY(nextRow)) {
- tdAppendSTSRowToDataCol(row, pSchema, pTarget);
- } else {
- tdAppendSTSRowToDataCol(row, pSchema, pTarget);
- }
- // TODO: merge with Multi-Version
- } else {
- ++pTarget->numOfRows;
- ++(*iter);
- tSkipListIterNext(pCommitIter->pIter);
+ tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
}
+ ++(*iter);
+ tSkipListIterNext(pCommitIter->pIter);
}
- if (pTarget->numOfRows >= maxRows) break;
+ if (pTarget->numOfRows >= (maxRows - 1)) break;
+ }
+
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++pTarget->numOfRows;
}
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index b52a34cfb2..48d32c0dd3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -20,7 +20,7 @@ static void tsdbFreeTbData(STbData *pTbData);
static char *tsdbGetTsTupleKey(const void *data);
static int tsdbTbDataComp(const void *arg1, const void *arg2);
static char *tsdbTbDataGetUid(const void *arg);
-static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row);
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge);
int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) {
STsdbMemTable *pMemTable;
@@ -82,14 +82,19 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
if (pIter == NULL) return 0;
- STSchema *pSchema = NULL;
- TSKEY rowKey = 0;
- TSKEY fKey = 0;
+ STSchema *pSchema = NULL;
+ TSKEY rowKey = 0;
+ TSKEY fKey = 0;
+ // only fetch lastKey from mem data as file data not used in this function actually
+ TSKEY lastKey = TSKEY_INITIAL_VAL;
bool isRowDel = false;
int filterIter = 0;
STSRow *row = NULL;
SMergeInfo mInfo;
+ // TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
+ // query handle)
+
if (pMergeInfo == NULL) pMergeInfo = &mInfo;
memset(pMergeInfo, 0, sizeof(*pMergeInfo));
@@ -111,7 +116,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
-
+ // 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
+ // 2. rowKey - would dup since Multi-Version supported
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
@@ -125,12 +131,14 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
+#if 0
} else if (fKey > rowKey) {
if (isRowDel) {
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
@@ -185,6 +193,94 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
fKey = tdGetKey(filterKeys[filterIter]);
}
}
+#endif
+#if 1
+ } else if (fKey > rowKey) {
+ if (isRowDel) {
+ // TODO: support delete function
+ pMergeInfo->rowsDeleteFailed++;
+ } else {
+ if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+
+ if (lastKey != rowKey) {
+ pMergeInfo->rowsInserted++;
+ pMergeInfo->nOperations++;
+ pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
+ pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
+ if (pCols) {
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++pCols->numOfRows;
+ }
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false);
+ }
+ lastKey = rowKey;
+ } else {
+ if (keepDup) {
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true);
+ } else {
+ // discard
+ }
+ }
+ }
+
+ tSkipListIterNext(pIter);
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || TD_ROW_KEY(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = TD_ROW_KEY(row);
+ isRowDel = TD_ROW_IS_DELETED(row);
+ }
+ } else { // fkey == rowKey
+ if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
+ ASSERT(!keepDup);
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ pMergeInfo->rowsDeleteSucceed++;
+ pMergeInfo->nOperations++;
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false);
+ } else {
+ if (keepDup) {
+ if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+ if (lastKey != rowKey) {
+ pMergeInfo->rowsUpdated++;
+ pMergeInfo->nOperations++;
+ pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
+ pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
+ lastKey = rowKey;
+ ++pCols->numOfRows;
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false);
+ } else {
+ tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true);
+ }
+ } else {
+ pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
+ pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
+ }
+ }
+
+ tSkipListIterNext(pIter);
+ row = tsdbNextIterRow(pIter);
+ if (row == NULL || TD_ROW_KEY(row) > maxKey) {
+ rowKey = INT64_MAX;
+ isRowDel = false;
+ } else {
+ rowKey = TD_ROW_KEY(row);
+ isRowDel = TD_ROW_IS_DELETED(row);
+ }
+
+ filterIter++;
+ if (filterIter >= nFilterKeys) {
+ fKey = INT64_MAX;
+ } else {
+ fKey = tdGetKey(filterKeys[filterIter]);
+ }
+ }
+#endif
+ }
+ if (lastKey != TSKEY_INITIAL_VAL) {
+ ++pCols->numOfRows;
}
return 0;
@@ -254,9 +350,12 @@ static STbData *tsdbNewTbData(tb_uid_t uid) {
pTbData->keyMin = TSKEY_MAX;
pTbData->keyMax = TSKEY_MIN;
pTbData->nrows = 0;
-
+#if 0
pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY,
tsdbGetTsTupleKey);
+#endif
+ pTbData->pData =
+ tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, tsdbGetTsTupleKey);
if (pTbData->pData == NULL) {
taosMemoryFree(pTbData);
return NULL;
@@ -291,7 +390,7 @@ static char *tsdbTbDataGetUid(const void *arg) {
STbData *pTbData = (STbData *)arg;
return (char *)(&(pTbData->uid));
}
-static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row) {
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, TD_ROW_SVER(row));
@@ -301,7 +400,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema *
}
}
- tdAppendSTSRowToDataCol(row, *ppSchema, pCols);
+ tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
}
return 0;
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 0da18ebdeb..aa305dd71c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -163,7 +163,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2);
// static void* destroyTableCheckInfo(SArray* pTableCheckInfo);
static bool tsdbGetExternalRow(tsdbReaderT pHandle);
-static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions);
+static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions);
static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
pBlockLoadInfo->slot = -1;
@@ -351,36 +351,43 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData
pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
}
}
-#if 0
-int nQUERY = 0;
+#if 1
+int nQUERY = 0;
#endif
-static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) {
+static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) {
if (vnodeIsRollup(pVnode)) {
int level = 0;
-#if 1
+#if 0
int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
SRetention* pRetention = retentions + i;
if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) {
break;
}
+ ++level;
}
#endif
-#if 0
- ++nQUERY;
- if(nQUERY%3 == 0) {
- level = 2;
- } else if(nQUERY%2 == 0) {
- level = 1;
- } else {
- level = 0;
+#if 1
+ switch ((nQUERY++) % 3) {
+ case 0:
+ level = 0;
+ break;
+ case 1:
+ level = 1;
+ break;
+ default:
+ level = 2;
+ break;
}
#endif
if (level == TSDB_RETENTION_L0) {
+ tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
return VND_RSMA0(pVnode);
} else if (level == TSDB_RETENTION_L1) {
+ tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
return VND_RSMA1(pVnode);
} else {
+ tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level);
return VND_RSMA2(pVnode);
}
}
@@ -393,7 +400,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
goto _end;
}
- STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions);
+ STsdb* pTsdb = getTsdbByRetentions(pVnode, pReadHandle, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions);
pReadHandle->order = pCond->order;
pReadHandle->pTsdb = pTsdb;
@@ -803,12 +810,16 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iiter);
}
-static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
+static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) {
STSRow *rmem = NULL, *rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
rmem = (STSRow*)SL_GET_NODE_DATA(node);
+ // TODO: filter max version
+ // if (TD_ROW_VER(rmem) > maxVer) {
+ // rmem = NULL;
+ // }
}
}
@@ -816,6 +827,10 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
rimem = (STSRow*)SL_GET_NODE_DATA(node);
+ // TODO: filter max version
+ // if (TD_ROW_VER(rimem) > maxVer) {
+ // rimem = NULL;
+ // }
}
}
@@ -837,6 +852,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
TSKEY r2 = TD_ROW_KEY(rimem);
if (r1 == r2) {
+#if 0
if (update == TD_ROW_DISCARD_UPDATE) {
pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
tSkipListIterNext(pCheckInfo->iter);
@@ -846,6 +862,13 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
} else {
pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
}
+#endif
+ if (TD_SUPPORT_UPDATE(update)) {
+ pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
+ } else {
+ pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
+ tSkipListIterNext(pCheckInfo->iter);
+ }
return r1;
} else if (r1 < r2 && ASCENDING_TRAVERSE(order)) {
pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM;
@@ -856,12 +879,18 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
}
}
-static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow) {
+
+static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow, TDRowVerT maxVer) {
STSRow *rmem = NULL, *rimem = NULL;
if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) {
rmem = (STSRow*)SL_GET_NODE_DATA(node);
+#if 0 // TODO: skiplist refactor
+ if (TD_ROW_VER(rmem) > maxVer) {
+ rmem = NULL;
+ }
+#endif
}
}
@@ -869,6 +898,11 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) {
rimem = (STSRow*)SL_GET_NODE_DATA(node);
+#if 0 // TODO: skiplist refactor
+ if (TD_ROW_VER(rimem) > maxVer) {
+ rimem = NULL;
+ }
+#endif
}
}
@@ -890,6 +924,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
TSKEY r2 = TD_ROW_KEY(rimem);
if (r1 == r2) {
+#if 0
if (update == TD_ROW_DISCARD_UPDATE) {
tSkipListIterNext(pCheckInfo->iter);
pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
@@ -903,6 +938,16 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
*extraRow = rimem;
return rmem;
}
+#endif
+ if (TD_SUPPORT_UPDATE(update)) {
+ pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
+ *extraRow = rimem;
+ return rmem;
+ } else {
+ tSkipListIterNext(pCheckInfo->iter);
+ pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
+ return rimem;
+ }
} else {
if (ASCENDING_TRAVERSE(order)) {
if (r1 < r2) {
@@ -973,7 +1018,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) {
initTableMemIterator(pHandle, pCheckInfo);
}
- STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL);
+ STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL, TD_VER_MAX);
if (row == NULL) {
return false;
}
@@ -1250,7 +1295,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
/*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo);
assert(cur->pos >= 0 && cur->pos <= binfo.rows);
- key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update);
+ key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update, TD_VER_MAX);
if (key != TSKEY_INITIAL_VAL) {
tsdbDebug("%p key in mem:%" PRId64 ", %s", pTsdbReadHandle, key, pTsdbReadHandle->idStr);
@@ -1497,10 +1542,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
TASSERT(0);
}
- if (sVal.valType == TD_VTYPE_NULL) {
- colDataAppendNULL(pColInfo, rowIndex);
- } else {
+ if (sVal.valType == TD_VTYPE_NORM) {
colDataAppend(pColInfo, rowIndex, sVal.val, false);
+ } else {
+ colDataAppendNULL(pColInfo, rowIndex);
}
}
} else { // handle the var-string
@@ -1513,10 +1558,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
TASSERT(0);
}
- if (sVal.valType == TD_VTYPE_NULL) {
- colDataAppendNULL(pColInfo, rowIndex);
- } else {
+ if (sVal.valType == TD_VTYPE_NORM) {
colDataAppend(pColInfo, rowIndex, sVal.val, false);
+ } else {
+ colDataAppendNULL(pColInfo, rowIndex);
}
}
}
@@ -1541,11 +1586,26 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
return numOfRows + num;
}
-// TODO fix bug for reverse copy data problem
-// Note: row1 always has high priority
-static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1,
- STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2,
- bool forceSetNull) {
+/**
+ * @brief // TODO fix bug for reverse copy data problem
+ * Note: row1 always has high priority
+ *
+ * @param pTsdbReadHandle
+ * @param capacity
+ * @param curRow
+ * @param row1
+ * @param row2
+ * @param numOfCols
+ * @param uid
+ * @param pSchema1
+ * @param pSchema2
+ * @param update
+ * @param lastRowKey
+ * @return int32_t The quantity of rows appended
+ */
+static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t* curRow, STSRow* row1,
+ STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2,
+ bool update, TSKEY* lastRowKey) {
#if 1
STSchema* pSchema;
STSRow* row;
@@ -1557,12 +1617,17 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
bool isChosenRowDataRow;
int32_t chosen_itr;
SCellVal sVal = {0};
+ TSKEY rowKey = TSKEY_INITIAL_VAL;
+ int32_t nResult = 0;
+ bool isMerge = true;
// the schema version info is embeded in STSRow
int32_t numOfColsOfRow1 = 0;
if (pSchema1 == NULL) {
- pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
+ // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1));
+ // TODO: use the real schemaVersion
+ pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
}
#ifdef TD_DEBUG_PRINT_ROW
@@ -1579,7 +1644,9 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
if (row2) {
isRow2DataRow = TD_IS_TP_ROW(row2);
if (pSchema2 == NULL) {
- pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
+ // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2));
+ // TODO: use the real schemaVersion
+ pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0);
}
if (isRow2DataRow) {
numOfColsOfRow2 = schemaNCols(pSchema2);
@@ -1610,19 +1677,19 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
colIdOfRow2 = tdKvRowColIdAt(row2, k);
}
- if (colIdOfRow1 == colIdOfRow2) {
+ if (colIdOfRow1 < colIdOfRow2) { // the most probability
if (colIdOfRow1 < pColInfo->info.colId) {
- j++;
- k++;
+ ++j;
continue;
}
row = row1;
pSchema = pSchema1;
isChosenRowDataRow = isRow1DataRow;
chosen_itr = j;
- } else if (colIdOfRow1 < colIdOfRow2) {
+ } else if (colIdOfRow1 == colIdOfRow2) {
if (colIdOfRow1 < pColInfo->info.colId) {
- j++;
+ ++j;
+ ++k;
continue;
}
row = row1;
@@ -1631,7 +1698,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
chosen_itr = j;
} else {
if (colIdOfRow2 < pColInfo->info.colId) {
- k++;
+ ++k;
continue;
}
row = row2;
@@ -1639,16 +1706,37 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
chosen_itr = k;
isChosenRowDataRow = isRow2DataRow;
}
+
if (isChosenRowDataRow) {
colId = pSchema->columns[chosen_itr].colId;
offset = pSchema->columns[chosen_itr].offset;
// TODO: use STSRowIter
tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr - 1, &sVal);
+ if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+ rowKey = *(TSKEY*)sVal.val;
+ if (rowKey != *lastRowKey) {
+ isMerge = false;
+ if (*lastRowKey != TSKEY_INITIAL_VAL) {
+ ++(*curRow);
+ }
+ ++nResult;
+ }
+ *lastRowKey = rowKey;
+ }
} else {
// TODO: use STSRowIter
if (chosen_itr == 0) {
colId = PRIMARYKEY_TIMESTAMP_COL_ID;
tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal);
+ rowKey = *(TSKEY*)sVal.val;
+ if (rowKey != *lastRowKey) {
+ isMerge = false;
+ if (*lastRowKey != TSKEY_INITIAL_VAL) {
+ ++(*curRow);
+ }
+ ++nResult;
+ }
+ *lastRowKey = rowKey;
} else {
SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1);
colId = pColIdx->colId;
@@ -1657,35 +1745,46 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
}
}
+ ASSERT(rowKey != TSKEY_INITIAL_VAL);
+
if (colId == pColInfo->info.colId) {
if (tdValTypeIsNorm(sVal.valType)) {
- colDataAppend(pColInfo, numOfRows, sVal.val, false);
- } else if (forceSetNull) {
- colDataAppend(pColInfo, numOfRows, NULL, true);
+ colDataAppend(pColInfo, *curRow, sVal.val, false);
+ } else if (tdValTypeIsNull(sVal.valType)) {
+ colDataAppend(pColInfo, *curRow, NULL, true);
+ } else if (tdValTypeIsNone(sVal.valType)) {
+ // TODO: Set null if nothing append for this row
+ if (!isMerge) {
+ colDataAppend(pColInfo, *curRow, NULL, true);
+ }
+ } else {
+ ASSERT(0);
}
- i++;
+ ++i;
if (row == row1) {
- j++;
+ ++j;
} else {
- k++;
+ ++k;
}
} else {
- if (forceSetNull) {
- colDataAppend(pColInfo, numOfRows, NULL, true);
+ if (!isMerge) {
+ colDataAppend(pColInfo, *curRow, NULL, true);
}
- i++;
+ ++i;
}
}
- if (forceSetNull) {
+ if (*lastRowKey != rowKey) {
while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
- colDataAppend(pColInfo, numOfRows, NULL, true);
- i++;
+ colDataAppend(pColInfo, *curRow, NULL, true);
+ ++i;
}
}
+
+ return nResult;
#endif
}
@@ -1859,6 +1958,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
// compared with the data from in-memory buffer, to generate the correct timestamp array list
int32_t numOfRows = 0;
+ int32_t curRow = 0;
int16_t rv1 = -1;
int16_t rv2 = -1;
@@ -1874,9 +1974,11 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
return;
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL;
+ TSKEY lastRowKey = TSKEY_INITIAL_VAL;
+
do {
STSRow* row2 = NULL;
- STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2);
+ STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2, TD_VER_MAX);
if (row1 == NULL) {
break;
}
@@ -1905,9 +2007,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
rv2 = TD_ROW_SVER(row2);
}
- mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols,
- pCheckInfo->tableId, pSchema1, pSchema2, true);
- numOfRows += 1;
+ numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+ pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey);
+ // numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
}
@@ -1918,6 +2020,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
moveToNextRowInMem(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
+#if 0
if (pCfg->update) {
if (pCfg->update == TD_ROW_PARTIAL_UPDATE) {
doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos);
@@ -1933,7 +2036,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE;
mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols,
- pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull);
+ pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull, &lastRowKey);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
@@ -1943,6 +2046,35 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
cur->lastKey = key + step;
cur->mixBlock = true;
+ moveToNextRowInMem(pCheckInfo);
+ pos += step;
+ } else {
+ moveToNextRowInMem(pCheckInfo);
+ }
+#endif
+ if (TD_SUPPORT_UPDATE(pCfg->update)) {
+ doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos);
+
+ if (rv1 != TD_ROW_SVER(row1)) {
+ // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
+ rv1 = TD_ROW_SVER(row1);
+ }
+ if (row2 && rv2 != TD_ROW_SVER(row2)) {
+ // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
+ rv2 = TD_ROW_SVER(row2);
+ }
+
+ numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+ pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey);
+ // ++numOfRows;
+ if (cur->win.skey == TSKEY_INITIAL_VAL) {
+ cur->win.skey = key;
+ }
+
+ cur->win.ekey = key;
+ cur->lastKey = key + step;
+ cur->mixBlock = true;
+
moveToNextRowInMem(pCheckInfo);
pos += step;
} else {
@@ -1958,11 +2090,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
assert(end != -1);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
+#if 0
if (pCfg->update == TD_ROW_DISCARD_UPDATE) {
moveToNextRowInMem(pCheckInfo);
} else {
end -= step;
}
+#endif
+ if (!TD_SUPPORT_UPDATE(pCfg->update)) {
+ moveToNextRowInMem(pCheckInfo);
+ } else {
+ end -= step;
+ }
}
int32_t qstart = 0, qend = 0;
@@ -2572,6 +2711,7 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) {
static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win,
STsdbReadHandle* pTsdbReadHandle) {
int numOfRows = 0;
+ int curRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
win->skey = TSKEY_INITIAL_VAL;
@@ -2579,9 +2719,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
int64_t st = taosGetTimestampUs();
int16_t rv = -1;
STSchema* pSchema = NULL;
+ TSKEY lastRowKey = TSKEY_INITIAL_VAL;
+
do {
- STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL);
+ STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL, TD_VER_MAX);
if (row == NULL) {
break;
}
@@ -2604,16 +2746,18 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0);
rv = TD_ROW_SVER(row);
}
- mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema,
- NULL, true);
+ numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema,
+ NULL, true, &lastRowKey);
- if (++numOfRows >= maxRowsToRead) {
+ if (numOfRows >= maxRowsToRead) {
moveToNextRowInMem(pCheckInfo);
break;
}
} while (moveToNextRowInMem(pCheckInfo));
+ taosMemoryFreeClear(pSchema); // free the STSChema
+
assert(numOfRows <= maxRowsToRead);
// if the buffer is not full in case of descending order query, move the data in the front of the buffer
@@ -2731,6 +2875,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
STSRow* pRow = NULL;
TSKEY key = TSKEY_INITIAL_VAL;
int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
+ TSKEY lastRowKey = TSKEY_INITIAL_VAL;
+ int32_t curRow = 0;
if (++pTsdbReadHandle->activeIndex < numOfTables) {
STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex);
@@ -2738,8 +2884,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// if (ret != TSDB_CODE_SUCCESS) {
// return false;
// }
- mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId,
- NULL, NULL, true);
+ mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, pRow, NULL, numOfCols, pCheckInfo->tableId,
+ NULL, NULL, true, &lastRowKey);
taosMemoryFreeClear(pRow);
// update the last key value
diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c
index d7c9a70c18..49d02adc3a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c
@@ -246,6 +246,12 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
return 0;
}
+static FORCE_INLINE void tsdbSwapDataCols(SDataCols *pDest, SDataCols *pSrc) {
+ SDataCol *pCols = pDest->cols;
+ memcpy(pDest, pSrc, sizeof(SDataCols));
+ pSrc->cols = pCols;
+}
+
int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
ASSERT(pBlock->numOfSubBlocks > 0);
STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo);
@@ -266,17 +272,29 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
// TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
- update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
+ TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
return -1;
}
+ // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line
+ if (pBlock->numOfSubBlocks == 1) {
+ tdResetDataCols(pReadh->pDCols[1]);
+ pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode;
+ if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL,
+ TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) {
+ return -1;
+ }
+ tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]);
+ ASSERT(pReadh->pDCols[0]->bitmapMode != 0);
+ }
- ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+ ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
return 0;
}
+// TODO: filter by Multi-Version
int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds,
bool mergeBitmap) {
ASSERT(pBlock->numOfSubBlocks > 0);
@@ -297,9 +315,21 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
// TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
- update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
+ TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0)
return -1;
}
+ // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line
+ if (pBlock->numOfSubBlocks == 1) {
+ tdResetDataCols(pReadh->pDCols[1]);
+ pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode;
+ if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL,
+ TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) {
+ return -1;
+ }
+ tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]);
+ ASSERT(pReadh->pDCols[0]->bitmapMode != 0);
+ }
+
if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) {
for (int i = 0; i < numOfColsIds; ++i) {
@@ -312,7 +342,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
}
}
- ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+ ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows);
ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
@@ -623,15 +653,15 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
}
if (dcol != 0) {
- ccol++;
+ ++ccol;
}
- dcol++;
+ ++dcol;
} else if (tcolId < pDataCol->colId) {
- ccol++;
+ ++ccol;
} else {
// Set current column as NULL and forward
dataColReset(pDataCol);
- dcol++;
+ ++dcol;
}
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c
index 480b131914..105bc330d3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSma.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSma.c
@@ -1635,17 +1635,20 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) {
SSmaCfg vCreateSmaReq = {0};
if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
- tsdbWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno));
+ tsdbWarn("vgId:%d tsma create msg received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno));
return -1;
}
- tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName,
- vCreateSmaReq.tSma.indexUid);
+
+ tsdbDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", REPO_ID(pTsdb),
+ vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid);
// record current timezone of server side
vCreateSmaReq.tSma.timezoneInt = tsTimezone;
if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) {
// TODO: handle error
+ tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb),
+ vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno));
tdDestroyTSma(&vCreateSmaReq.tSma);
return -1;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 06da3b0e1d..8460400b59 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -339,7 +339,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
goto _exit;
}
- rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs);
+ rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp));
if (rsp.pArray == NULL) {
rcode = -1;
terrno = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index bc6139c304..3b5d0c209f 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -87,9 +87,7 @@ typedef struct SResultInfo { // TODO refactor
typedef struct STableQueryInfo {
TSKEY lastKey; // last check ts, todo remove it later
SResultRowPosition pos; // current active time window
-// int32_t groupIndex; // group id in table list
// SVariant tag;
-// SResultRowInfo resInfo; // result info
} STableQueryInfo;
typedef enum {
@@ -363,11 +361,12 @@ typedef struct STableScanInfo {
} STableScanInfo;
typedef struct STagScanInfo {
- SColumnInfo *pCols;
- SSDataBlock *pRes;
- int32_t totalTables;
- int32_t curPos;
- void *pReader;
+ SColumnInfo *pCols;
+ SSDataBlock *pRes;
+ SArray *pColMatchInfo;
+ int32_t curPos;
+ SReadHandle readHandle;
+ STableGroupInfo *pTableGroups;
} STagScanInfo;
typedef struct SStreamBlockScanInfo {
@@ -579,9 +578,8 @@ typedef struct SSortOperatorInfo {
uint32_t sortBufSize; // max buffer size for in-memory sort
SArray* pSortInfo;
SSortHandle* pSortHandle;
- SArray* inputSlotMap; // for index map from table scan output
+ SArray* pColMatchInfo; // for index map from table scan output
int32_t bufPageSize;
-// int32_t numOfRowsInRes;
// TODO extact struct
int64_t startTs; // sort start time
@@ -646,7 +644,7 @@ void cleanupAggSup(SAggSupporter* pAggSup);
void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity);
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo);
SSDataBlock* loadNextDataBlock(void* param);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
@@ -704,7 +702,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
#if 0
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
@@ -717,7 +715,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol);
-void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput);
void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput);
STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win);
diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h
index 2072707b30..d74628a72f 100644
--- a/source/libs/executor/inc/tsort.h
+++ b/source/libs/executor/inc/tsort.h
@@ -117,18 +117,25 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle);
/**
*
* @param pHandle
- * @param colIndex
+ * @param colId
* @return
*/
-bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex);
+bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId);
/**
*
* @param pHandle
- * @param colIndex
+ * @param colId
* @return
*/
-void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex);
+void* tsortGetValue(STupleHandle* pVHandle, int32_t colId);
+
+/**
+ *
+ * @param pSortHandle
+ * @return
+ */
+SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle);
#ifdef __cplusplus
}
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index dc0070cbee..d78ae2b79b 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3520,7 +3520,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSortedMergeOperatorInfo* pInfo = pOperator->info;
if (pOperator->status == OP_RES_TO_RETURN) {
- return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+ return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL);
}
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
@@ -4701,7 +4701,7 @@ static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo);
static SArray* extractColumnInfo(SNodeList* pNodeList);
static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols);
-static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget);
+static SArray* createSortInfo(SNodeList* pNodeList);
static SArray* createIndexMap(SNodeList* pNodeList);
static SArray* extractPartitionColInfo(SNodeList* pNodeList);
static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
@@ -4739,7 +4739,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
SQueryTableDataCond cond = {0};
-
int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
@@ -4783,6 +4782,25 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList,
pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId);
return pOperator;
+ } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
+ STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode;
+ SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
+
+ int32_t code =
+ doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
+ if (code != TSDB_CODE_SUCCESS) {
+ return NULL;
+ }
+
+ int32_t num = 0;
+ SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num);
+
+ int32_t numOfOutputCols = 0;
+ SArray* colList =
+ extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols);
+
+ SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo);
+ return pOperator;
} else {
ASSERT(0);
}
@@ -4852,16 +4870,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode;
SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
- SArray* info = createSortInfo(pSortPhyNode->pSortKeys, pSortPhyNode->pTargets);
- SArray* slotMap = createIndexMap(pSortPhyNode->pTargets);
+ SArray* info = createSortInfo(pSortPhyNode->pSortKeys);
int32_t numOfCols = 0;
- SExprInfo* pExprInfo = NULL;
- if (pSortPhyNode->pExprs != NULL) {
- pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols);
- }
+ SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols);
- pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, slotMap, pTaskInfo);
+ int32_t numOfOutputCols = 0;
+ SArray* pColList =
+ extractColMatchInfo(pSortPhyNode->pTargets, pSortPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols);
+
+ pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) {
SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
@@ -5019,7 +5037,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) {
return pList;
}
-SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) {
+SArray* createSortInfo(SNodeList* pNodeList) {
size_t numOfCols = LIST_LENGTH(pNodeList);
SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo));
if (pList == NULL) {
@@ -5034,22 +5052,7 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) {
bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST);
SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr;
-
- bool found = false;
- for (int32_t j = 0; j < LIST_LENGTH(pNodeListTarget); ++j) {
- STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j);
-
- SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr;
- if (pColNode->slotId == pColNodeT->slotId) { // to find slotId in PhysiSort OutputDataBlockDesc
- bi.slotId = pTarget->slotId;
- found = true;
- break;
- }
- }
-
- if (!found) {
- qError("sort slot id does not found");
- }
+ bi.slotId = pColNode->slotId;
taosArrayPush(pList, &bi);
}
@@ -5088,7 +5091,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
SColMatchInfo c = {0};
c.output = true;
- c.colId = pColNode->colId;
+ c.colId = pColNode->colId;
c.targetSlotId = pNode->slotId;
taosArrayPush(pList, &c);
}
@@ -5166,9 +5169,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle*
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
-#if 0
- return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId);
-#endif
+
return tsdbQueryTables(pHandle->vnode, &cond, pTableGroupInfo, queryId, taskId);
_error:
diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c
index 3b62bdd664..86a28605b2 100644
--- a/source/libs/executor/src/indexoperator.c
+++ b/source/libs/executor/src/indexoperator.c
@@ -252,8 +252,8 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu
return TSDB_CODE_QRY_INVALID_INPUT;
}
static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
- SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName,
- strlen(left->colName), right->condValue, strlen(right->condValue));
+ SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName),
+ right->condValue, strlen(right->condValue));
if (tm == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 8c9fdfe4e6..b28a65d1d2 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -13,15 +13,16 @@
* along with this program. If not, see .
*/
-#include "ttime.h"
+#include
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "os.h"
#include "querynodes.h"
+#include "systable.h"
#include "tglobal.h"
#include "tname.h"
-#include "systable.h"
+#include "ttime.h"
#include "tdatablock.h"
#include "tmsg.h"
@@ -538,10 +539,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
while (tqNextDataBlock(pInfo->readerHandle)) {
SArray* pCols = NULL;
- uint64_t groupId;
- int32_t numOfRows;
- int16_t outputCol;
- int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows, &outputCol);
+ uint64_t groupId = 0;
+ uint64_t uid = 0;
+ int32_t numOfRows = 0;
+ int16_t outputCol = 0;
+
+ int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &uid, &numOfRows, &outputCol);
if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
pTaskInfo->code = code;
@@ -550,6 +553,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
pInfo->pRes->info.groupId = groupId;
pInfo->pRes->info.rows = numOfRows;
+ pInfo->pRes->info.uid = uid;
int32_t numOfCols = pInfo->pRes->info.numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
@@ -605,10 +609,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
- taosMemoryFreeClear(pInfo);
- taosMemoryFreeClear(pOperator);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
- return NULL;
+ goto _error;
}
int32_t numOfOutput = taosArrayGetSize(pColList);
@@ -625,16 +627,13 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColIds);
int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList);
if (code != 0) {
- taosMemoryFreeClear(pInfo);
- taosMemoryFreeClear(pOperator);
- return NULL;
+ goto _error;
}
pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES);
if (pInfo->pBlockLists == NULL) {
- taosMemoryFreeClear(pInfo);
- taosMemoryFreeClear(pOperator);
- return NULL;
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _error;
}
pInfo->readerHandle = streamReadHandle;
@@ -646,7 +645,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
- pOperator->numOfExprs = pResBlock->info.numOfCols;
+ pOperator->numOfExprs = pResBlock->info.numOfCols;
pOperator->fpSet._openFn = operatorDummyOpenFn;
pOperator->fpSet.getNextFn = doStreamBlockScan;
pOperator->fpSet.closeFn = operatorDummyCloseFn;
@@ -655,6 +654,11 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL);
return pOperator;
+
+ _error:
+ taosMemoryFreeClear(pInfo);
+ taosMemoryFreeClear(pOperator);
+ return NULL;
}
static void destroySysScanOperator(void* param, int32_t numOfOutput) {
@@ -1159,16 +1163,17 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe
}
static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
-#if 0
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+
+#if 0
int32_t maxNumOfTables = (int32_t)pResultInfo->capacity;
STagScanInfo *pInfo = pOperator->info;
SSDataBlock *pRes = pInfo->pRes;
- *newgroup = false;
int32_t count = 0;
SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);
@@ -1237,55 +1242,54 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
pOperator->status = OP_EXEC_DONE;
//qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
} else { // return only the tags|table name etc.
- SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo
+#endif
- count = 0;
- while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
- int32_t i = pInfo->curPos++;
+ STagScanInfo* pInfo = pOperator->info;
+ SExprInfo* pExprInfo = &pOperator->pExpr[0];
+ SSDataBlock* pRes = pInfo->pRes;
- STableQueryInfo* item = taosArrayGetP(pa, i);
+ SArray* pa = taosArrayGetP(pInfo->pTableGroups->pGroupList, 0);
- char *data = NULL, *dst = NULL;
- int16_t type = 0, bytes = 0;
- for(int32_t j = 0; j < pOperator->numOfExprs; ++j) {
- // not assign value in case of user defined constant output column
- if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) {
- continue;
- }
+ char str[512] = {0};
+ int32_t count = 0;
+ SMetaReader mr = {0};
- SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j);
- type = pExprInfo[j].base.resSchema.type;
- bytes = pExprInfo[j].base.resSchema.bytes;
+ while (pInfo->curPos < pInfo->pTableGroups->numOfTables && count < pOperator->resultInfo.capacity) {
+ STableKeyInfo* item = taosArrayGet(pa, pInfo->curPos);
- if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
- data = tsdbGetTableName(item->pTable);
- } else {
- data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
- }
+ for (int32_t j = 0; j < pOperator->numOfExprs; ++j) {
+ SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
- dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
- doSetTagValueToResultBuf(dst, data, type, bytes);
+ // refactor later
+ if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) {
+ metaReaderInit(&mr, pInfo->readHandle.meta, 0);
+ metaGetTableEntryByUid(&mr, item->uid);
+
+ STR_TO_VARSTR(str, mr.me.name);
+ metaReaderClear(&mr);
+
+ colDataAppend(pDst, count, str, false);
+
+ // data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
+ // dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
+ // doSetTagValueToResultBuf(dst, data, type, bytes);
}
count += 1;
}
- if (pInfo->curPos >= pInfo->totalTables) {
+ if (++pInfo->curPos >= pInfo->pTableGroups->numOfTables) {
pOperator->status = OP_EXEC_DONE;
}
-
- //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
}
+ // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
if (pOperator->status == OP_EXEC_DONE) {
- setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
+ setTaskStatus(pTaskInfo, TASK_COMPLETED);
}
pRes->info.rows = count;
- return (pRes->info.rows == 0)? NULL:pInfo->pRes;
-
-#endif
- return TSDB_CODE_SUCCESS;
+ return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
}
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
@@ -1293,14 +1297,18 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
pInfo->pRes = blockDataDestroy(pInfo->pRes);
}
-SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput,
+ SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
- pInfo->pReader = readHandle;
+ pInfo->pTableGroups = pTableGroupInfo;
+ pInfo->pColMatchInfo = pColMatchInfo;
+ pInfo->pRes = pResBlock;
+ pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
pOperator->name = "TagScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
@@ -1308,9 +1316,12 @@ SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->pExpr = pExpr;
- pOperator->numOfExprs = numOfOutput;
+ pOperator->numOfExprs = numOfOutput;
pOperator->pTaskInfo = pTaskInfo;
+ initResultSizeInfo(pOperator, 4096);
+ blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL);
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 0f973b0cf0..b34889ec46 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -5,7 +5,7 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator);
static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput);
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols,
- SArray* pIndexMap, SExecTaskInfo* pTaskInfo) {
+ SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
int32_t rowSize = pResBlock->info.rowSize;
@@ -20,17 +20,19 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
pInfo->binfo.pRes = pResBlock;
initResultSizeInfo(pOperator, 1024);
- pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header
- pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer
pInfo->pSortInfo = pSortInfo;
- pInfo->inputSlotMap = pIndexMap;
+ pInfo->pColMatchInfo= pColMatchColInfo;
pOperator->name = "SortOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
pOperator->blocking = true;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
+ // lazy evaluation for the following parameter since the input datablock is not known till now.
+// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header
+// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer
+
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, NULL);
@@ -45,14 +47,12 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
return NULL;
}
-// TODO merge aggregate super table
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-
bool isNull = tsortIsNullVal(pTupleHandle, i);
if (isNull) {
- colDataAppend(pColInfo, pBlock->info.rows, NULL, true);
+ colDataAppendNULL(pColInfo, pBlock->info.rows);
} else {
char* pData = tsortGetValue(pTupleHandle, i);
colDataAppend(pColInfo, pBlock->info.rows, pData, false);
@@ -62,11 +62,16 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
pBlock->info.rows += 1;
}
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) {
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) {
blockDataCleanup(pDataBlock);
- blockDataEnsureCapacity(pDataBlock, capacity);
+ ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols);
- blockDataEnsureCapacity(pDataBlock, capacity);
+ SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
+ if (p == NULL) {
+ return NULL;
+ }
+
+ blockDataEnsureCapacity(p, capacity);
while (1) {
STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
@@ -74,12 +79,32 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
break;
}
- appendOneRowToDataBlock(pDataBlock, pTupleHandle);
- if (pDataBlock->info.rows >= capacity) {
+ appendOneRowToDataBlock(p, pTupleHandle);
+ if (p->info.rows >= capacity) {
return pDataBlock;
}
}
+ if (p->info.rows > 0) {
+ int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
+ for(int32_t i = 0; i < numOfCols; ++i) {
+ SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
+
+ for(int32_t j = 0; j < p->info.numOfCols; ++j) {
+ SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, j);
+ if (pSrc->info.colId == pmInfo->colId) {
+ SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+ colDataAssign(pDst, pSrc, p->info.rows);
+ break;
+ }
+ }
+ }
+
+ pDataBlock->info.rows = p->info.rows;
+ pDataBlock->info.capacity = p->info.rows;
+ }
+
+ blockDataDestroy(p);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
}
@@ -106,16 +131,16 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
SSortOperatorInfo* pInfo = pOperator->info;
if (pOperator->status == OP_RES_TO_RETURN) {
- return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+ return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
}
- int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
- pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT,
- pInfo->bufPageSize, numOfBufPage, pInfo->binfo.pRes, pTaskInfo->id.str);
+// pInfo->binfo.pRes is not equalled to the input datablock.
+// int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
+ pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT,
+ -1, -1, NULL, pTaskInfo->id.str);
tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator);
-
SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
ps->param = pOperator->pDownstream[0];
tsortAddSource(pInfo->pSortHandle, ps);
@@ -127,7 +152,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
}
pOperator->status = OP_RES_TO_RETURN;
- return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+ return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
}
void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
@@ -135,5 +160,5 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
taosArrayDestroy(pInfo->pSortInfo);
- taosArrayDestroy(pInfo->inputSlotMap);
+ taosArrayDestroy(pInfo->pColMatchInfo);
}
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 50aa4cfc01..d585988e5e 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -64,25 +64,8 @@ struct SSortHandle {
static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param);
-static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) {
- SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
- pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
- pBlock->info.numOfCols = numOfCols;
-
- for(int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData colInfo = {0};
-
- colInfo.info.type = pSchema[i].type;
- colInfo.info.bytes = pSchema[i].bytes;
- colInfo.info.colId = pSchema[i].colId;
- taosArrayPush(pBlock->pDataBlock, &colInfo);
-
- if (IS_VAR_DATA_TYPE(colInfo.info.type)) {
- pBlock->info.hasVarCol = true;
- }
- }
-
- return pBlock;
+SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) {
+ return createOneDataBlock(pSortHandle->pDataBlock, false);
}
/**
@@ -98,7 +81,10 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, SArray* pIndexMap, int32_t
pSortHandle->numOfPages = numOfPages;
pSortHandle->pSortInfo = pSortInfo;
pSortHandle->pIndexMap = pIndexMap;
- pSortHandle->pDataBlock = createOneDataBlock(pBlock, false);
+
+ if (pBlock != NULL) {
+ pSortHandle->pDataBlock = createOneDataBlock(pBlock, false);
+ }
pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES);
pSortHandle->cmpParam.orderInfo = pSortInfo;
@@ -530,6 +516,17 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
if (pHandle->pDataBlock == NULL) {
pHandle->pDataBlock = createOneDataBlock(pBlock, false);
+
+ // calculate the buffer pages according to the total available buffers.
+ int32_t rowSize = blockDataGetRowSize(pBlock);
+ if (rowSize * 4 > 4096) {
+ pHandle->pageSize = rowSize * 4;
+ } else {
+ pHandle->pageSize = 4096;
+ }
+ // todo!!
+ pHandle->numOfPages = 1024;
+ sortBufSize = pHandle->numOfPages * pHandle->pageSize;
}
// perform the scalar function calculation before apply the sort
@@ -538,7 +535,6 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
}
// todo relocate the columns
-
int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock, pHandle->pIndexMap);
if (code != 0) {
return code;
@@ -557,7 +553,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
}
}
- if (pHandle->pDataBlock->info.rows > 0) {
+ if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) {
size_t size = blockDataGetSize(pHandle->pDataBlock);
// Perform the in-memory sort and then flush data in the buffer into disk.
@@ -689,7 +685,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) {
bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) {
SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex);
- return colDataIsNull(pColInfoSrc, 0, pVHandle->rowIndex, NULL);
+ return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex);
}
void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index 87cd3e24a8..abb9525cc5 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -73,6 +73,11 @@ bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo)
int32_t spreadFunction(SqlFunctionCtx* pCtx);
int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t histogramFunction(SqlFunctionCtx* pCtx);
+int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 38922833f9..8fa4bd0c4d 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -225,6 +225,26 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
return TSDB_CODE_SUCCESS;
}
+static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+ if (4 != LIST_LENGTH(pFunc->pParameterList)) {
+ return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+ }
+
+ uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+ if (!IS_NUMERIC_TYPE(colType)) {
+ return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+ }
+
+ if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
+ ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
+ ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
+ return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+ }
+
+ pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE };
+ return TSDB_CODE_SUCCESS;
+}
+
static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// todo
return TSDB_CODE_SUCCESS;
@@ -242,8 +262,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l
"The parameters of first/last can only be columns");
}
- uint8_t paraType = ((SExprNode*)pPara)->resType.type;
- pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType};
+ pFunc->node.resType = ((SExprNode*)pPara)->resType;
return TSDB_CODE_SUCCESS;
}
@@ -600,6 +619,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.processFunc = diffFunction,
.finalizeFunc = functionFinalize
},
+ {
+ .name = "histogram",
+ .type = FUNCTION_TYPE_HISTOGRAM,
+ .classification = FUNC_MGT_AGG_FUNC,
+ .translateFunc = translateHistogram,
+ .getEnvFunc = getHistogramFuncEnv,
+ .initFunc = histogramFunctionSetup,
+ .processFunc = histogramFunction,
+ .finalizeFunc = histogramFinalize
+ },
{
.name = "abs",
.type = FUNCTION_TYPE_ABS,
@@ -917,7 +946,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.translateFunc = translateTbnameColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
- .sprocessFunc = NULL,
+ .sprocessFunc = qTbnameFunction,
.finalizeFunc = NULL
},
{
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 0eba442e66..3c9eca85dd 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -20,6 +20,8 @@
#include "tdatablock.h"
#include "tpercentile.h"
+#define HISTOGRAM_MAX_BINS_NUM 100
+
typedef struct SSumRes {
union {
int64_t isum;
@@ -89,6 +91,22 @@ typedef struct SSpreadInfo {
double max;
} SSpreadInfo;
+typedef struct SHistoFuncBin {
+ double lower;
+ double upper;
+ union {
+ int64_t count;
+ double percentage;
+ };
+} SHistoFuncBin;
+
+typedef struct SHistoFuncInfo {
+ int32_t numOfBins;
+ bool normalized;
+ SHistoFuncBin bins[];
+} SHistoFuncInfo;
+
+
#define SET_VAL(_info, numOfElem, res) \
do { \
if ((numOfElem) <= 0) { \
@@ -1777,3 +1795,34 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
}
return functionFinalize(pCtx, pBlock);
}
+
+bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
+ pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
+ return true;
+}
+
+bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
+ if (!functionSetup(pCtx, pResultInfo)) {
+ return false;
+ }
+
+ SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+ char* binType = pCtx->param[1].param.pz;
+ char* binDesc = pCtx->param[2].param.pz;
+ int64_t nornalized = pCtx->param[3].param.i;
+
+
+ return true;
+}
+
+int32_t histogramFunction(SqlFunctionCtx *pCtx) {
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+ SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+ //if (pInfo->hasResult == true) {
+ // SET_DOUBLE_VAL(&pInfo->result, pInfo->max - pInfo->min);
+ //}
+ return functionFinalize(pCtx, pBlock);
+}
diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 5b1573c88f..0a99ef61ce 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -594,7 +594,9 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS
//TODO: free the array output->pDataBlock
output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
- taosArrayPush(output->pDataBlock, input->columnData);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ taosArrayPush(output->pDataBlock, (input + i)->columnData);
+ }
return 0;
}
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index ba9fca2969..f5e4a9c6e6 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -81,6 +81,9 @@ typedef struct SUdf {
TUdfAggStartFunc aggStartFunc;
TUdfAggProcessFunc aggProcFunc;
TUdfAggFinishFunc aggFinishFunc;
+
+ TUdfInitFunc initFunc;
+ TUdfDestroyFunc destroyFunc;
} SUdf;
// TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
@@ -101,7 +104,19 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
return UDFC_CODE_LOAD_UDF_FAILURE;
}
- //TODO: init and destroy function
+
+ char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0};
+ char *initSuffix = "_init";
+ strcpy(initFuncName, udfName);
+ strncat(initFuncName, initSuffix, strlen(initSuffix));
+ uv_dlsym(&udf->lib, initFuncName, (void**)(&udf->initFunc));
+
+ char destroyFuncName[TSDB_FUNC_NAME_LEN+5] = {0};
+ char *destroySuffix = "_destroy";
+ strcpy(destroyFuncName, udfName);
+ strncat(destroyFuncName, destroySuffix, strlen(destroySuffix));
+ uv_dlsym(&udf->lib, destroyFuncName, (void**)(&udf->destroyFunc));
+
if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) {
char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
strcpy(processFuncName, udfName);
@@ -159,6 +174,9 @@ void udfdProcessRequest(uv_work_t *req) {
if (udf->state == UDF_STATE_INIT) {
udf->state = UDF_STATE_LOADING;
udfdLoadUdf(setup->udfName, udf);
+ if (udf->initFunc) {
+ udf->initFunc();
+ }
udf->state = UDF_STATE_READY;
uv_cond_broadcast(&udf->condReady);
uv_mutex_unlock(&udf->lock);
@@ -170,7 +188,6 @@ void udfdProcessRequest(uv_work_t *req) {
}
SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle));
handle->udf = udf;
- // TODO: allocate private structure and call init function and set it to handle
SUdfResponse rsp;
rsp.seqNum = request.seqNum;
rsp.type = request.type;
@@ -275,10 +292,12 @@ void udfdProcessRequest(uv_work_t *req) {
if (unloadUdf) {
uv_cond_destroy(&udf->condReady);
uv_mutex_destroy(&udf->lock);
+ if (udf->destroyFunc) {
+ (udf->destroyFunc)();
+ }
uv_dlclose(&udf->lib);
taosMemoryFree(udf);
}
- // TODO: call destroy and free udf private
taosMemoryFree(handle);
SUdfResponse response;
diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h
index 3ea986ad48..d474d87409 100644
--- a/source/libs/index/inc/indexCache.h
+++ b/source/libs/index/inc/indexCache.h
@@ -31,6 +31,7 @@ extern "C" {
typedef struct MemTable {
T_REF_DECLARE()
SSkipList* mem;
+ void* pCache;
} MemTable;
typedef struct IndexCache {
T_REF_DECLARE()
@@ -47,6 +48,7 @@ typedef struct IndexCache {
} IndexCache;
#define CACHE_VERSION(cache) atomic_load_32(&cache->version)
+
typedef struct CacheTerm {
// key
char* colVal;
diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h
index 0d8418ba65..3b07429089 100644
--- a/source/libs/index/inc/indexComm.h
+++ b/source/libs/index/inc/indexComm.h
@@ -24,7 +24,7 @@ extern char JSON_COLUMN[];
extern char JSON_VALUE_DELIM;
char* indexPackJsonData(SIndexTerm* itm);
-
+char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
#ifdef __cplusplus
}
#endif
diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h
index 5c7b8b9afe..7b7050d80e 100644
--- a/source/libs/index/inc/indexInt.h
+++ b/source/libs/index/inc/indexInt.h
@@ -166,7 +166,9 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
} while (0)
#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
-#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F)
+
+#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F)
+
#define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \
do { \
uint8_t oldTy = ty; \
diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c
index 83b5025ad0..e0c24ac3bd 100644
--- a/source/libs/index/src/index.c
+++ b/source/libs/index/src/index.c
@@ -244,8 +244,8 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde
return 0;
}
-SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType,
- const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) {
+SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
+ int32_t nColName, const char* colVal, int32_t nColVal) {
SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm)));
if (tm == NULL) {
return NULL;
diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c
index 13768ce682..b3ae7b7dbe 100644
--- a/source/libs/index/src/indexCache.c
+++ b/source/libs/index/src/indexCache.c
@@ -34,44 +34,96 @@ static char* indexCacheTermGet(const void* pData);
static MemTable* indexInternalCacheCreate(int8_t type);
-static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
-static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchSuffix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRegex(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
/*comm func of compare, used in (LE/LT/GE/GT compare)*/
-static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s,
RangeType type);
+static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+
+static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
+ RangeType type);
typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
-static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; }
-static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; }
-static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; }
-static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) {
+ int32_t ret = func(a, b);
+ switch (comType) {
+ case QUERY_LESS_THAN: {
+ if (ret < 0) return MATCH;
+ } break;
+ case QUERY_LESS_EQUAL: {
+ if (ret <= 0) return MATCH;
+ break;
+ }
+ case QUERY_GREATER_THAN: {
+ if (ret > 0) return MATCH;
+ break;
+ }
+ case QUERY_GREATER_EQUAL: {
+ if (ret >= 0) return MATCH;
+ }
+ }
+ return CONTINUE;
+}
+static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
+ __compar_fn_t func = getComparFunc(type, 0);
+ return tDoCommpare(func, QUERY_LESS_THAN, a, b);
+}
+static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
+ __compar_fn_t func = getComparFunc(type, 0);
+ return tDoCommpare(func, QUERY_LESS_EQUAL, a, b);
+}
+static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
+ __compar_fn_t func = getComparFunc(type, 0);
+ return tDoCommpare(func, QUERY_GREATER_THAN, a, b);
+}
+static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
+ __compar_fn_t func = getComparFunc(type, 0);
+ return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b);
+}
static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual,
tCompareGreaterThan, tCompareGreaterEqual};
-static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
- cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan,
- cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange};
+static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
+ {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual,
+ cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange},
+ {cacheSearchTerm_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON,
+ cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON,
+ cacheSearchRange_JSON}};
static void doMergeWork(SSchedMsg* msg);
static bool indexCacheIteratorNext(Iterate* itera);
-static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
if (cache == NULL) {
return 0;
}
+ MemTable* mem = cache;
+ IndexCache* pCache = mem->pCache;
- MemTable* mem = cache;
- char* key = indexCacheTermGet(ct);
+ CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+ pCt->colVal = term->colVal;
+ pCt->version = atomic_load_32(&pCache->version);
+
+ char* key = indexCacheTermGet(pCt);
SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
while (tSkipListIterNext(iter)) {
@@ -80,7 +132,7 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S
break;
}
CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
- if (0 == strcmp(c->colVal, ct->colVal)) {
+ if (0 == strcmp(c->colVal, pCt->colVal)) {
if (c->operaType == ADD_VALUE) {
INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
// taosArrayPush(result, &c->uid);
@@ -92,30 +144,39 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S
break;
}
}
+
+ taosMemoryFree(pCt);
tSkipListDestroyIter(iter);
return 0;
}
-static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
// impl later
return 0;
}
-static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchSuffix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
// impl later
return 0;
}
-static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchRegex(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
// impl later
return 0;
}
-static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
RangeType type) {
if (cache == NULL) {
return 0;
}
+
_cache_range_compare cmpFn = rangeCompare[type];
- MemTable* mem = cache;
- char* key = indexCacheTermGet(ct);
+ MemTable* mem = cache;
+ IndexCache* pCache = mem->pCache;
+
+ CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+ pCt->colVal = term->colVal;
+ pCt->version = atomic_load_32(&pCache->version);
+
+ char* key = indexCacheTermGet(pCt);
SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
while (tSkipListIterNext(iter)) {
@@ -124,7 +185,7 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult
break;
}
CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
- TExeCond cond = cmpFn(c->colVal, ct->colVal, ct->colType);
+ TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType);
if (cond == MATCH) {
if (c->operaType == ADD_VALUE) {
INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
@@ -134,26 +195,153 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult
INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
}
} else if (cond == CONTINUE) {
+ continue;
} else if (cond == BREAK) {
break;
}
}
+ taosMemoryFree(pCt);
tSkipListDestroyIter(iter);
return TSDB_CODE_SUCCESS;
}
-static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
- return cacheSearchCompareFunc(cache, ct, tr, s, LT);
+static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc(cache, term, tr, s, LT);
}
-static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
- return cacheSearchCompareFunc(cache, ct, tr, s, LE);
+static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc(cache, term, tr, s, LE);
}
-static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
- return cacheSearchCompareFunc(cache, ct, tr, s, GT);
+static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc(cache, term, tr, s, GT);
}
-static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
- return cacheSearchCompareFunc(cache, ct, tr, s, GE);
+static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc(cache, term, tr, s, GE);
}
-static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+
+static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ if (cache == NULL) {
+ return 0;
+ }
+ MemTable* mem = cache;
+ IndexCache* pCache = mem->pCache;
+
+ CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+ pCt->colVal = term->colVal;
+ pCt->version = atomic_load_32(&pCache->version);
+
+ char* exBuf = NULL;
+ if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+ exBuf = indexPackJsonData(term);
+ pCt->colVal = exBuf;
+ }
+ char* key = indexCacheTermGet(pCt);
+
+ SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
+ while (tSkipListIterNext(iter)) {
+ SSkipListNode* node = tSkipListIterGet(iter);
+ if (node == NULL) {
+ break;
+ }
+ CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
+ if (0 == strcmp(c->colVal, pCt->colVal)) {
+ if (c->operaType == ADD_VALUE) {
+ INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+ // taosArrayPush(result, &c->uid);
+ *s = kTypeValue;
+ } else if (c->operaType == DEL_VALUE) {
+ INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+ }
+ } else {
+ break;
+ }
+ }
+
+ taosMemoryFree(pCt);
+ taosMemoryFree(exBuf);
+ tSkipListDestroyIter(iter);
+ return 0;
+
+ return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT);
+}
+static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc_JSON(cache, term, tr, s, LE);
+}
+static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc_JSON(cache, term, tr, s, GT);
+}
+static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE);
+}
+static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+ return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
+ RangeType type) {
+ if (cache == NULL) {
+ return 0;
+ }
+ _cache_range_compare cmpFn = rangeCompare[type];
+
+ MemTable* mem = cache;
+ IndexCache* pCache = mem->pCache;
+
+ CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
+ pCt->colVal = term->colVal;
+ pCt->version = atomic_load_32(&pCache->version);
+
+ int8_t dType = INDEX_TYPE_GET_TYPE(term->colType);
+ int skip = 0;
+ char* exBuf = NULL;
+
+ if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+ exBuf = indexPackJsonDataPrefix(term, &skip);
+ pCt->colVal = exBuf;
+ }
+ char* key = indexCacheTermGet(pCt);
+
+ SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
+ while (tSkipListIterNext(iter)) {
+ SSkipListNode* node = tSkipListIterGet(iter);
+ if (node == NULL) {
+ break;
+ }
+ CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
+
+ TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType);
+ if (cond == MATCH) {
+ if (c->operaType == ADD_VALUE) {
+ INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+ // taosArrayPush(result, &c->uid);
+ *s = kTypeValue;
+ } else if (c->operaType == DEL_VALUE) {
+ INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+ }
+ } else if (cond == CONTINUE) {
+ continue;
+ } else if (cond == BREAK) {
+ break;
+ }
+ }
+
+ taosMemoryFree(pCt);
+ taosMemoryFree(exBuf);
+ tSkipListDestroyIter(iter);
+
+ return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
// impl later
return 0;
}
@@ -167,6 +355,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
};
cache->mem = indexInternalCacheCreate(type);
+ cache->mem->pCache = cache;
cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName);
cache->type = type;
cache->index = idx;
@@ -325,6 +514,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
indexCacheRef(cache);
cache->imm = cache->mem;
cache->mem = indexInternalCacheCreate(cache->type);
+ cache->mem->pCache = cache;
cache->occupiedMem = 0;
// sched to merge
// unref cache in bgwork
@@ -379,11 +569,19 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u
return 0;
}
-static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SIdxTempResult* tr, STermValueType* s) {
+static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTempResult* tr, STermValueType* s) {
if (mem == NULL) {
return 0;
}
- return cacheSearch[qtype](mem, ct, tr, s);
+
+ SIndexTerm* term = query->term;
+ EIndexQueryType qtype = query->qType;
+
+ if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) {
+ return cacheSearch[1][qtype](mem, term, tr, s);
+ } else {
+ return cacheSearch[0][qtype](mem, term, tr, s);
+ }
}
int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) {
int64_t st = taosGetTimestampUs();
@@ -400,25 +598,24 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result
indexMemRef(imm);
taosThreadMutexUnlock(&pCache->mtx);
- SIndexTerm* term = query->term;
- EIndexQueryType qtype = query->qType;
+ // SIndexTerm* term = query->term;
+ // EIndexQueryType qtype = query->qType;
- bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
- char* p = term->colVal;
- if (hasJson) {
- p = indexPackJsonData(term);
- }
- CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)};
+ // bool isJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
+ // char* p = term->colVal;
+ // if (isJson) {
+ // p = indexPackJsonData(term);
+ //}
+ // CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)};
- int ret = indexQueryMem(mem, &ct, qtype, result, s);
+ int ret = indexQueryMem(mem, query, result, s);
if (ret == 0 && *s != kTypeDeletion) {
// continue search in imm
- ret = indexQueryMem(imm, &ct, qtype, result, s);
- }
-
- if (hasJson) {
- taosMemoryFreeClear(p);
+ ret = indexQueryMem(imm, query, result, s);
}
+ // if (isJson) {
+ // taosMemoryFreeClear(p);
+ //}
indexMemUnRef(mem);
indexMemUnRef(imm);
diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c
index 4dea5fa011..cdd7b35675 100644
--- a/source/libs/index/src/indexComm.c
+++ b/source/libs/index/src/indexComm.c
@@ -46,3 +46,30 @@ char* indexPackJsonData(SIndexTerm* itm) {
return buf;
}
+char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) {
+ /*
+ * |<-----colname---->|<-----dataType---->|<--------colVal---------->|
+ * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->|
+ */
+ uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType);
+
+ int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1;
+ char* buf = (char*)taosMemoryCalloc(1, sz);
+ char* p = buf;
+
+ memcpy(p, itm->colName, itm->nColName);
+ p += itm->nColName;
+
+ memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM));
+ p += sizeof(JSON_VALUE_DELIM);
+
+ memcpy(p, &ty, sizeof(ty));
+ p += sizeof(ty);
+
+ memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM));
+ p += sizeof(JSON_VALUE_DELIM);
+
+ *skip = p - buf;
+
+ return buf;
+}
diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c
index 5aed2bd6b0..b5551e825f 100644
--- a/source/libs/index/src/indexTfile.c
+++ b/source/libs/index/src/indexTfile.c
@@ -308,20 +308,20 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
}
static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) {
- bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
- int ret = 0;
- char* p = tem->colVal;
- uint64_t sz = tem->nColVal;
+ bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+ int ret = 0;
+ char* p = tem->colVal;
+ int skip = 0;
+
if (hasJson) {
- p = indexPackJsonData(tem);
- sz = strlen(p);
+ p = indexPackJsonDataPrefix(tem, &skip);
}
SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
- FstSlice h = fstSliceCreate((uint8_t*)p, sz);
+ FstSlice h = fstSliceCreate((uint8_t*)p, skip);
fstStreamBuilderSetRange(sb, &h, type);
fstSliceDestroy(&h);
@@ -606,16 +606,16 @@ static bool tfileIteratorNext(Iterate* iiter) {
static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
- TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter));
- if (tIter == NULL) {
+ TFileFstIter* iter = taosMemoryCalloc(1, sizeof(TFileFstIter));
+ if (iter == NULL) {
return NULL;
}
- tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
- tIter->fb = fstSearch(reader->fst, tIter->ctx);
- tIter->st = streamBuilderIntoStream(tIter->fb);
- tIter->rdr = reader;
- return tIter;
+ iter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
+ iter->fb = fstSearch(reader->fst, iter->ctx);
+ iter->st = streamBuilderIntoStream(iter->fb);
+ iter->rdr = reader;
+ return iter;
}
Iterate* tfileIteratorCreate(TFileReader* reader) {
diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc
index d8ea6a8233..896451c686 100644
--- a/source/libs/index/test/indexTests.cc
+++ b/source/libs/index/test/indexTests.cc
@@ -483,7 +483,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
std::string colName("voltage");
std::string colVal("ab");
- SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {term, QUERY_TERM};
@@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) {
std::string colName("voltage");
{
std::string colVal("v1");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
@@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) {
}
{
std::string colVal("v3");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v2");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v3");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v3");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
@@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) {
std::cout << "--------first----------" << std::endl;
{
std::string colVal("v3");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++);
indexTermDestroy(term);
}
{
std::string colVal("v4");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, othColId, version++, suid++);
indexTermDestroy(term);
@@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) {
std::string colVal("v4");
for (size_t i = 0; i < 10; i++) {
colVal[colVal.size() - 1] = 'a' + i;
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
coj->Put(term, colId, version++, suid++);
indexTermDestroy(term);
@@ -623,7 +623,7 @@ TEST_F(IndexCacheEnv, cache_test) {
// begin query
{
std::string colVal("v3");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {term, QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
@@ -638,7 +638,7 @@ TEST_F(IndexCacheEnv, cache_test) {
}
{
std::string colVal("v2");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexTermQuery query = {term, QUERY_TERM};
SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid));
@@ -670,7 +670,7 @@ class IndexObj {
return ret;
}
void Del(const std::string& colName, const std::string& colVal, uint64_t uid) {
- SIndexTerm* term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@@ -679,7 +679,7 @@ class IndexObj {
}
int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world",
size_t numOfTable = 100 * 10000) {
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@@ -701,7 +701,7 @@ class IndexObj {
// opt
tColVal[taosRand() % colValSize] = 'a' + k % 26;
}
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
tColVal.c_str(), tColVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@@ -737,7 +737,7 @@ class IndexObj {
int SearchOne(const std::string& colName, const std::string& colVal) {
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
@@ -759,7 +759,7 @@ class IndexObj {
}
int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) {
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
@@ -784,7 +784,7 @@ class IndexObj {
void PutOne(const std::string& colName, const std::string& colVal) {
SIndexMultiTerm* terms = indexMultiTermCreate();
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
Put(terms, 10);
@@ -792,7 +792,7 @@ class IndexObj {
}
void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) {
SIndexMultiTerm* terms = indexMultiTermCreate();
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
Put(terms, val);
@@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
{
std::string colName("tag1"), colVal("Hello");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
size_t size = 200;
std::string colName("tag1"), colVal("hello");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
size_t size = 200;
std::string colName("tag1"), colVal("Hello");
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
@@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
{
std::string colName("tag1"), colVal("Hello");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
- SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
indexMultiTermQueryAdd(mq, term, QUERY_TERM);
diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc
index e1e5004701..f789d23136 100644
--- a/source/libs/index/test/jsonUT.cc
+++ b/source/libs/index/test/jsonUT.cc
@@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) {
{
std::string colName("test");
std::string colVal("ab");
- SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) {
{
std::string colName("voltage");
std::string colVal("ab1");
- SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) {
{
std::string colName("voltage");
std::string colVal("123");
- SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) {
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
- SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
{
std::string colName("test");
std::string colVal("ab");
- SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
std::string colVal("abxxxxxxxxxxxx");
for (int i = 0; i < 1000; i++) {
colVal[i % colVal.size()] = '0' + i % 128;
- SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
{
std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx");
- SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
std::string colVal("ab");
SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
- SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SArray* result = taosArrayInit(1, sizeof(uint64_t));
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index fbb1f34217..6874b5b7d4 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -2174,17 +2174,6 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
return TSDB_CODE_SUCCESS;
}
-static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
- SNode* pNode;
- FOREACH(pNode, pStmt->pTags) {
- SColumnDefNode* pCol = (SColumnDefNode*)pNode;
- if (pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1) {
- return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
- }
- }
- return TSDB_CODE_SUCCESS;
-}
-
static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
if (NULL == pFuncs) {
return TSDB_CODE_SUCCESS;
@@ -2196,6 +2185,113 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs
return TSDB_CODE_SUCCESS;
}
+static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pTags) {
+ int32_t ntags = LIST_LENGTH(pTags);
+ if (0 == ntags) {
+ return TSDB_CODE_SUCCESS;
+ } else if (ntags > TSDB_MAX_TAGS) {
+ return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM);
+ }
+
+ int32_t code = TSDB_CODE_SUCCESS;
+ int32_t tagsSize = 0;
+ SNode* pNode = NULL;
+ FOREACH(pNode, pTags) {
+ SColumnDefNode* pTag = (SColumnDefNode*)pNode;
+ int32_t len = strlen(pTag->colName);
+ if (NULL != taosHashGet(pHash, pTag->colName, len)) {
+ code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN);
+ }
+ if (TSDB_CODE_SUCCESS == code && pTag->dataType.type == TSDB_DATA_TYPE_JSON && ntags > 1) {
+ code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_BINARY_LEN) ||
+ (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && pTag->dataType.bytes > TSDB_MAX_NCHAR_LEN)) {
+ code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
+ }
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = taosHashPut(pHash, pTag->colName, len, &pTag, POINTER_BYTES);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ tagsSize += pTag->dataType.bytes;
+ } else {
+ break;
+ }
+ }
+
+ if (TSDB_CODE_SUCCESS == code && tagsSize > TSDB_MAX_TAGS_LEN) {
+ code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_LENGTH, TSDB_MAX_TAGS_LEN);
+ }
+
+ return code;
+}
+
+static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, SNodeList* pCols) {
+ int32_t ncols = LIST_LENGTH(pCols);
+ if (ncols < TSDB_MIN_COLUMNS) {
+ return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
+ } else if (ncols > TSDB_MAX_COLUMNS) {
+ return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TOO_MANY_COLUMNS);
+ }
+
+ int32_t code = TSDB_CODE_SUCCESS;
+
+ bool first = true;
+ int32_t rowSize = 0;
+ SNode* pNode = NULL;
+ FOREACH(pNode, pCols) {
+ SColumnDefNode* pCol = (SColumnDefNode*)pNode;
+ if (first) {
+ first = false;
+ if (TSDB_DATA_TYPE_TIMESTAMP != pCol->dataType.type) {
+ code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN);
+ }
+ }
+ int32_t len = strlen(pCol->colName);
+ if (TSDB_CODE_SUCCESS == code && NULL != taosHashGet(pHash, pCol->colName, len)) {
+ code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_BINARY_LEN) ||
+ (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_NCHAR_LEN)) {
+ code = code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
+ }
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ rowSize += pCol->dataType.bytes;
+ } else {
+ break;
+ }
+ }
+
+ if (TSDB_CODE_SUCCESS == code && rowSize > TSDB_MAX_BYTES_PER_ROW) {
+ code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW);
+ }
+
+ return code;
+}
+
+static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+ SHashObj* pHash = taosHashInit(LIST_LENGTH(pStmt->pTags) + LIST_LENGTH(pStmt->pCols),
+ taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+ if (NULL == pHash) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+
+ int32_t code = checkTableTagsSchema(pCxt, pHash, pStmt->pTags);
+ if (TSDB_CODE_SUCCESS == code) {
+ code = checkTableColsSchema(pCxt, pHash, pStmt->pCols);
+ }
+
+ taosHashCleanup(pHash);
+ return code;
+}
+
static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
int32_t code = checkRangeOption(pCxt, "delay", pStmt->pOptions->delay, TSDB_MIN_ROLLUP_DELAY, TSDB_MAX_ROLLUP_DELAY);
if (TSDB_CODE_SUCCESS == code) {
@@ -2211,7 +2307,7 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
code = checkTableSmaOption(pCxt, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
- code = checkTableTags(pCxt, pStmt);
+ code = checkTableSchema(pCxt, pStmt);
}
return code;
}
@@ -3838,6 +3934,10 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl
goto over;
}
+ if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code && pClause->ignoreNotExists) {
+ code = TSDB_CODE_SUCCESS;
+ }
+
*pIsSuperTable = false;
SVgroupInfo info = {0};
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index e839536218..e37dbd1edd 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -129,7 +129,23 @@ static char* getSyntaxErrFormat(int32_t errCode) {
case TSDB_CODE_PAR_INVALID_DROP_STABLE:
return "Cannot drop super table in batch";
case TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE:
- return "start(end) time of query range required or time range too large";
+ return "Start(end) time of query range required or time range too large";
+ case TSDB_CODE_PAR_DUPLICATED_COLUMN:
+ return "Duplicated column names";
+ case TSDB_CODE_PAR_INVALID_TAGS_LENGTH:
+ return "Tags length exceeds max length %d";
+ case TSDB_CODE_PAR_INVALID_ROW_LENGTH:
+ return "Row length exceeds max length %d";
+ case TSDB_CODE_PAR_INVALID_COLUMNS_NUM:
+ return "Illegal number of columns";
+ case TSDB_CODE_PAR_TOO_MANY_COLUMNS:
+ return "Too many columns";
+ case TSDB_CODE_PAR_INVALID_FIRST_COLUMN:
+ return "First column must be timestamp";
+ case TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN:
+ return "Invalid binary/nchar column length";
+ case TSDB_CODE_PAR_INVALID_TAGS_NUM:
+ return "Invalid number of tag columns";
case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory";
default:
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index 0bc700696b..3e19ccbd82 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -277,6 +277,11 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols);
}
+ // rewrite the expression in subsequent clauses
+ if (TSDB_CODE_SUCCESS == code) {
+ code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM);
+ }
+
pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta);
if (TSDB_CODE_SUCCESS == code) {
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 67609355d7..dbce9abf36 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -123,14 +123,40 @@ static SNodeList* osdGetAllFuncs(SLogicNode* pNode) {
return NULL;
}
+static bool needOptimizeDataRequire(const SFunctionNode* pFunc) {
+ if (!fmIsSpecialDataRequiredFunc(pFunc->funcId)) {
+ return false;
+ }
+ SNode* pPara = NULL;
+ FOREACH(pPara, pFunc->pParameterList) {
+ if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) {
+ if (!fmIsDynamicScanOptimizedFunc(pFunc->funcId)) {
+ return false;
+ }
+ SNode* pPara = NULL;
+ FOREACH(pPara, pFunc->pParameterList) {
+ if (QUERY_NODE_COLUMN != nodeType(pPara) && QUERY_NODE_VALUE != nodeType(pPara)) {
+ return false;
+ }
+ }
+ return true;
+}
+
static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) {
SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent);
SNode* pFunc = NULL;
FOREACH(pFunc, pAllFuncs) {
int32_t code = TSDB_CODE_SUCCESS;
- if (fmIsSpecialDataRequiredFunc(((SFunctionNode*)pFunc)->funcId)) {
+ if (needOptimizeDataRequire((SFunctionNode*)pFunc)) {
code = nodesListMakeStrictAppend(pSdrFuncs, nodesCloneNode(pFunc));
- } else if (fmIsDynamicScanOptimizedFunc(((SFunctionNode*)pFunc)->funcId)) {
+ } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) {
code = nodesListMakeStrictAppend(pDsoFuncs, nodesCloneNode(pFunc));
}
if (TSDB_CODE_SUCCESS != code) {
@@ -541,9 +567,14 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
if (QUERY_NODE_OPERATOR != nodeType(pCond)) {
return false;
}
- SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
- SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
+
SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions;
+ if (OP_TYPE_EQUAL != pOper->opType) {
+ return false;
+ }
+
+ SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
+ SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) {
return cpdIsPrimaryKey(pOper->pRight, pRightCols);
} else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) {
diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp
index deb20c65a4..4938618db6 100644
--- a/source/libs/planner/test/planOptimizeTest.cpp
+++ b/source/libs/planner/test/planOptimizeTest.cpp
@@ -20,11 +20,21 @@ using namespace std;
class PlanOptimizeTest : public PlannerTestBase {};
+TEST_F(PlanOptimizeTest, optimizeScanData) {
+ useDb("root", "test");
+
+ run("SELECT COUNT(*) FROM t1");
+
+ run("SELECT COUNT(c1) FROM t1");
+
+ run("SELECT COUNT(CAST(c1 AS BIGINT)) FROM t1");
+}
+
TEST_F(PlanOptimizeTest, orderByPrimaryKey) {
useDb("root", "test");
- run("select * from t1 order by ts");
- run("select * from t1 order by ts desc");
- run("select c1 from t1 order by ts");
- run("select c1 from t1 order by ts desc");
+ run("SELECT * FROM t1 ORDER BY ts");
+ run("SELECT * FROM t1 ORDER BY ts DESC");
+ run("SELECT c1 FROM t1 ORDER BY ts");
+ run("SELECT c1 FROM t1 ORDER BY ts DESC");
}
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 51ac48adaf..77caf67a2b 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1513,3 +1513,9 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t*) colDataGetData(pInput->columnData, 4));
return TSDB_CODE_SUCCESS;
}
+
+int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+ ASSERT(inputNum == 1);
+ colDataAppend(pOutput->columnData, pOutput->numOfRows, colDataGetData(pInput->columnData, 0), false);
+ return TSDB_CODE_SUCCESS;
+}
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 84aa5559d0..1e37533f2c 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -1023,8 +1023,7 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
- *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
- * getVectorDoubleValueFnRight(RIGHT_COL, 0);
+ *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0);
}
}
}
@@ -1050,8 +1049,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
colDataAppendNULL(pOutputCol, i);
continue; // TODO set null or ignore
}
- *output = getVectorDoubleValueFnLeft(LEFT_COL, i)
- * getVectorDoubleValueFnRight(RIGHT_COL, i);
+ *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i);
}
} else if (pLeft->numOfRows == 1) {
vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i);
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index a70d366e9b..53772601ca 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -22,7 +22,7 @@
#include "trpc.h"
SSchedulerMgmt schMgmt = {
- .jobRef = -1,
+ .jobRef = -1,
};
FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); }
@@ -72,7 +72,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *
int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql,
int64_t startTs, bool syncSchedule) {
int32_t code = 0;
- int64_t refId = -1;
+ int64_t refId = -1;
SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob));
if (NULL == pJob) {
qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob));
@@ -124,7 +124,7 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray
}
atomic_add_fetch_32(&schMgmt.jobNum, 1);
-
+
if (NULL == schAcquireJob(refId)) {
SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId);
SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
@@ -1085,19 +1085,22 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
case TDMT_VND_CREATE_TABLE_RSP: {
SVCreateTbBatchRsp batchRsp = {0};
if (msg) {
- SCH_ERR_JRET(tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp));
- if (batchRsp.pArray) {
- int32_t num = taosArrayGetSize(batchRsp.pArray);
- for (int32_t i = 0; i < num; ++i) {
- SVCreateTbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
+ SCoder coder = {0};
+ tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER);
+ code = tDecodeSVCreateTbBatchRsp(&coder, &batchRsp);
+ if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) {
+ for (int32_t i = 0; i < batchRsp.nRsps; ++i) {
+ SVCreateTbRsp *rsp = batchRsp.pRsps + i;
if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) {
- taosArrayDestroy(batchRsp.pArray);
+ tCoderClear(&coder);
SCH_ERR_JRET(rsp->code);
+ } else if (TSDB_CODE_SUCCESS != rsp->code) {
+ code = rsp->code;
}
}
-
- taosArrayDestroy(batchRsp.pArray);
}
+ tCoderClear(&coder);
+ SCH_ERR_JRET(code);
}
SCH_ERR_JRET(rspCode);
@@ -1110,13 +1113,14 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
SCoder coder = {0};
tCoderInit(&coder, TD_LITTLE_ENDIAN, msg, msgSize, TD_DECODER);
code = tDecodeSVDropTbBatchRsp(&coder, &batchRsp);
- if (TSDB_CODE_SUCCESS == code && batchRsp.pArray) {
- int32_t num = taosArrayGetSize(batchRsp.pArray);
- for (int32_t i = 0; i < num; ++i) {
- SVDropTbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
+ if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) {
+ for (int32_t i = 0; i < batchRsp.nRsps; ++i) {
+ SVDropTbRsp *rsp = batchRsp.pRsps + i;
if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) {
tCoderClear(&coder);
SCH_ERR_JRET(rsp->code);
+ } else if (TSDB_CODE_SUCCESS != rsp->code) {
+ code = rsp->code;
}
}
}
@@ -2282,10 +2286,10 @@ int32_t schCancelJob(SSchJob *pJob) {
}
void schCloseJobRef(void) {
- if (!atomic_load_8((int8_t*)&schMgmt.exit)) {
+ if (!atomic_load_8((int8_t *)&schMgmt.exit)) {
return;
}
-
+
SCH_LOCK(SCH_WRITE, &schMgmt.lock);
if (atomic_load_32(&schMgmt.jobNum) <= 0 && schMgmt.jobRef >= 0) {
taosCloseRef(schMgmt.jobRef);
@@ -2791,8 +2795,8 @@ void schedulerFreeTaskList(SArray *taskList) {
}
void schedulerDestroy(void) {
- atomic_store_8((int8_t*)&schMgmt.exit, 1);
-
+ atomic_store_8((int8_t *)&schMgmt.exit, 1);
+
if (schMgmt.jobRef >= 0) {
SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0);
int64_t refId = 0;
diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c
index ae153251c3..6bf17b8a82 100644
--- a/source/libs/sync/src/syncRaftLog.c
+++ b/source/libs/sync/src/syncRaftLog.c
@@ -57,6 +57,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
syncMeta.seqNum = pEntry->seqNum;
syncMeta.term = pEntry->term;
code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
+ if (code < 0) perror("wal write error: ");
assert(code == 0);
walFsync(pWal, true);
diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c
index c0d8ca44ee..6047643881 100644
--- a/source/libs/transport/src/transSrv.c
+++ b/source/libs/transport/src/transSrv.c
@@ -784,6 +784,11 @@ static void uvDestroyConn(uv_handle_t* handle) {
tDebug("server conn %p destroy", conn);
// uv_timer_stop(&conn->pTimer);
transQueueDestroy(&conn->srvMsgs);
+
+ if (conn->regArg.init == 1) {
+ transFreeMsg(conn->regArg.msg.pCont);
+ conn->regArg.init = 0;
+ }
QUEUE_REMOVE(&conn->queue);
taosMemoryFree(conn->pTcp);
if (conn->regArg.init == 1) {
diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index dc31086e9f..da1c36dcc4 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -198,12 +198,14 @@ int walRoll(SWal *pWal) {
if (pWal->pWriteIdxTFile != NULL) {
code = taosCloseFile(&pWal->pWriteIdxTFile);
if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (pWal->pWriteLogTFile != NULL) {
code = taosCloseFile(&pWal->pWriteLogTFile);
if (code != 0) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
@@ -263,14 +265,19 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog
if (index == pWal->vers.lastVer + 1) {
if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
pWal->vers.firstVer = index;
- code = walRoll(pWal);
- ASSERT(code == 0);
+ if (walRoll(pWal) < 0) {
+ return -1;
+ }
} else {
int64_t passed = walGetSeq() - pWal->lastRollSeq;
if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
- walRoll(pWal);
+ if (walRoll(pWal) < 0) {
+ return -1;
+ }
} else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
- walRoll(pWal);
+ if (walRoll(pWal) < 0) {
+ return -1;
+ }
}
}
} else {
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 00fe8bd0e9..1470496c68 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -443,6 +443,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status erro
TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error")
TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order")
+// parser
+TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TABLE_NOT_EXIST, "Table does not exist")
+
//planner
TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error")
diff --git a/tests/requirements.txt b/tests/requirements.txt
new file mode 100644
index 0000000000..ce459414c4
--- /dev/null
+++ b/tests/requirements.txt
@@ -0,0 +1,5 @@
+taospy
+numpy
+fabric2
+psutil
+pandas
diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim
index 66d3c48f5d..71f7969c83 100644
--- a/tests/script/tsim/query/explain.sim
+++ b/tests/script/tsim/query/explain.sim
@@ -25,15 +25,15 @@ sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=2
sql create table tb4 using st1 tags(4);
sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)");
-sql create table tb1 using st2 tags(1);
-sql insert into tb1 values (now, 1, "Hash Join (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)");
+#sql create table tb1 using st2 tags(1);
+#sql insert into tb1 values (now, 1, "Hash Join (cost=230.47..713.98 rows=101 width=488) (actual time=0.711..7.427 rows=100 loops=1)");
-sql create table tb2 using st2 tags(2);
-sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2 (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)");
-sql create table tb3 using st2 tags(3);
-sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)");
-sql create table tb4 using st2 tags(4);
-sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)");
+#sql create table tb2 using st2 tags(2);
+#sql insert into tb2 values (now, 2, "Seq Scan on tenk2 t2 (cost=0.00..445.00 rows=10000 width=244) (actual time=0.007..2.583 rows=10000 loops=1)");
+#sql create table tb3 using st2 tags(3);
+#sql insert into tb3 values (now, 3, "Hash (cost=229.20..229.20 rows=101 width=244) (actual time=0.659..0.659 rows=100 loops=1)");
+#sql create table tb4 using st2 tags(4);
+#sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)");
print ======== step2
diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim
index 80d2dafa84..2946a89ff6 100644
--- a/tests/script/tsim/query/scalarFunction.sim
+++ b/tests/script/tsim/query/scalarFunction.sim
@@ -102,20 +102,26 @@ print ====> $data60 $data61 $data62 $data63 $data64 $data65
print ====> $data70 $data71 $data72 $data73 $data74 $data75
print ====> $data80 $data81 $data82 $data83 $data84 $data85
print ====> $data90 $data91 $data92 $data93 $data94 $data95
+print ====> rows = $rows and rowNum = $rowNum for ct1
if $rows != $rowNum then
return -1
endi
+
print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb
sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb
+print ====> rows = $rows and totalRows = $totalRows for stb
if $rows != $totalRows then
return -1
endi
+
print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb
sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb
+print ====> rows = $rows and rowNum = $rowNum for ntb
if $rows != $rowNum then
return -1
endi
+
print ====> log
sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1
print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1
diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim
index a69b6249fc..c39956c0df 100644
--- a/tests/script/tsim/query/session.sim
+++ b/tests/script/tsim/query/session.sim
@@ -65,6 +65,15 @@ sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.031', 5)
sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.036', 6)
sql INSERT INTO dev_002 VALUES('2020-05-13 10:00:00.51', 7)
+# vnode does not return the precision of the table
+print ====> create database d1 precision 'us'
+sql create database d1 precision 'us'
+sql use d1
+sql create table dev_001 (ts timestamp ,i timestamp ,j int)
+sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
+sql create table secondts(ts timestamp,t2 timestamp,i int)
+sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
+
$loop_test = 0
loop_test_pos:
@@ -288,15 +297,6 @@ sql_error sql select count(*) from dev_001 session(ts,0s)
sql_error select count(*) from dev_001 session(i,1y)
sql_error select count(*) from dev_001 session(ts,1d) where ts <'2020-05-20 0:0:0'
-# vnode does not return the precision of the table
-print ====> create database d1 precision 'us'
-sql create database d1 precision 'us'
-sql use d1
-sql create table dev_001 (ts timestamp ,i timestamp ,j int)
-sql insert into dev_001 values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
-sql create table secondts(ts timestamp,t2 timestamp,i int)
-sql insert into secondts values(1623046993681000,now,1)(1623046993681001,now+1s,2)(1623046993681002,now+2s,3)(1623046993681004,now+5s,4)
-
#print ====> select count(*) from dev_001 session(ts,1u)
#sql select _wstartts, count(*) from dev_001 session(ts,1u)
#print rows: $rows
diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim
index 8acd07cfe4..c76569b40f 100644
--- a/tests/script/tsim/query/udf.sim
+++ b/tests/script/tsim/query/udf.sim
@@ -43,6 +43,27 @@ if $data00 != 2.236067977 then
return -1
endi
+sql create table t2 (ts timestamp, f1 int, f2 int);
+sql insert into t2 values(now, 0, 0)(now+1s, 1, 1);
+sql select udf1(f1, f2) from t2;
+if $rows != 2 then
+ return -1
+endi
+if $data00 != 88 then
+ return -1
+endi
+if $data10 != 88 then
+ return -1
+endi
+
+sql select udf2(f1, f2) from t2;
+if $rows != 1 then
+ return -1
+endi
+if $data00 != 1.414213562 then
+ return -1
+endi
+
#sql drop function udf1;
#sql drop function udf2;
system sh/exec.sh -n dnode1 -s stop -x SIGKILL
diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim
index be3b718fae..913ced74aa 100644
--- a/tests/script/tsim/table/basic1.sim
+++ b/tests/script/tsim/table/basic1.sim
@@ -91,9 +91,9 @@ print =============== create normal table
sql create database ndb
sql use ndb
sql create table nt0 (ts timestamp, i int)
-sql create table if not exists nt0 (ts timestamp, i int)
+# sql create table if not exists nt0 (ts timestamp, i int)
sql create table nt1 (ts timestamp, i int)
-sql create table if not exists nt1 (ts timestamp, i int)
+# sql create table if not exists nt1 (ts timestamp, i int)
sql create table if not exists nt3 (ts timestamp, i int)
sql show tables
diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py
index b4878e42c9..a39bc21946 100644
--- a/tests/system-test/2-query/join.py
+++ b/tests/system-test/2-query/join.py
@@ -5,6 +5,7 @@ from util.sql import *
from util.cases import *
from util.dnodes import *
+PRIMARY_COL = "ts"
INT_COL = "c1"
BINT_COL = "c2"
@@ -18,9 +19,10 @@ BINARY_COL = "c8"
NCHAR_COL = "c9"
TS_COL = "c10"
-UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ]
+NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
CHAR_COL = [ BINARY_COL, NCHAR_COL, ]
-TS_TYPE_COL = [TS_COL]
+BOOLEAN_COL = [ BOOL_COL, ]
+TS_TYPE_COL = [ TS_COL, ]
class TDTestCase:
@@ -28,88 +30,171 @@ class TDTestCase:
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
- def __length_condition(self):
- length_condition = []
+ def __query_condition(self,tbname):
+ query_condition = []
for char_col in CHAR_COL:
- length_condition.extend(
+ query_condition.extend(
(
- char_col,
- f"upper( {char_col} )",
+ f"{tbname}.{char_col}",
+ f"upper( {tbname}.{char_col} )",
)
)
- length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL)
- length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
- length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL )
+ query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
+ query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
+ query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
+ for num_col in NUM_COL:
+ query_condition.extend(
+ (
+ f"{tbname}.{num_col}",
+ f"sin( {tbname}.{num_col} )"
+ )
+ )
+ query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL )
- length_condition.append('''"test1234!@#$%^&*():'> 0 "
- return ""
+ def __join_condition(self, tb_list, filter=PRIMARY_COL):
+ # sourcery skip: flip-comparison
+ if 1 == len(tb_list):
+ join_filter = f"{tb_list[0]}.{filter} = {tb_list[0]}.{filter} "
+ elif 2 == len(tb_list):
+ join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} "
+ else:
+ join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} "
+ for i in range(1, len(tb_list)-1 ):
+ join_filter += f"and {tb_list[i]}.{filter} = {tb_list[i+1]}.{filter}"
- def __group_condition(self, col, having = ""):
+ return join_filter
+
+ def __where_condition(self, col, tbname):
+ if col in NUM_COL:
+ return f" abs( {tbname}.{col} ) >= 0"
+ elif col in CHAR_COL:
+ return f" lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
+ elif col in BOOLEAN_COL:
+ return f" {tbname}.{col} in (false, true) "
+ elif col in TS_TYPE_COL or col in PRIMARY_COL:
+ return f" cast( {tbname}.{col} as binary(16) ) is not null "
+ else:
+ return ""
+
+ def __group_condition(self, tbname, col, having = ""):
return f" group by {col} having {having}" if having else f" group by {col} "
- def __length_current_check(self, tbname):
- length_condition = self.__length_condition()
- for condition in length_condition:
- where_condition = self.__where_condition(condition)
- group_having = self.__group_condition(condition, having=f"{condition} is not null " )
- group_no_having= self.__group_condition(condition )
+ def __join_check(self, tblist, checkrows, join_flag=True):
+ query_conditions = self.__query_condition(tblist[0])
+ join_condition = self.__join_condition(tb_list=tblist) if join_flag else " "
+ for condition in query_conditions:
+ where_condition = self.__where_condition(col=condition, tbname=tblist[0])
+ group_having = self.__group_condition(tbname=tblist[0], col=condition, having=f"{condition} is not null " )
+ group_no_having= self.__group_condition(tbname=tblist[0], col=condition )
groups = ["", group_having, group_no_having]
-
for group_condition in groups:
- tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ")
- datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
- length_data = [ len(str(data)) if data else None for data in datas ]
- tdSql.query(f"select length( {condition} ) from {tbname} {where_condition} {group_condition}")
- for i in range(len(length_data)):
- tdSql.checkData(i, 0, length_data[i] ) if length_data[i] else tdSql.checkData(i, 0, None)
+ if where_condition:
+ sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition} "
+ else:
+ sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} {group_condition} "
- def __length_err_check(self,tbname):
- sqls = []
+ if not join_flag :
+ tdSql.error(sql=sql)
+ if len(tblist) == 2:
+ if "ct1" in tblist or "t1" in tblist:
+ self.__join_current(sql, checkrows)
+ elif where_condition or "not null" in group_condition:
+ self.__join_current(sql, checkrows + 2 )
+ elif group_condition:
+ self.__join_current(sql, checkrows + 3 )
+ else:
+ self.__join_current(sql, checkrows + 5 )
+ if len(tblist) > 2 or len(tblist) < 1:
+ tdSql.error(sql=sql)
- for un_char_col in UN_CHAR_COL:
- sqls.extend(
- (
- f"select length( {un_char_col} ) from {tbname} ",
- f"select length(ceil( {un_char_col} )) from {tbname} ",
- f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ",
- )
- )
+ # def __join_err_check(self,tbname):
+ # sqls = []
- sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL )
- sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
+ # for un_char_col in NUM_COL:
+ # sqls.extend(
+ # (
+ # f"select length( {un_char_col} ) from {tbname} ",
+ # f"select length(ceil( {un_char_col} )) from {tbname} ",
+ # f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ",
+ # )
+ # )
- sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL)
- sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
- sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL)
- sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL )
- sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL )
- sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL )
- sqls.extend(
- (
- f"select length() from {tbname} ",
- f"select length(*) from {tbname} ",
- f"select length(ccccccc) from {tbname} ",
- f"select length(111) from {tbname} ",
- f"select length(c8, 11) from {tbname} ",
- )
- )
+ # sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL )
+ # sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
+
+ # sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL)
+ # sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL )
+ # sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL)
+ # sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL )
+ # sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL )
+ # sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL )
+ # sqls.extend(
+ # (
+ # f"select length() from {tbname} ",
+ # f"select length(*) from {tbname} ",
+ # f"select length(ccccccc) from {tbname} ",
+ # f"select length(111) from {tbname} ",
+ # f"select length(c8, 11) from {tbname} ",
+ # )
+ # )
+
+ # return sqls
+
+ def __join_current(self, sql, checkrows):
+ tdSql.query(sql=sql)
+ tdSql.checkRows(checkrows)
- return sqls
def __test_current(self):
+ # sourcery skip: extract-duplicate-method, inline-immediately-returned-variable
tdLog.printNoPrefix("==========current sql condition check , must return query ok==========")
- tbname = ["ct1", "ct2", "ct4", "t1"]
- for tb in tbname:
- self.__length_current_check(tb)
- tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========")
+ tblist_1 = ["ct1", "ct2"]
+ self.__join_check(tblist_1, 1)
+ tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========")
+ tblist_2 = ["ct2", "ct4"]
+ self.__join_check(tblist_2, self.rows)
+ tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========")
+ tblist_3 = ["t1", "ct4"]
+ self.__join_check(tblist_3, 1)
+ tdLog.printNoPrefix(f"==========current sql condition check in {tblist_3} over==========")
+ tblist_4 = ["t1", "ct1"]
+ self.__join_check(tblist_4, 1)
+ tdLog.printNoPrefix(f"==========current sql condition check in {tblist_4} over==========")
def __test_error(self):
+ # sourcery skip: extract-duplicate-method, move-assign-in-block
tdLog.printNoPrefix("==========err sql condition check , must return error==========")
+ err_list_1 = ["ct1","ct2", "ct4"]
+ err_list_2 = ["ct1","ct2", "t1"]
+ err_list_3 = ["ct1","ct4", "t1"]
+ err_list_4 = ["ct2","ct4", "t1"]
+ err_list_5 = ["ct1", "ct2","ct4", "t1"]
+ self.__join_check(err_list_1, -1)
+ tdLog.printNoPrefix(f"==========err sql condition check in {err_list_1} over==========")
+ self.__join_check(err_list_2, -1)
+ tdLog.printNoPrefix(f"==========err sql condition check in {err_list_2} over==========")
+ self.__join_check(err_list_3, -1)
+ tdLog.printNoPrefix(f"==========err sql condition check in {err_list_3} over==========")
+ self.__join_check(err_list_4, -1)
+ tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========")
+ self.__join_check(err_list_5, -1)
+ tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========")
+ self.__join_check(["ct2", "ct4"], -1, join_flag=False)
+ tdLog.printNoPrefix("==========err sql condition check in has no join condition over==========")
+
+ tdSql.error( f"select c1, c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" )
+ tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{INT_COL}=ct4.{INT_COL}" )
+ tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{TS_COL}=ct4.{TS_COL}" )
+ tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{TS_COL}" )
+ tdSql.error( f"select ct2.c1, ct1.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" )
+ tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and c1 is not null " )
+ tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and ct1.c1 is not null " )
+
+
tbname = ["ct1", "ct2", "ct4", "t1"]
for tb in tbname:
@@ -150,33 +235,33 @@ class TDTestCase:
now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
for i in range(rows):
tdSql.execute(
- f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )"
+ f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
)
tdSql.execute(
- f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )"
+ f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
)
tdSql.execute(
- f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )"
+ f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
)
tdSql.execute(
f'''insert into ct1 values
- ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } )
- ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } )
+ ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
+ ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
'''
)
tdSql.execute(
f'''insert into ct4 values
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
- ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{ now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
- { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000}
+ { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
)
(
{ now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
- { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000}
+ { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
)
'''
)
@@ -184,15 +269,15 @@ class TDTestCase:
tdSql.execute(
f'''insert into ct2 values
( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
- ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
(
{ now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
- { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 }
+ { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
)
(
{ now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
- { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 }
+ { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
)
'''
)
@@ -200,7 +285,7 @@ class TDTestCase:
for i in range(rows):
insert_data = f'''insert into t1 values
( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
- "binary_{i}", "nchar_{i}", { now_time - 1000 * i } )
+ "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
'''
tdSql.execute(insert_data)
tdSql.execute(
@@ -210,12 +295,12 @@ class TDTestCase:
( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
{ 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
- "binary_limit-1", "nchar_limit-1", { now_time - 86400000 }
+ "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
)
(
{ now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
{ 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
- "binary_limit-2", "nchar_limit-2", { now_time - 172800000 }
+ "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
)
'''
)
@@ -228,7 +313,8 @@ class TDTestCase:
self.__create_tb()
tdLog.printNoPrefix("==========step2:insert data")
- self.__insert_data(10)
+ self.rows = 10
+ self.__insert_data(self.rows)
tdLog.printNoPrefix("==========step3:all check")
self.all_test()
diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py
new file mode 100644
index 0000000000..28e869e044
--- /dev/null
+++ b/tests/system-test/2-query/sqrt.py
@@ -0,0 +1,551 @@
+import taos
+import sys
+import datetime
+import inspect
+import math
+from util.log import *
+from util.sql import *
+from util.cases import *
+
+
+class TDTestCase:
+ updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 ,
+ "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143,
+ "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143}
+ def init(self, conn, powSql):
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor())
+
+ def prepare_datas(self):
+ tdSql.execute(
+ '''create table stb1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ tags (t1 int)
+ '''
+ )
+
+ tdSql.execute(
+ '''
+ create table t1
+ (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+ '''
+ )
+ for i in range(4):
+ tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+ for i in range(9):
+ tdSql.execute(
+ f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+ )
+ tdSql.execute(
+ f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+ )
+ tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
+ tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+ tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
+ tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+
+ tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+ tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+ tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+
+ tdSql.execute(
+ f'''insert into t1 values
+ ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
+ ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
+ ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
+ ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
+ ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
+ ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
+ ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
+ ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
+ ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
+ ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+ '''
+ )
+
+ def check_result_auto_sqrt(self ,origin_query , pow_query):
+
+ pow_result = tdSql.getResult(pow_query)
+ origin_result = tdSql.getResult(origin_query)
+
+ auto_result =[]
+
+ for row in origin_result:
+ row_check = []
+ for elem in row:
+ if elem == None:
+ elem = None
+ elif elem < 0:
+ elem = None
+ else:
+ elem = math.sqrt(elem)
+ row_check.append(elem)
+ auto_result.append(row_check)
+
+ check_status = True
+
+ for row_index , row in enumerate(pow_result):
+ for col_index , elem in enumerate(row):
+ if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None):
+ check_status = False
+ elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001):
+ check_status = False
+ else:
+ pass
+ if not check_status:
+ tdLog.notice("sqrt function value has not as expected , sql is \"%s\" "%pow_query )
+ sys.exit(1)
+ else:
+ tdLog.info("sqrt value check pass , it work as expected ,sql is \"%s\" "%pow_query )
+
+ def test_errors(self):
+ error_sql_lists = [
+ "select sqrt from t1",
+ # "select sqrt(-+--+c1 ) from t1",
+ # "select +-sqrt(c1) from t1",
+ # "select ++-sqrt(c1) from t1",
+ # "select ++--sqrt(c1) from t1",
+ # "select - -sqrt(c1)*0 from t1",
+ # "select sqrt(tbname+1) from t1 ",
+ "select sqrt(123--123)==1 from t1",
+ "select sqrt(c1) as 'd1' from t1",
+ "select sqrt(c1 ,c2) from t1",
+ "select sqrt(c1 ,NULL ) from t1",
+ "select sqrt(,) from t1;",
+ "select sqrt(sqrt(c1) ab from t1)",
+ "select sqrt(c1 ) as int from t1",
+ "select sqrt from stb1",
+ # "select sqrt(-+--+c1) from stb1",
+ # "select +-sqrt(c1) from stb1",
+ # "select ++-sqrt(c1) from stb1",
+ # "select ++--sqrt(c1) from stb1",
+ # "select - -sqrt(c1)*0 from stb1",
+ # "select sqrt(tbname+1) from stb1 ",
+ "select sqrt(123--123)==1 from stb1",
+ "select sqrt(c1) as 'd1' from stb1",
+ "select sqrt(c1 ,c2 ) from stb1",
+ "select sqrt(c1 ,NULL) from stb1",
+ "select sqrt(,) from stb1;",
+ "select sqrt(sqrt(c1) ab from stb1)",
+ "select sqrt(c1) as int from stb1"
+ ]
+ for error_sql in error_sql_lists:
+ tdSql.error(error_sql)
+
+ def support_types(self):
+ type_error_sql_lists = [
+ "select sqrt(ts) from t1" ,
+ "select sqrt(c7) from t1",
+ "select sqrt(c8) from t1",
+ "select sqrt(c9) from t1",
+ "select sqrt(ts) from ct1" ,
+ "select sqrt(c7) from ct1",
+ "select sqrt(c8) from ct1",
+ "select sqrt(c9) from ct1",
+ "select sqrt(ts) from ct3" ,
+ "select sqrt(c7) from ct3",
+ "select sqrt(c8) from ct3",
+ "select sqrt(c9) from ct3",
+ "select sqrt(ts) from ct4" ,
+ "select sqrt(c7) from ct4",
+ "select sqrt(c8) from ct4",
+ "select sqrt(c9) from ct4",
+ "select sqrt(ts) from stb1" ,
+ "select sqrt(c7) from stb1",
+ "select sqrt(c8) from stb1",
+ "select sqrt(c9) from stb1" ,
+
+ "select sqrt(ts) from stbbb1" ,
+ "select sqrt(c7) from stbbb1",
+
+ "select sqrt(ts) from tbname",
+ "select sqrt(c9) from tbname"
+
+ ]
+
+ for type_sql in type_error_sql_lists:
+ tdSql.error(type_sql)
+
+
+ type_sql_lists = [
+ "select sqrt(c1) from t1",
+ "select sqrt(c2) from t1",
+ "select sqrt(c3) from t1",
+ "select sqrt(c4) from t1",
+ "select sqrt(c5) from t1",
+ "select sqrt(c6) from t1",
+
+ "select sqrt(c1) from ct1",
+ "select sqrt(c2) from ct1",
+ "select sqrt(c3) from ct1",
+ "select sqrt(c4) from ct1",
+ "select sqrt(c5) from ct1",
+ "select sqrt(c6) from ct1",
+
+ "select sqrt(c1) from ct3",
+ "select sqrt(c2) from ct3",
+ "select sqrt(c3) from ct3",
+ "select sqrt(c4) from ct3",
+ "select sqrt(c5) from ct3",
+ "select sqrt(c6) from ct3",
+
+ "select sqrt(c1) from stb1",
+ "select sqrt(c2) from stb1",
+ "select sqrt(c3) from stb1",
+ "select sqrt(c4) from stb1",
+ "select sqrt(c5) from stb1",
+ "select sqrt(c6) from stb1",
+
+ "select sqrt(c6) as alisb from stb1",
+ "select sqrt(c6) alisb from stb1",
+ ]
+
+ for type_sql in type_sql_lists:
+ tdSql.query(type_sql)
+
+ def basic_sqrt_function(self):
+
+ # basic query
+ tdSql.query("select c1 from ct3")
+ tdSql.checkRows(0)
+ tdSql.query("select c1 from t1")
+ tdSql.checkRows(12)
+ tdSql.query("select c1 from stb1")
+ tdSql.checkRows(25)
+
+ # used for empty table , ct3 is empty
+ tdSql.query("select sqrt(c1) from ct3")
+ tdSql.checkRows(0)
+ tdSql.query("select sqrt(c2) from ct3")
+ tdSql.checkRows(0)
+ tdSql.query("select sqrt(c3) from ct3")
+ tdSql.checkRows(0)
+ tdSql.query("select sqrt(c4) from ct3")
+ tdSql.checkRows(0)
+ tdSql.query("select sqrt(c5) from ct3")
+ tdSql.checkRows(0)
+ tdSql.query("select sqrt(c6) from ct3")
+ tdSql.checkRows(0)
+
+
+ # # used for regular table
+ tdSql.query("select sqrt(c1) from t1")
+ tdSql.checkData(0, 0, None)
+ tdSql.checkData(1 , 0, 1.000000000)
+ tdSql.checkData(3 , 0, 1.732050808)
+ tdSql.checkData(5 , 0, None)
+
+ tdSql.query("select c1, c2, c3 , c4, c5 from t1")
+ tdSql.checkData(1, 4, 1.11000)
+ tdSql.checkData(3, 3, 33)
+ tdSql.checkData(5, 4, None)
+
+ tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1")
+ tdSql.checkData(1, 5, 1.11000)
+ tdSql.checkData(3, 4, 33)
+ tdSql.checkData(5, 5, None)
+
+ self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from t1")
+
+ # used for sub table
+ tdSql.query("select c2 ,sqrt(c2) from ct1")
+ tdSql.checkData(0, 1, 298.140906284)
+ tdSql.checkData(1 , 1, 278.885281074)
+ tdSql.checkData(3 , 1, 235.701081881)
+ tdSql.checkData(4 , 1, 0.000000000)
+
+ tdSql.query("select c1, c5 ,sqrt(c5) from ct4")
+ tdSql.checkData(0 , 2, None)
+ tdSql.checkData(1 , 2, 2.979932904)
+ tdSql.checkData(2 , 2, 2.787471970)
+ tdSql.checkData(3 , 2, 2.580697551)
+ tdSql.checkData(5 , 2, None)
+
+ self.check_result_auto_sqrt( "select c1, c2, c3 , c4, c5 from ct1", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c4), sqrt(c5) from ct1")
+
+ # nest query for sqrt functions
+ tdSql.query("select c4 , sqrt(c4) ,sqrt(sqrt(c4)) , sqrt(sqrt(sqrt(c4))) from ct1;")
+ tdSql.checkData(0 , 0 , 88)
+ tdSql.checkData(0 , 1 , 9.380831520)
+ tdSql.checkData(0 , 2 , 3.062814314)
+ tdSql.checkData(0 , 3 , 1.750089802)
+
+ tdSql.checkData(1 , 0 , 77)
+ tdSql.checkData(1 , 1 , 8.774964387)
+ tdSql.checkData(1 , 2 , 2.962256638)
+ tdSql.checkData(1 , 3 , 1.721120750)
+
+ tdSql.checkData(11 , 0 , -99)
+ tdSql.checkData(11 , 1 , None)
+ tdSql.checkData(11 , 2 , None)
+ tdSql.checkData(11 , 3 , None)
+
+ # used for stable table
+
+ tdSql.query("select sqrt(c1) from stb1")
+ tdSql.checkRows(25)
+
+
+ # used for not exists table
+ tdSql.error("select sqrt(c1) from stbbb1")
+ tdSql.error("select sqrt(c1) from tbname")
+ tdSql.error("select sqrt(c1) from ct5")
+
+ # mix with common col
+ tdSql.query("select c1, sqrt(c1) from ct1")
+ tdSql.checkData(0 , 0 ,8)
+ tdSql.checkData(0 , 1 ,2.828427125)
+ tdSql.checkData(4 , 0 ,0)
+ tdSql.checkData(4 , 1 ,0.000000000)
+ tdSql.query("select c2, sqrt(c2) from ct4")
+ tdSql.checkData(0 , 0 , None)
+ tdSql.checkData(0 , 1 ,None)
+ tdSql.checkData(4 , 0 ,55555)
+ tdSql.checkData(4 , 1 ,235.701081881)
+ tdSql.checkData(5 , 0 ,None)
+ tdSql.checkData(5 , 1 ,None)
+
+ # mix with common functions
+ tdSql.query("select c1, sqrt(c1),sqrt(c1), sqrt(sqrt(c1)) from ct4 ")
+ tdSql.checkData(0 , 0 ,None)
+ tdSql.checkData(0 , 1 ,None)
+ tdSql.checkData(0 , 2 ,None)
+ tdSql.checkData(0 , 3 ,None)
+
+ tdSql.checkData(3 , 0 , 6)
+ tdSql.checkData(3 , 1 ,2.449489743)
+ tdSql.checkData(3 , 2 ,2.449489743)
+ tdSql.checkData(3 , 3 ,1.565084580)
+
+ tdSql.query("select c1, sqrt(c1),c5, floor(c5) from stb1 ")
+
+ # # mix with agg functions , not support
+ tdSql.error("select c1, sqrt(c1),c5, count(c5) from stb1 ")
+ tdSql.error("select c1, sqrt(c1),c5, count(c5) from ct1 ")
+ tdSql.error("select sqrt(c1), count(c5) from stb1 ")
+ tdSql.error("select sqrt(c1), count(c5) from ct1 ")
+ tdSql.error("select c1, count(c5) from ct1 ")
+ tdSql.error("select c1, count(c5) from stb1 ")
+
+ # agg functions mix with agg functions
+
+ tdSql.query("select max(c5), count(c5) from stb1")
+ tdSql.query("select max(c5), count(c5) from ct1")
+
+
+ # bug fix for count
+ tdSql.query("select count(c1) from ct4 ")
+ tdSql.checkData(0,0,9)
+ tdSql.query("select count(*) from ct4 ")
+ tdSql.checkData(0,0,12)
+ tdSql.query("select count(c1) from stb1 ")
+ tdSql.checkData(0,0,22)
+ tdSql.query("select count(*) from stb1 ")
+ tdSql.checkData(0,0,25)
+
+ # # bug fix for compute
+ tdSql.query("select c1, sqrt(c1) -0 ,sqrt(c1-4)-0 from ct4 ")
+ tdSql.checkData(0, 0, None)
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(0, 2, None)
+ tdSql.checkData(1, 0, 8)
+ tdSql.checkData(1, 1, 2.828427125)
+ tdSql.checkData(1, 2, 2.000000000)
+
+ tdSql.query(" select c1, sqrt(c1) -0 ,sqrt(c1-0.1)-0.1 from ct4")
+ tdSql.checkData(0, 0, None)
+ tdSql.checkData(0, 1, None)
+ tdSql.checkData(0, 2, None)
+ tdSql.checkData(1, 0, 8)
+ tdSql.checkData(1, 1, 2.828427125)
+ tdSql.checkData(1, 2, 2.710693865)
+
+ tdSql.query("select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1")
+
+ def test_big_number(self):
+
+ tdSql.query("select c1, sqrt(100000000) from ct1") # bigint to double data overflow
+ tdSql.checkData(4, 1, 10000.000000000)
+
+
+ tdSql.query("select c1, sqrt(10000000000000) from ct1") # bigint to double data overflow
+ tdSql.checkData(4, 1, 3162277.660168380)
+
+ tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000) from ct1") # bigint to double data overflow
+ tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value
+ tdSql.checkData(1, 1, 3162277660171.025390625)
+
+ tdSql.query("select c1, sqrt(10000000000000000000000000000000000) from ct1") # bigint to double data overflow
+ tdSql.query("select c1, sqrt(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value
+ tdSql.checkData(4, 1, 100000000000000000.000000000)
+
+ tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow
+ tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value
+
+ tdSql.checkData(4, 1, 100000000000000000000.000000000)
+
+ tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow
+
+ def pow_base_test(self):
+
+ # base is an regular number ,int or double
+ tdSql.query("select c1, sqrt(c1) from ct1")
+ tdSql.checkData(0, 1,2.828427125)
+ tdSql.checkRows(13)
+
+ # # bug for compute in functions
+ # tdSql.query("select c1, abs(1/0) from ct1")
+ # tdSql.checkData(0, 0, 8)
+ # tdSql.checkData(0, 1, 1)
+
+ tdSql.query("select c1, sqrt(1) from ct1")
+ tdSql.checkData(0, 1, 1.000000000)
+ tdSql.checkRows(13)
+
+ # two cols start sqrt(x,y)
+ tdSql.query("select c1,c2, sqrt(c2) from ct1")
+ tdSql.checkData(0, 2, 298.140906284)
+ tdSql.checkData(1, 2, 278.885281074)
+ tdSql.checkData(4, 2, 0.000000000)
+
+ def abs_func_filter(self):
+ tdSql.execute("use db")
+ tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1>5 ")
+ tdSql.checkRows(3)
+ tdSql.checkData(0,0,8)
+ tdSql.checkData(0,1,8.000000000)
+ tdSql.checkData(0,2,8.000000000)
+ tdSql.checkData(0,3,7.900000000)
+ tdSql.checkData(0,4,3.000000000)
+
+ tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ")
+ tdSql.checkRows(1)
+ tdSql.checkData(0,0,5)
+ tdSql.checkData(0,1,5.000000000)
+ tdSql.checkData(0,2,5.000000000)
+ tdSql.checkData(0,3,4.900000000)
+ tdSql.checkData(0,4,2.000000000)
+
+ tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ")
+ tdSql.checkRows(1)
+ tdSql.checkData(0,0,5)
+ tdSql.checkData(0,1,5.000000000)
+ tdSql.checkData(0,2,5.000000000)
+ tdSql.checkData(0,3,4.900000000)
+ tdSql.checkData(0,4,2.000000000)
+
+ tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=sqrt(c1) limit 1 ")
+ tdSql.checkRows(1)
+ tdSql.checkData(0,0,1)
+ tdSql.checkData(0,1,11111)
+ tdSql.checkData(0,2,1.000000000)
+ tdSql.checkData(0,3,1.000000000)
+ tdSql.checkData(0,4,0.900000000)
+ tdSql.checkData(0,5,1.000000000)
+
+ def pow_Arithmetic(self):
+ pass
+
+ def check_boundary_values(self):
+
+ tdSql.execute("drop database if exists bound_test")
+ tdSql.execute("create database if not exists bound_test")
+ time.sleep(3)
+ tdSql.execute("use bound_test")
+ tdSql.execute(
+ "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);"
+ )
+ tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )')
+ tdSql.execute(
+ f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+ )
+ tdSql.execute(
+ f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+ )
+ tdSql.execute(
+ f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+ )
+ tdSql.execute(
+ f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+ )
+ tdSql.error(
+ f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
+ )
+ self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from sub1_bound")
+
+ self.check_result_auto_sqrt( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c3), sqrt(c2) ,sqrt(c1) from sub1_bound")
+
+ self.check_result_auto_sqrt("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select sqrt(abs(c1)) from sub1_bound" )
+
+ # check basic elem for table per row
+ tdSql.query("select sqrt(abs(c1)) ,sqrt(abs(c2)) , sqrt(abs(c3)) , sqrt(abs(c4)), sqrt(abs(c5)), sqrt(abs(c6)) from sub1_bound ")
+ tdSql.checkData(0,0,math.sqrt(2147483647))
+ tdSql.checkData(0,1,math.sqrt(9223372036854775807))
+ tdSql.checkData(0,2,math.sqrt(32767))
+ tdSql.checkData(0,3,math.sqrt(127))
+ tdSql.checkData(0,4,math.sqrt(339999995214436424907732413799364296704.00000))
+ tdSql.checkData(1,0,math.sqrt(2147483647))
+ tdSql.checkData(1,1,math.sqrt(9223372036854775807))
+ tdSql.checkData(1,2,math.sqrt(32767))
+ tdSql.checkData(1,3,math.sqrt(127))
+ tdSql.checkData(1,4,math.sqrt(339999995214436424907732413799364296704.00000))
+ tdSql.checkData(3,0,math.sqrt(2147483646))
+ tdSql.checkData(3,1,math.sqrt(9223372036854775806))
+ tdSql.checkData(3,2,math.sqrt(32766))
+ tdSql.checkData(3,3,math.sqrt(126))
+ tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000))
+
+ # check + - * / in functions
+ tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ")
+ tdSql.checkData(0,0,math.sqrt(2147483648.000000000))
+ tdSql.checkData(0,1,math.sqrt(9223372036854775807))
+ tdSql.checkData(0,2,math.sqrt(32767.000000000))
+ tdSql.checkData(0,3,math.sqrt(63.500000000))
+
+
+ def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
+ tdSql.prepare()
+
+ tdLog.printNoPrefix("==========step1:create table ==============")
+
+ self.prepare_datas()
+
+ tdLog.printNoPrefix("==========step2:test errors ==============")
+
+ self.test_errors()
+
+ tdLog.printNoPrefix("==========step3:support types ============")
+
+ self.support_types()
+
+ tdLog.printNoPrefix("==========step4: sqrt basic query ============")
+
+ self.basic_sqrt_function()
+
+ tdLog.printNoPrefix("==========step5: big number sqrt query ============")
+
+ self.test_big_number()
+
+ tdLog.printNoPrefix("==========step6: base number for sqrt query ============")
+
+ self.pow_base_test()
+
+ tdLog.printNoPrefix("==========step7: sqrt boundary query ============")
+
+ self.check_boundary_values()
+
+ tdLog.printNoPrefix("==========step8: sqrt filter query ============")
+
+ self.abs_func_filter()
+
+
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 1dfb160987..51f1649cc8 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -24,3 +24,4 @@ python3 ./test.py -f 2-query/floor.py
python3 ./test.py -f 2-query/round.py
python3 ./test.py -f 2-query/log.py
python3 ./test.py -f 2-query/pow.py
+python3 ./test.py -f 2-query/sqrt.py
diff --git a/tools/taos-tools b/tools/taos-tools
index 2f3dfddd4d..59e0ebaf49 160000
--- a/tools/taos-tools
+++ b/tools/taos-tools
@@ -1 +1 @@
-Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c
+Subproject commit 59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20