From 8be56fb73c8573bd9113908607909988a2f5d3a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 11:35:04 +0800 Subject: [PATCH 01/55] [td-1148] enable the binary tag to be used as the join condition. --- src/client/src/tscFunctionImpl.c | 6 +- src/client/src/tscSQLParser.c | 5 -- src/client/src/tscSubquery.c | 9 +-- src/common/inc/tvariant.h | 2 + src/common/src/tvariant.c | 34 +++++++++- src/query/inc/qExecutor.h | 2 +- src/query/inc/qTsbuf.h | 21 ++++--- src/query/src/qExecutor.c | 10 ++- src/query/src/qTsbuf.c | 66 +++++++++++++------- src/query/tests/tsBufTest.cpp | 104 ++++++++++++++++++++++--------- 10 files changed, 176 insertions(+), 83 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 5c708dffee..205be5e3a6 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3900,11 +3900,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE); } } @@ -3923,7 +3923,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f933366711..43a43b6d31 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3064,7 +3064,6 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { const char* msg1 = "invalid join query condition"; - const char* msg2 = "join on binary/nchar not supported"; const char* msg3 = "type of join columns must be identical"; const char* msg4 = "invalid column name in join condition"; @@ -3108,10 +3107,6 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (pTagSchema1->type == TSDB_DATA_TYPE_BINARY || pTagSchema1->type == TSDB_DATA_TYPE_NCHAR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - pTagCond->joinInfo.hasJoin = true; return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 75644c355c..eedb229e1c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -93,13 +93,14 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); #endif - if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem1.ts, elem2.ts))) { + int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); + if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { if (!tsBufNextPos(pSupporter1->pTSBuf)) { break; } numOfInput1++; - } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem2.ts, elem1.ts))) { + } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { if (!tsBufNextPos(pSupporter2->pTSBuf)) { break; } @@ -119,8 +120,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ win->ekey = elem1.ts; } - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); } else { pLimit->offset -= 1; } diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 4fd6ea5541..211635b4af 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -46,6 +46,8 @@ void tVariantDestroy(tVariant *pV); void tVariantAssign(tVariant *pDst, const tVariant *pSrc); +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2); + int32_t tVariantToString(tVariant *pVar, char *dst); int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 5460d21252..33f8eb127f 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -12,12 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "os.h" #include "tvariant.h" #include "hash.h" -#include "hashfunc.h" -#include "os.h" -#include "hash.h" #include "taos.h" #include "taosdef.h" #include "tstoken.h" @@ -172,6 +170,36 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } } +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { + assert((p1->nType != TSDB_DATA_TYPE_NULL) || (p2->nType != TSDB_DATA_TYPE_NULL)); + + switch (p1->nType) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + if (p1->nLen == p2->nLen) { + return memcmp(p1->pz, p2->pz, p1->nLen); + } else { + return p1->nLen > p2->nLen? 1:-1; + } + }; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (p1->dKey == p2->dKey) { + return 0; + } else { + return p1->dKey > p2->dKey? 1:-1; + } + + default: + if (p1->i64Key == p2->i64Key) { + return 0; + } else { + return p1->i64Key > p2->i64Key? 1:-1; + } + } +} + int32_t tVariantToString(tVariant *pVar, char *dst) { if (pVar == NULL || dst == NULL) return 0; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b5487561b2..d824a269c9 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -98,7 +98,7 @@ typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query - int64_t tag; + tVariant tag; STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 59b224e096..f6d672b26d 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -22,6 +22,7 @@ extern "C" { #include "os.h" #include "taosdef.h" +#include "tvariant.h" #define MEM_BUF_SIZE (1 << 20) #define TS_COMP_FILE_MAGIC 0x87F5EC4C @@ -42,9 +43,9 @@ typedef struct STSRawBlock { } STSRawBlock; typedef struct STSElem { - TSKEY ts; - int64_t tag; - int32_t vnode; + TSKEY ts; + tVariant tag; + int32_t vnode; } STSElem; typedef struct STSCursor { @@ -55,11 +56,11 @@ typedef struct STSCursor { } STSCursor; typedef struct STSBlock { - int64_t tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed + tVariant tag; // tag value + int32_t numOfElem; // number of elements + int32_t compLen; // size after compressed + int32_t padding; // 0xFFFFFFFF by default, after the payload + char* payload; // actual data that is compressed } STSBlock; /* @@ -109,7 +110,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ void* tsBufDestroy(STSBuf* pTSBuf); -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); STSBuf* tsBufClone(STSBuf* pTSBuf); @@ -122,7 +123,7 @@ void tsBufResetPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); bool tsBufNextPos(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag); +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c8efee03cd..52408f78bf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1103,7 +1103,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first - if (pCtx[0].tag.i64Key != elem.tag) { + if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) { return TS_JOIN_TAG_NOT_EQUALS; } @@ -3644,9 +3644,8 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { if (pTableQueryInfo->cur.vgroupIndex == -1) { - pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; - - tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); + tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag); + tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag); // keep the cursor info of current meter pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; @@ -4501,8 +4500,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vgroupIndex == -1) { - int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag); // failed to find data with the specified tag value if (elem.vnode < 0) { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 25eb33ff7d..eb47a1b7a1 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -8,7 +8,6 @@ static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInf static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - /** * todo error handling * support auto closeable tmp file @@ -225,11 +224,12 @@ static void writeDataToDisk(STSBuf* pTSBuf) { } STSBlock* pBlock = &pTSBuf->block; - - pBlock->numOfElem = pTSBuf->tsData.len / TSDB_KEYSIZE; + STSList* pTsData = &pTSBuf->tsData; + + pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; pBlock->compLen = - tsCompressTimestamp(pTSBuf->tsData.rawBuf, pTSBuf->tsData.len, pTSBuf->tsData.len / TSDB_KEYSIZE, pBlock->payload, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, + TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); UNUSED(r); @@ -241,13 +241,18 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * * both side has the compressed length is used to support load data forwards/backwords. */ - fwrite(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); + fwrite(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + fwrite(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + fwrite(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + fwrite(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + } + fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; @@ -298,9 +303,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } - - size_t sz = fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); - UNUSED(sz); + + fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + + // NOTE: mix types tags are not supported + size_t sz = 0; + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + assert(tp != NULL); + + memset(tp, 0, pBlock->tag.nLen + 1); + pBlock->tag.pz = tp; + + sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + } + sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); UNUSED(sz); sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); @@ -361,7 +381,7 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { return TSDB_CODE_SUCCESS; } -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len) { +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) { STSVnodeBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; @@ -375,15 +395,15 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData } assert(pBlockInfo->info.vnode == vnodeId); - - if (pTSBuf->block.tag != tag && ptsData->len > 0) { + + if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first writeDataToDisk(pTSBuf); } else { expandBuffer(ptsData, len); } - pTSBuf->block.tag = tag; + tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); // todo check return value @@ -465,7 +485,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int return 0; } -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int64_t tag) { +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) { bool decomp = false; int64_t offset = 0; @@ -484,7 +504,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo return -1; } - if (pTSBuf->block.tag == tag) { + if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { return i; } } @@ -669,8 +689,8 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = pBlock->tag; - + tVariantAssign(&elem1.tag, &pBlock->tag); + return elem1; } @@ -800,7 +820,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ return pTSBuf; } -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { STSElem elem = {.vnode = -1}; if (pTSBuf == NULL) { @@ -881,7 +901,9 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag, elem.ts); + if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) { + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + } } pTSBuf->cur.order = old; diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f8738eec9c..b78c5314f2 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -32,14 +32,16 @@ void simpleTest() { // write 10 ts points int32_t num = 10; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(10, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -57,14 +59,16 @@ void largeTSTest() { // write 10 ts points int32_t num = 1000000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(num, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); // the data has been flush to disk, no data in cache EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -80,14 +84,18 @@ void multiTagsTest() { STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); int32_t num = 10000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int64_t start = 10000000; int32_t numOfTags = 50; int32_t step = 30; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -96,7 +104,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -118,9 +126,14 @@ void multiVnodeTagsTest() { for (int32_t j = 0; j < 20; ++j) { // vnodeId:0 start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -131,11 +144,11 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -157,9 +170,14 @@ void loadDataTest() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %" PRIu64 "\n", i, list[0]); free(list); @@ -172,11 +190,11 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -230,16 +248,21 @@ void TSTraverse() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); start += step * num; list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); @@ -272,12 +295,16 @@ void TSTraverse() { int32_t startVnode = 1; int32_t startTag = 2; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = startTag; + + tsBufGetElemStartPos(pTSBuf, startVnode, &t); int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -286,7 +313,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag == 0) { startVnode -= 1; @@ -316,13 +345,14 @@ void TSTraverse() { startVnode = 1; startTag = 2; + t.i64Key = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tsBufGetElemStartPos(pTSBuf, startVnode, &t); totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -331,7 +361,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag < 0) { startVnode -= 1; @@ -375,12 +407,17 @@ void mergeDiffVnodeBufferTest() { int32_t num = 1000; int32_t numOfTags = 10; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + // vnodeId:0 int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -403,6 +440,9 @@ void mergeIdenticalVnodeBufferTest() { STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int32_t step = 30; int32_t num = 1000; int32_t numOfTags = 10; @@ -411,17 +451,21 @@ void mergeIdenticalVnodeBufferTest() { int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); + t.i64Key = i; - tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -438,7 +482,7 @@ void mergeIdenticalVnodeBufferTest() { STSElem elem = tsBufGetElem(pTSBuf1); EXPECT_EQ(elem.vnode, 12); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); } tsBufDestroy(pTSBuf1); @@ -446,6 +490,8 @@ void mergeIdenticalVnodeBufferTest() { } } // namespace + +//TODO add binary tag value test case TEST(testCase, tsBufTest) { simpleTest(); largeTSTest(); @@ -453,7 +499,7 @@ TEST(testCase, tsBufTest) { multiVnodeTagsTest(); loadDataTest(); invalidFileTest(); - // randomIncTsTest(); +// randomIncTsTest(); TSTraverse(); mergeDiffVnodeBufferTest(); mergeIdenticalVnodeBufferTest(); From 9f940295d607fe709cf42c4eed6afe06df68e086 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 14:24:58 +0800 Subject: [PATCH 02/55] [td-225] fix bugs during stable query with the fill option. --- src/client/src/tscLocalMerge.c | 49 +++++-------------- tests/script/general/parser/lastrow_query.sim | 24 +++++++++ 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 186c2871a1..0a7bbac601 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -370,7 +370,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision); @@ -843,28 +843,6 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); } -static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, - SLocalReducer *pLocalReducer) { - assert(0); - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); - - char *src = pResPages[i]->data + (pRes->numOfRows - 1) * pField->bytes; - char *dst = pRes->data + pRes->numOfRows * offset; - - for (int32_t j = 0; j < pRes->numOfRows; ++j) { - memcpy(dst, src, (size_t)pField->bytes); - dst += pField->bytes; - src -= pField->bytes; - } - } -} - static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { assert(pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); @@ -925,7 +903,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SFillInfo *pFillInfo = pLocalReducer->pFillInfo; - int64_t actualETime = MAX(pQueryInfo->window.skey, pQueryInfo->window.ekey); + // todo extract function + int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -984,14 +963,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo); } - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); - } - } else { // todo bug?? - reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); } pRes->numOfRowsGroup += pRes->numOfRows; @@ -1249,15 +1224,15 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur // tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num); #endif - - // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { genFinalResWithoutFill(pRes, pLocalReducer, pQueryInfo); } else { SFillInfo* pFillInfo = pLocalReducer->pFillInfo; if (pFillInfo != NULL) { - taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, pQueryInfo->window.ekey); + TSKEY ekey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; + + taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, ekey); taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } @@ -1292,7 +1267,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->fillType != TSDB_FILL_NONE) { - TSKEY skey = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t newTime = taosGetIntervalStartTimestamp(skey, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); taosResetFillInfo(pLocalReducer->pFillInfo, newTime); @@ -1345,7 +1320,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { // if fillType == TSDB_FILL_NONE, return directly if (pQueryInfo->fillType != TSDB_FILL_NONE && ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { - int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; + int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey; int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 98eb5a8d6d..7954a8d228 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -128,3 +128,27 @@ if $rows != 86399 then return -1 endi +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 30 +if $rows != 48 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 2 +if $rows != 4 then + return -1 +endi + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 order by ts desc slimit 1 soffset 1 limit 250000 offset 1 +if $rows != 86399 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 1 +if $rows != 2 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 25 offset 1 +if $rows != 46 then + return -1 +endi From 5aff8962a45952cbd54377435cda13647d24714c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 14:52:50 +0800 Subject: [PATCH 03/55] [td-225] fix bugs in retrieve error. --- src/tsdb/src/tsdbRead.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c89ae0698a..bb3f7ca195 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1795,14 +1795,17 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { if (pQueryHandle->checkFiles) { bool exists = true; + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - return code; + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; + + return false; } if (exists) { - elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); return exists; } From 3e2ea659cba021a5186f38f695e9f97147b2aa3b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 18:17:46 +0800 Subject: [PATCH 04/55] [td-225] fix bug in "show configs"; --- src/mnode/src/mnodeDnode.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 261f63d5cf..f6a687d864 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -857,13 +857,13 @@ static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = TSDB_CFG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config name"); + tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CFG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config value"); + tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -898,27 +898,32 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v int32_t cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->option, TSDB_CFG_OPTION_LEN); + cols++; + int32_t t = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { case TAOS_CFG_VTYPE_INT16: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_INT32: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_FLOAT: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_STRING: case TAOS_CFG_VTYPE_IPSTR: case TAOS_CFG_VTYPE_DIRECTORY: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->ptr, TSDB_CFG_VALUE_LEN); numOfRows++; break; default: From ff20d564abf28b4307a43edd6a52e60ebf780852 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 23:05:07 +0800 Subject: [PATCH 05/55] [td-1109] --- src/inc/taosmsg.h | 2 +- src/inc/ttokendef.h | 2 +- src/mnode/src/mnodeDnode.c | 34 +++++++++++++--------------------- src/mnode/src/mnodeShow.c | 2 +- src/query/inc/sql.y | 2 +- src/query/src/qTokenizer.c | 2 +- src/query/src/sql.c | 17 +++++++---------- 7 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 265bf47d6d..9ae3fd71e2 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -133,7 +133,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, - TSDB_MGMT_TABLE_CONFIGS, + TSDB_MGMT_TABLE_VARIABLES, TSDB_MGMT_TABLE_CONNS, TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_GRANTS, diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index e77f5741d3..c1579fe9e7 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -69,7 +69,7 @@ #define TK_QUERIES 51 #define TK_CONNECTIONS 52 #define TK_STREAMS 53 -#define TK_CONFIGS 54 +#define TK_VARIABLES 54 #define TK_SCORES 55 #define TK_GRANTS 56 #define TK_VNODES 57 diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index f6a687d864..772894376c 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -49,7 +49,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg); static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; -static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *rpcMsg); +static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg); static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -161,8 +161,8 @@ int32_t mnodeInitDnodes() { mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mnodeProcessDnodeStatusMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mnodeGetModuleMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mnodeRetrieveModules); - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeGetConfigMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeRetrieveConfigs); + mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeGetConfigMeta); + mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeRetrieveConfigs); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mnodeGetVnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); @@ -728,7 +728,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo } static bool mnodeCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { - uint32_t status = pDnode->moduleStatus & (1 << moduleType); + uint32_t status = pDnode->moduleStatus & (1u << moduleType); return status > 0; } @@ -753,7 +753,7 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); + strcpy(pSchema[cols].name, "end_point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -787,7 +787,9 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; - char * pWrite; + + char* pWrite; + char* moduleName[5] = { "MNODE", "HTTP", "MONITOR", "MQTT", "UNKNOWN" }; while (numOfRows < rows) { SDnodeObj *pDnode = NULL; @@ -802,28 +804,18 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - 1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - switch (moduleType) { - case TSDB_MOD_MNODE: - strcpy(pWrite, "mnode"); - break; - case TSDB_MOD_HTTP: - strcpy(pWrite, "http"); - break; - case TSDB_MOD_MONITOR: - strcpy(pWrite, "monitor"); - break; - default: - strcpy(pWrite, "unknown"); - } + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, moduleName[moduleType], pShow->bytes[cols]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; bool enable = mnodeCheckModuleInDnode(pDnode, moduleType); - strcpy(pWrite, enable ? "enable" : "disable"); + + char* v = enable? "enable":"disable"; + STR_TO_VARSTR(pWrite, v); cols++; numOfRows++; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 8a84b66a33..e553853658 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -98,7 +98,7 @@ static char *mnodeGetShowType(int32_t showType) { case TSDB_MGMT_TABLE_MODULE: return "show modules"; case TSDB_MGMT_TABLE_QUERIES: return "show queries"; case TSDB_MGMT_TABLE_STREAMS: return "show streams"; - case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; + case TSDB_MGMT_TABLE_VARIABLES: return "show configs"; case TSDB_MGMT_TABLE_CONNS: return "show connections"; case TSDB_MGMT_TABLE_SCORES: return "show scores"; case TSDB_MGMT_TABLE_GRANTS: return "show grants"; diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 2b2a967edf..edf59327ab 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -73,7 +73,7 @@ cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW CONFIGS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } +cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 227aded5f1..6fb63c4a20 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -85,7 +85,7 @@ static SKeyword keywordTable[] = { {"QUERIES", TK_QUERIES}, {"CONNECTIONS", TK_CONNECTIONS}, {"STREAMS", TK_STREAMS}, - {"CONFIGS", TK_CONFIGS}, + {"VARIABLES", TK_VARIABLES}, {"SCORES", TK_SCORES}, {"GRANTS", TK_GRANTS}, {"DOT", TK_DOT}, diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 75ef2f3218..0b977977af 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -488,7 +488,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* QUERIES => nothing */ 0, /* CONNECTIONS => nothing */ 0, /* STREAMS => nothing */ - 0, /* CONFIGS => nothing */ + 0, /* VARIABLES => nothing */ 0, /* SCORES => nothing */ 0, /* GRANTS => nothing */ 0, /* VNODES => nothing */ @@ -781,7 +781,7 @@ static const char *const yyTokenName[] = { /* 51 */ "QUERIES", /* 52 */ "CONNECTIONS", /* 53 */ "STREAMS", - /* 54 */ "CONFIGS", + /* 54 */ "VARIABLES", /* 55 */ "SCORES", /* 56 */ "GRANTS", /* 57 */ "VNODES", @@ -1017,7 +1017,7 @@ static const char *const yyRuleName[] = { /* 7 */ "cmd ::= SHOW QUERIES", /* 8 */ "cmd ::= SHOW CONNECTIONS", /* 9 */ "cmd ::= SHOW STREAMS", - /* 10 */ "cmd ::= SHOW CONFIGS", + /* 10 */ "cmd ::= SHOW VARIABLES", /* 11 */ "cmd ::= SHOW SCORES", /* 12 */ "cmd ::= SHOW GRANTS", /* 13 */ "cmd ::= SHOW VNODES", @@ -1702,7 +1702,7 @@ static const struct { { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW CONFIGS */ + { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ @@ -2029,8 +2029,8 @@ static void yy_reduce( case 9: /* cmd ::= SHOW STREAMS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } break; - case 10: /* cmd ::= SHOW CONFIGS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } + case 10: /* cmd ::= SHOW VARIABLES */ +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } break; case 11: /* cmd ::= SHOW SCORES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } @@ -2277,6 +2277,7 @@ static void yy_reduce( yymsp[-1].minor.yy268 = yylhsminor.yy268; break; case 95: /* db_optr ::= db_optr fsync */ + case 107: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==107); { yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy268 = yylhsminor.yy268; break; @@ -2297,10 +2298,6 @@ static void yy_reduce( case 99: /* alter_db_optr ::= */ { setDefaultCreateDbOption(&yymsp[1].minor.yy268);} break; - case 107: /* alter_db_optr ::= alter_db_optr fsync */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; case 108: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; From 39618becb74e3dcc5b2284d6501bf0fc6daa9c79 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 00:01:01 +0800 Subject: [PATCH 06/55] [td-1151] --- src/client/inc/tscUtil.h | 4 +- src/client/src/tscFunctionImpl.c | 30 ++++++++++----- src/client/src/tscSQLParser.c | 66 +++++++++++++++++++++----------- src/client/src/tscServer.c | 16 ++++---- src/client/src/tscUtil.c | 12 +++--- src/common/inc/tname.h | 2 + src/common/src/tname.c | 19 ++++++++- src/common/src/tvariant.c | 4 ++ src/inc/taosdef.h | 3 +- src/inc/taosmsg.h | 6 +-- src/query/src/qExecutor.c | 40 +++++++++++++------ 11 files changed, 140 insertions(+), 62 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2ca6ba6691..7c406d1676 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -138,8 +138,8 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, - SSchema* pColSchema, int16_t colType); +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, + SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 205be5e3a6..f1f235cd36 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2906,18 +2906,30 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size); - - char *pData = GET_INPUT_CHAR(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); - } else { + if (pCtx->numOfParams == 1) { // the number of output rows should not affect the final number of rows, so set it to be 1 + INC_INIT_VAL(pCtx, 1); + + char* output = pCtx->aOutputBuf; for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); + tVariantDump(&pCtx->param[0], output, pCtx->outputType, true); + output += pCtx->outputBytes; + } + + } else { + + INC_INIT_VAL(pCtx, pCtx->size); + + char *pData = GET_INPUT_CHAR(pCtx); + if (pCtx->order == TSDB_ORDER_ASC) { + memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); + } else { + for(int32_t i = 0; i < pCtx->size; ++i) { + memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, + pCtx->inputBytes); + } } } - + pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 43a43b6d31..fc33e22eff 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -33,8 +33,8 @@ #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" -// -1 is tbname column index, so here use the -2 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-2) +// -1 is tbname column index, so here use the -3 as the initial value +#define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } #define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) @@ -1248,7 +1248,9 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields - if (pItem->pNode->nSQLOptr == TK_ALL || pItem->pNode->nSQLOptr == TK_ID || pItem->pNode->nSQLOptr == TK_STRING) { + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || (optr == TK_INTEGER || optr == TK_FLOAT)) { // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); @@ -1256,7 +1258,6 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // if the name of column is quoted, remove it and set the right information for later process extractColumnNameFromString(pItem); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { @@ -1372,10 +1373,10 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); } -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, pColSchema->bytes, flag); + pColSchema->bytes, pColSchema->bytes, TSDB_COL_IS_TAG(flag)); tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName)); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1391,6 +1392,8 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex if (TSDB_COL_IS_TAG(flag)) { tscColumnListInsert(pTableMetaInfo->tagColList, pIndex); } + + return pExpr; } static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { @@ -1434,7 +1437,11 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL) { // project on all fields + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getTableIndexByName(&pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); @@ -1450,28 +1457,43 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } else { doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos); } - } else if (pItem->pNode->nSQLOptr == TK_ID) { // simple column projection query + + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + } else if (optr == TK_ID || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); - } + // user-specified constant value as a new result column + if ((optr == TK_INTEGER || optr == TK_FLOAT) || (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { + index.columnIndex = TSDB_UD_COLUMN_INDEX; + index.tableIndex = 0; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = tGetTableNameColumnSchema(); - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); - } else { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + SSchema colSchema = tGetUserSpecifiedColumnSchema(pItem->pNode->val.pz, pItem->pNode->val.nType, pItem->aliasName); + SSqlExpr* pExpr = tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + pExpr->numOfParams = 1; + tVariantAssign(&pExpr->param[0], &pItem->pNode->val); - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } else { // columns from the queried table + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + SSchema colSchema = tGetTableNameColumnSchema(); + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); + } else { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + addProjectQueryCol(pQueryInfo, startPos, &index, pItem); } - addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2037,7 +2059,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo refactor static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && columnIndex >= -1 && tableIndex >= 0); + assert(num == 1 && columnIndex >= -2 && tableIndex >= 0); SColumnList columnList = {0}; columnList.num = num; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6b75b680b1..03d696a8ee 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -622,7 +622,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); - return -1; // todo add test for this + return TSDB_CODE_TSC_INVALID_SQL; // todo add test for this } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -631,17 +631,17 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->intervalTime < 0) { tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; @@ -719,7 +719,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { tscError("invalid filter info"); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } } } @@ -731,7 +731,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); @@ -1279,7 +1279,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for alter table msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; @@ -1631,7 +1631,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); tscError("%p failed to malloc for heartbeat msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 64a871ff74..716d48fa03 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -926,7 +926,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize, bool isTagCol) { + int16_t size, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); @@ -935,8 +935,10 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else if (pColIndex->columnIndex == TSDB_UD_COLUMN_INDEX) { + pExpr->colInfo.colId = TSDB_UD_COLUMN_INDEX; } else { - if (isTagCol) { + if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name)); @@ -948,9 +950,9 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol } } - pExpr->colInfo.flag = isTagCol? TSDB_COL_TAG:TSDB_COL_NORMAL; - + pExpr->colInfo.flag = colType; pExpr->colInfo.colIndex = pColIndex->columnIndex; + pExpr->resType = type; pExpr->resBytes = size; pExpr->interBytes = interSize; @@ -1291,7 +1293,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_UD_COLUMN_INDEX) { return true; } diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 2263a5dae1..41d6fe7280 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -28,6 +28,8 @@ void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); SSchema tGetTableNameColumnSchema(); +SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name); + bool tscValidateTableNameLength(size_t len); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index d80ddb0ee2..6aa308cb45 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -43,7 +43,24 @@ SSchema tGetTableNameColumnSchema() { s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; s.type = TSDB_DATA_TYPE_BINARY; s.colId = TSDB_TBNAME_COLUMN_INDEX; - strncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + return s; +} + +SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name) { + SSchema s = {0}; + + s.type = type; + if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { + size_t len = strlen(v); + s.bytes = len + VARSTR_HEADER_SIZE; + } else { + s.bytes = tDataTypeDesc[type].nSize; + } + + s.colId = TSDB_UD_COLUMN_INDEX; + tstrncpy(s.name, name, sizeof(s.name)); + return s; } diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 33f8eb127f..6716a1827e 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -167,7 +167,11 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } + + return; } + + pDst->nLen = tDataTypeDesc[pDst->nType].nSize; } int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 3dea8da18a..12c4699665 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -286,7 +286,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-2) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 9ae3fd71e2..7a3fc6b050 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -167,9 +167,9 @@ enum _mgmt_table { #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column extern char *taosMsg[]; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 52408f78bf..b7434edadf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -35,6 +35,9 @@ */ #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f) == TSDB_COL_UDC) + #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -371,14 +374,14 @@ static bool hasTagValOutput(SQuery* pQuery) { * @return */ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (pStatis != NULL && !TSDB_COL_IS_TAG(pColIndex->flag)) { + if (pStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { *pColStatis = &pStatis[pColIndex->colIndex]; assert((*pColStatis)->colId == pColIndex->colId); } else { *pColStatis = NULL; } - if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } @@ -884,14 +887,14 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { - dataBlock = NULL; - } else { + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { SColIndex* pColIndex = &pQuery->pSelectExpr[col].base.colInfo; SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); assert(p->info.colId == pColIndex->colId); dataBlock = p->pData; + } else { + dataBlock = NULL; } } @@ -1536,7 +1539,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputBytes = pQuery->tagColList[index].bytes; pCtx->inputType = pQuery->tagColList[index].type; } - + } else if (TSDB_COL_IS_UD_COL(pIndex->flag)) { + pCtx->inputBytes = pSqlFuncMsg->arg[0].argBytes; + pCtx->inputType = pSqlFuncMsg->arg[0].argType; } else { pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; @@ -5231,6 +5236,8 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } + } else if (pExprMsg->colInfo.flag == TSDB_COL_UDC) { // user specified column data + return TSDB_UD_COLUMN_INDEX; } else { while (j < pQueryMsg->numOfCols) { if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { @@ -5590,9 +5597,18 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * bytes = tDataTypeDesc[type].nSize; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column SSchema s = tGetTableNameColumnSchema(); - type = s.type; + type = s.type; bytes = s.bytes; - } else{ + } else if (pExprs[i].base.colInfo.colId == TSDB_UD_COLUMN_INDEX) { + assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); + + type = pExprs[i].base.arg[0].argType; + bytes = pExprs[i].base.arg[0].argBytes; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + bytes += VARSTR_HEADER_SIZE; + } + } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); @@ -5765,7 +5781,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { // todo opt performance SColIndex *pColIndex = &pSqlExprMsg->colInfo; - if (!TSDB_COL_IS_TAG(pColIndex->flag)) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { int32_t f = 0; for (f = 0; f < pQuery->numOfCols; ++f) { if (pColIndex->colId == pQuery->colList[f].colId) { @@ -5773,8 +5789,10 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - - assert (f < pQuery->numOfCols); + + assert(f < pQuery->numOfCols); + } else if (pColIndex->colId == TSDB_UD_COLUMN_INDEX) { + // do nothing } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { From 58bad4cf38e9d7b02ac4b895cffe080ea6a5ed72 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 13:18:49 +0800 Subject: [PATCH 07/55] [td-1151] fix the coverity warnings --- src/client/src/tscSQLParser.c | 6 ++-- src/client/src/tscSubquery.c | 57 ++++++++++++----------------------- src/query/inc/qTsbuf.h | 4 +-- src/query/src/qExecutor.c | 40 +++++++++++++----------- src/query/src/qResultbuf.c | 11 +++++-- src/query/src/qTsbuf.c | 2 +- src/util/src/hash.c | 2 +- 7 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index fc33e22eff..b7c0c65c57 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1166,7 +1166,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t size_t numOfNode = taosArrayGetSize(colList); for(int32_t k = 0; k < numOfNode; ++k) { SColIndex* pIndex = taosArrayGet(colList, k); - if (pIndex->flag == 1) { + if (TSDB_COL_IS_TAG(pIndex->flag)) { + tExprTreeDestroy(&pNode, NULL); taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1366,7 +1367,8 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn ids.num = 1; ids.ids[0] = *pIndex; - if (pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta) || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || + pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { ids.num = 0; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index eedb229e1c..e9244fa968 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -685,6 +685,9 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; tscQueueAsyncRes(pParentSql); + + taosArrayDestroy(s1); + taosArrayDestroy(s2); return; } @@ -1284,8 +1287,14 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); - + + // todo add test SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + if (pState == NULL) { + pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pSql->res.code; + } + pState->numOfTotal = pQueryInfo->numOfTables; pState->numOfRemain = pState->numOfTotal; @@ -1297,7 +1306,8 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); pState->numOfRemain = i; pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - + taosTFree(pState); + return pSql->res.code; } @@ -1511,7 +1521,6 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in tstrerror(code), subqueryIndex, trsupport->numOfRetry); SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); - if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); @@ -1522,8 +1531,14 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in return pParentSql->res.code; } - taos_free_result(pSql); - return tscProcessSql(pNew); + int32_t ret = tscProcessSql(pNew); + + // if failed to process sql, let following code handle the pSql + if (ret == TSDB_CODE_SUCCESS) { + taos_free_result(pSql); + } + + return code; } void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -2063,39 +2078,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { doBuildResFromSubqueries(pSql); tsem_post(&pSql->rspSem); - return; - - // continue retrieve data from vnode -// if (!tscHasRemainDataInSubqueryResultSet(pSql)) { -// tscDebug("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1); -// SSubqueryState* pState = NULL; -// -// // free all sub sqlobj -// for (int32_t i = 0; i < pSql->numOfSubs; ++i) { -// SSqlObj* pChildObj = pSql->pSubs[i]; -// if (pChildObj == NULL) { -// continue; -// } -// -// SJoinSupporter* pSupporter = (SJoinSupporter*)pChildObj->param; -// pState = pSupporter->pState; -// -// tscDestroyJoinSupporter(pChildObj->param); -// taos_free_result(pChildObj); -// } -// -// free(pState); -// -// pRes->completed = true; // set query completed -// tsem_post(&pSql->rspSem); -// return; -// } - - tscFetchDatablockFromSubquery(pSql); - if (pRes->code != TSDB_CODE_SUCCESS) { - return; - } } if (pSql->res.code == TSDB_CODE_SUCCESS) { diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index f6d672b26d..46e6f79014 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -85,8 +85,8 @@ typedef struct STSBuf { uint32_t fileSize; STSVnodeBlockInfoEx* pData; - int32_t numOfAlloc; - int32_t numOfVnodes; + uint32_t numOfAlloc; + uint32_t numOfVnodes; char* assistBuf; int32_t bufSize; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b7434edadf..32525a2444 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -967,6 +967,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; + assert(tsCols != NULL); while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; @@ -2730,7 +2731,8 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num); + + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, (size_t)(bytes * pData->num)); } offset += (int32_t)pData->num; @@ -3298,7 +3300,8 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); - tsBufNextPos(pRuntimeEnv->pTSBuf); + bool ret = tsBufNextPos(pRuntimeEnv->pTSBuf); + assert(ret); } // reverse order time range @@ -5225,7 +5228,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { if (pExprMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - return -1; + return TSDB_TBNAME_COLUMN_INDEX; } while(j < pQueryMsg->numOfTags) { @@ -5644,18 +5647,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols); - - SColumnInfo *pCol = &pQueryMsg->colList[j]; - - int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, - &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); - assert(ret == TSDB_CODE_SUCCESS); + if (j < 0 || j >= pQueryMsg->numOfCols) { + assert(0); + } else { + SColumnInfo *pCol = &pQueryMsg->colList[j]; + int32_t ret = + getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); + assert(ret == TSDB_CODE_SUCCESS); + } } } - *pExprInfo = pExprs; + *pExprInfo = pExprs; return TSDB_CODE_SUCCESS; } @@ -6194,14 +6198,16 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // make sure file exist if (FD_VALID(fd)) { - int32_t s = lseek(fd, 0, SEEK_END); - UNUSED(s); - qDebug("QInfo:%p ts comp data return, file:%s, size:%d", pQInfo, pQuery->sdata[0]->data, s); + uint64_t s = lseek(fd, 0, SEEK_END); + + qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { size_t sz = read(fd, data, s); - UNUSED(sz); + if(sz < s) { // todo handle error + assert(0); + } } else { - // todo handle error + UNUSED(s); } close(fd); diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 51ff892b33..acedad0f0e 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -137,8 +137,15 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { } //3. write to disk. - fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - fwrite(t, size, 1, pResultBuf->file); + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + if (ret != 0) { // todo handle the error case + + } + + ret = fwrite(t, size, 1, pResultBuf->file); + if (ret != size) { // todo handle the error case + + } if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { pResultBuf->fileSize = pg->info.offset + pg->info.length; diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index eb47a1b7a1..a6fc3da05f 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -75,7 +75,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - if ((int32_t)header.numOfVnode > pTSBuf->numOfAlloc) { + if (header.numOfVnode > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfVnode; STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 714f36f1cb..7d10545ce7 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -657,7 +657,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; From 1892645002239726c3b3a6138865e85e6d00efbb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 13:24:04 +0800 Subject: [PATCH 08/55] [td-1151] fix the coverity warnings --- src/tsdb/src/tsdbRead.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index bb3f7ca195..7abf86c6bd 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1072,8 +1072,8 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, - int32_t numOfExisted, int32_t *start, int32_t *end) { +static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, + int32_t* start, int32_t* end) { *start = -1; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { @@ -1233,6 +1233,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); + assert(end != -1); + if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it moveToNextRowInMem(pCheckInfo); } From 8972ebc1013ad7373d4ba7f901845890844ef576 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 15:50:52 +0800 Subject: [PATCH 09/55] [td-1103] refactor the structure for reducing memory consumption during interval query. --- src/query/inc/qExecutor.h | 16 +-- src/query/inc/qUtil.h | 3 +- src/query/inc/tsqlfunction.h | 14 +- src/query/src/qExecutor.c | 271 ++++++++++++++++++----------------- src/query/src/qUtil.c | 44 +++--- 5 files changed, 177 insertions(+), 171 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index d824a269c9..320ae8d137 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -42,20 +42,16 @@ typedef struct SSqlGroupbyExpr { } SSqlGroupbyExpr; typedef struct SPosInfo { - int32_t pageId; - int32_t rowId; + int32_t pageId:20; + int32_t rowId:12; } SPosInfo; -typedef struct SWindowStatus { - bool closed; -} SWindowStatus; - typedef struct SWindowResult { - uint16_t numOfRows; // number of rows of current time window - SWindowStatus status; // this result status: closed or opened SPosInfo pos; // Position of current result in disk-based output buffer + uint16_t numOfRows; // number of rows of current time window + bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo - STimeWindow window; // The time window that current result covers. + TSKEY skey; // start key of current time window } SWindowResult; /** @@ -79,6 +75,7 @@ typedef struct SWindowResInfo { int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key int64_t threshold; // threshold to halt query and return the generated results. + int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { @@ -123,6 +120,7 @@ typedef struct SQueryCostInfo { uint64_t elapsedTime; uint64_t computTime; uint64_t internalSupSize; + uint64_t numOfTimeWindows; } SQueryCostInfo; typedef struct SQuery { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b7f57bf1b5..98b0ed0011 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -38,7 +38,8 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf return &pWindowResInfo->pResult[slot]; } -#define curTimeWindow(_winres) ((_winres)->curIndex) +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval)} #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 58d2ff260e..1dafe15989 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -137,13 +137,13 @@ typedef struct SInterpInfoDetail { } SInterpInfoDetail; typedef struct SResultInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - int32_t numOfRes; // num of output result in current buffer - int32_t bufLen; // buffer size - void* interResultBuf; // output result buffer + int8_t hasResult; // result generated, not NULL value + bool initialized:1; // output buffer has been initialized + bool complete:1; // query has completed + bool superTableQ:1; // is super table query + int16_t numOfRes; // num of output result in current buffer + uint32_t bufLen; // buffer size + void* interResultBuf; // output result buffer } SResultInfo; struct SQLFunctionCtx; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 32525a2444..15277ee228 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -27,7 +27,8 @@ #include "query.h" #include "queryLog.h" #include "tlosertree.h" -#include "tscompression.h" + +#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1); /** * check if the primary column is load by default, otherwise, the program will @@ -215,16 +216,16 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { */ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; - + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - + int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS_DUMMY) { continue; } - + assert(pResInfo->numOfRes > numOfRes); pResInfo->numOfRes = numOfRes; } @@ -318,10 +319,10 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.function static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery* pQuery = pRuntimeEnv->pQuery; - + if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; - + qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); assert(pQuery->rec.rows >= 0); @@ -415,6 +416,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -450,8 +452,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.skey = pWindowResInfo->prevSKey; w.ekey = w.skey + pQuery->intervalTime - 1; } else { - int32_t slot = curTimeWindow(pWindowResInfo); - w = getWindowResult(pWindowResInfo, slot)->window; + int32_t slot = curTimeWindowIndex(pWindowResInfo); + SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes); } if (w.skey > ts || w.ekey < ts) { @@ -552,15 +555,15 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes } // set time window for current result - pWindowRes->window = *win; + pWindowRes->skey = win->skey; setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; } -static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot].status; + return pWindowResInfo->pResult[slot].closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -602,7 +605,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe // no qualified results exist, abort check int32_t numOfClosed = 0; - + if (pWindowResInfo->size == 0) { return pWindowResInfo->size; } @@ -620,16 +623,17 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { + if (pResult->closed) { numOfClosed += 1; continue; } - if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + TSKEY ekey = pResult->skey + pWindowResInfo->interval; + if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { - skey = pResult->window.skey; + skey = pResult->skey; break; } } @@ -642,26 +646,26 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - + setQueryStatus(pQuery, QUERY_RESBUF_FULL); } else { qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed); } } - + // output has reached the limitation, set query completed if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && pRuntimeEnv->scanFlag == MASTER_SCAN) { setQueryStatus(pQuery, QUERY_COMPLETED); } - + assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); return numOfClosed; } @@ -675,7 +679,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); STableQueryInfo* item = pQuery->current; - + if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); @@ -706,12 +710,12 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; @@ -735,12 +739,11 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, - int32_t offset) { +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; @@ -825,14 +828,14 @@ static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) //todo binary search static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (colId == p->info.colId) { return p->pData; } } - + return NULL; } @@ -961,7 +964,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + bool pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } @@ -990,8 +993,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } pWindowResInfo->curIndex = index; @@ -1044,8 +1047,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return -1; } - pWindowRes->window.skey = v; - pWindowRes->window.ekey = v; + pWindowRes->skey = v; + assert(pRuntimeEnv->windowResInfo.interval == 0); if (pWindowRes->pos.pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); @@ -1144,7 +1147,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx if (functionId == TSDB_FUNC_TS) { return true; } - + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { return false; } @@ -1251,8 +1254,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; @@ -1275,8 +1278,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { - pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); + closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } } @@ -1331,10 +1334,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + STableQueryInfo* pTableQInfo = pQuery->current; SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { @@ -1377,10 +1380,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, SDataStatis *pStatis, void *param, int32_t colIndex) { - + int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId; int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; - + SDataStatis *tpField = NULL; pCtx->hasNull = hasNullValue(&pQuery->pSelectExpr[colIndex].base.colInfo, pStatis, &tpField); pCtx->aInputElemBuf = inputData; @@ -1436,7 +1439,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - + if (pQuery->fillVal != NULL) { if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) { pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; @@ -1469,13 +1472,13 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p if (isSelectivityWithTagsQuery(pQuery)) { int32_t num = 0; int16_t tagLen = 0; - + SQLFunctionCtx *p = NULL; SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; - + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1494,7 +1497,7 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - taosTFree(pTagCtx); + taosTFree(pTagCtx); } } } @@ -1547,7 +1550,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; } - + assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1809,7 +1812,7 @@ bool colIdCheck(SQuery *pQuery) { return false; } } - + return true; } @@ -1969,7 +1972,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i } pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); - + assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2123,21 +2126,21 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } - + pRuntimeEnv->summary.loadBlockStatis += 1; - + if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { assert(status == BLK_DATA_ALL_NEEDED); - + // load the data block statistics to perform further filter pRuntimeEnv->summary.loadBlockStatis += 1; if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { } - + if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; @@ -2145,7 +2148,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return BLK_DATA_DISCARD; } - + pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); @@ -2250,11 +2253,11 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB SQuery* pQuery = pRuntimeEnv->pQuery; if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; - + if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { int32_t remain = (int32_t)(pRec->capacity - pRec->rows); int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain)); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pQuery->pSelectExpr[i].bytes; assert(bytes > 0 && newSize > 0); @@ -2278,7 +2281,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), newSize, pRec->capacity, newSize - pRec->rows); - + pRec->capacity = newSize; } } @@ -2388,7 +2391,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { char* val = tsdbGetTableName(pTable); assert(val != NULL); - + tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY); } else { char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); @@ -2396,7 +2399,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa tag->nType = TSDB_DATA_TYPE_NULL; return; } - + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; @@ -2443,7 +2446,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx]; - + // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { continue; @@ -2493,14 +2496,14 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes // in case of tag column, the tag information should be extracted from input buffer if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { tVariantDestroy(&pCtx[i].tag); - + int32_t type = pCtx[i].outputType; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); } else { tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); } - + } } @@ -2590,7 +2593,7 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { int32_t type = pQuery->pSelectExpr[i].type; @@ -2839,7 +2842,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); - assert(ts == pWindowRes->window.skey); + assert(ts == pWindowRes->skey); int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -2979,10 +2982,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * if (pTableQueryInfo == NULL) { return; } - + // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - + // TODO validate the assertion // if (!QUERY_IS_ASC_QUERY(pQuery)) { // assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); @@ -2998,7 +3001,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - + SWITCH_ORDER(pTableQueryInfo->cur.order); pTableQueryInfo->cur.vgroupIndex = -1; @@ -3008,10 +3011,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i); - if (!pStatus->closed) { + bool closed = getTimeWindowResStatus(pWindowResInfo, i); + if (!closed) { continue; } @@ -3185,7 +3188,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { qDebug("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows, pQuery->limit.offset - pQuery->rec.rows); - + pQuery->limit.offset -= pQuery->rec.rows; pQuery->rec.rows = 0; @@ -3197,14 +3200,14 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { int64_t numOfSkip = pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; pQuery->limit.offset = 0; - + qDebug("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip, 0, pQuery->rec.rows); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - + memmove(pQuery->sdata[i]->data, (char*) pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes); pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; @@ -3237,7 +3240,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = getWindowResult(pWindowResInfo, i); - if (!pResult->status.closed) { + if (!pResult->closed) { continue; } @@ -3275,10 +3278,10 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) || (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery))); - + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, @@ -3359,7 +3362,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pTableQueryInfo->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; - + pTableQueryInfo->win = pStatus->w; pQuery->window = pTableQueryInfo->win; } @@ -3375,7 +3378,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position @@ -3428,7 +3431,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; - + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, cond.twindow.skey, cond.twindow.ekey); @@ -3678,7 +3681,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { @@ -3709,7 +3712,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (!QUERY_IS_ASC_QUERY(pQuery)) { assert(win.ekey == pQuery->window.ekey); } - + pWindowResInfo->prevSKey = w.skey; } @@ -3735,7 +3738,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { */ STimeWindow *w = &pDataBlockInfo->window; STableQueryInfo* pTableQueryInfo = pQuery->current; - + bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); @@ -3856,7 +3859,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; @@ -3948,10 +3951,10 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; - + while (1) { int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity); - + // todo apply limit output function /* reached the start position of according to offset value, return immediately */ if (pQuery->limit.offset == 0) { @@ -3962,7 +3965,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int if (pQuery->limit.offset < ret) { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); - + ret -= (int32_t)pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer @@ -3970,14 +3973,14 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, ret * pQuery->pSelectExpr[i].bytes); } - + pQuery->limit.offset = 0; return ret; } else { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, " "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0, pQuery->limit.offset - ret); - + pQuery->limit.offset -= ret; pQuery->rec.rows = 0; ret = 0; @@ -3998,13 +4001,14 @@ static void queryCostStatis(SQInfo *pQInfo) { pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64, pQInfo, pSummary->internalSupSize); + qDebug("QInfo:%p :cost summary: internal size:%"PRId64", numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, + pSummary->numOfTimeWindows); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed @@ -4094,7 +4098,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; - + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4143,21 +4147,21 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { // set the abort info pQuery->pos = startPos; - + // reset the query start timestamp pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; pQuery->window.skey = pTableQueryInfo->win.skey; *start = pTableQueryInfo->win.skey; - + pWindowResInfo->prevSKey = tw.skey; int32_t index = pRuntimeEnv->windowResInfo.curIndex; - + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock); pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index - + qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); - + return true; } else { // do nothing *start = tw.skey; @@ -4225,7 +4229,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) - && (cond.order == TSDB_ORDER_ASC) + && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) && (!isFixedOutputQuery(pRuntimeEnv)) @@ -4250,21 +4254,21 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { int32_t numOfCols = pQuery->numOfOutput; int32_t offset = 0; - + SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; - + pFillCol[i].col.bytes = pExprInfo->bytes; pFillCol[i].col.type = (int8_t)pExprInfo->type; pFillCol[i].col.offset = offset; pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query pFillCol[i].functionId = pExprInfo->base.functionId; pFillCol[i].fillVal.i = pQuery->fillVal[i]; - + offset += pExprInfo->bytes; } - + return pFillCol; } @@ -4285,7 +4289,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (code != TSDB_CODE_SUCCESS) { return code; } - + pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4558,7 +4562,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); taosArrayPush(g1, &tx); - + STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; // include only current table @@ -4566,7 +4570,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); pRuntimeEnv->pQueryHandle = NULL; } - + if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); } else { @@ -4580,10 +4584,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } initCtxOutputBuf(pRuntimeEnv); - + SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); - + setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); if (isFirstLastRowQuery(pQuery)) { assert(taosArrayGetSize(s) == 1); @@ -4596,13 +4600,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->current = taosArrayGetP(first, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - + int64_t numOfRes = getNumOfResult(pRuntimeEnv); if (numOfRes > 0) { pQuery->rec.rows += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } - + skipResults(pRuntimeEnv); pQInfo->groupIndex += 1; @@ -4661,14 +4665,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + taosArrayDestroy(s); if (pWindowResInfo->size <= 0) { continue; } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; - pStatus->closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { @@ -4822,11 +4825,11 @@ static void doSaveContext(SQInfo *pQInfo) { SET_REVERSE_SCAN_FLAG(pRuntimeEnv); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + STsdbQueryCond cond = { .order = pQuery->order.order, .colList = pQuery->colList, @@ -4968,14 +4971,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. return; } - + pQuery->current = pTableInfo; // set current query table info - + scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); finalizeQueryResult(pRuntimeEnv); @@ -4993,10 +4996,10 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - + // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); @@ -5087,7 +5090,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { int32_t numOfFilled = 0; TSKEY newStartKey = TSKEY_INITIAL_VAL; - + // skip blocks without load the actual data block from file if no filter condition present skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { @@ -5114,7 +5117,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata); numOfFilled = 0; - + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); @@ -5180,11 +5183,11 @@ static void tableQueryImpl(SQInfo *pQInfo) { // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); SArray* g = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* item = taosArrayGetP(g, 0); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); @@ -5384,7 +5387,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; pColFilter->filterstr = htons(pFilterMsg->filterstr); @@ -5805,7 +5808,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - + assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); } } @@ -6062,7 +6065,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } - + return code; _error: @@ -6140,7 +6143,7 @@ static void freeQInfo(SQInfo *pQInfo) { taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - + if (pQuery->pGroupbyExpr != NULL) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); taosTFree(pQuery->pGroupbyExpr); @@ -6217,7 +6220,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); if (fd != -1) { - close(fd); + close(fd); } } @@ -6236,7 +6239,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit); setQueryStatus(pQuery, QUERY_OVER); } - + return TSDB_CODE_SUCCESS; } @@ -6338,7 +6341,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi pExprs = NULL; pGroupbyExpr = NULL; pTagColumnInfo = NULL; - + if ((*pQInfo) == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _over; @@ -6353,7 +6356,7 @@ _over: if (pGroupbyExpr != NULL) { taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); - } + } free(pTagColumnInfo); free(pExprs); free(pExprMsg); @@ -6520,7 +6523,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = 0; (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } - + (*pRsp)->precision = htons(pQuery->precision); if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { doDumpQueryResult(pQInfo, (*pRsp)->data); @@ -6597,7 +6600,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { if (numOfGroup == 0) { return; } - + SArray* pa = GET_TABLEGROUP(pQInfo, 0); size_t num = taosArrayGetSize(pa); @@ -6698,7 +6701,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { } else { type = pExprInfo[j].type; bytes = pExprInfo[j].bytes; - + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a01eb33ec7..509362863c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -45,8 +45,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + + SQueryCostInfo* pSummary = &pRuntimeEnv->summary; // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); @@ -54,8 +54,11 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; - pRuntimeEnv->summary.internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pWindowResInfo->interval = pRuntimeEnv->pQuery->intervalTime; + + pSummary->internalSupSize += sizeof(SWindowResult) * threshold; + pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pSummary->numOfTimeWindows = threshold; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); @@ -126,8 +129,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { // remove the window slot from hash table - taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type); + if (pResult->closed) { // remove the window slot from hash table + taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type); } else { break; } @@ -149,12 +152,12 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize, + taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize, (char *)&v, sizeof(int32_t)); } @@ -173,7 +176,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].status.closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { ++i; } @@ -184,11 +187,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].status.closed) { + if (pWindowResInfo->pResult[i].closed) { continue; } - pWindowResInfo->pResult[i].status.closed = true; + pWindowResInfo->pResult[i].closed = true; } } @@ -204,7 +207,7 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].window.skey < pWindowResInfo->pResult[1].window.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1; if (order != resultOrder) { return; @@ -212,11 +215,12 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.ekey < lastKey)) { + TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval; + while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) { ++i; } } @@ -227,11 +231,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->status.closed == true); + return (getWindowResult(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->status.closed = true; + getWindowResult(pWindowResInfo, slot)->closed = true; } void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { @@ -253,8 +257,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow pWindowRes->numOfRows = 0; pWindowRes->pos = (SPosInfo){-1, -1}; - pWindowRes->status.closed = false; - pWindowRes->window = TSWINDOW_INITIALIZER; + pWindowRes->closed = false; + pWindowRes->skey = TSKEY_INITIAL_VAL; } /** @@ -264,8 +268,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - dst->window = src->window; - dst->status = src->status; + dst->skey = src->skey; + dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; From 2238b29ffecc124d1fa4ccbd053e8d3ecb349f91 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 15:51:52 +0800 Subject: [PATCH 10/55] [td-1103] fix a typo --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 15277ee228..67464c36eb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -28,7 +28,7 @@ #include "queryLog.h" #include "tlosertree.h" -#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1); +#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) /** * check if the primary column is load by default, otherwise, the program will From 9031c50ca4cc9b77b04cc1dc59da1f0cd30e203f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 19 Aug 2020 01:22:36 +0800 Subject: [PATCH 11/55] [td-1103] fix bugs --- src/common/src/tvariant.c | 12 +++++++++++- src/query/inc/qExecutor.h | 2 +- src/query/inc/qUtil.h | 4 ++-- src/query/inc/tsqlfunction.h | 10 +++++----- src/query/src/qExecutor.c | 22 +++++++++++++++------- src/query/src/qTsbuf.c | 13 +++++++------ 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 6716a1827e..6e8111aa72 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -175,7 +175,17 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { - assert((p1->nType != TSDB_DATA_TYPE_NULL) || (p2->nType != TSDB_DATA_TYPE_NULL)); + if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) { + return 0; + } + + if (p1->nType == TSDB_DATA_TYPE_NULL) { + return -1; + } + + if (p2->nType == TSDB_DATA_TYPE_NULL) { + return 1; + } switch (p1->nType) { case TSDB_DATA_TYPE_BINARY: diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 320ae8d137..5126dd347e 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -118,7 +118,7 @@ typedef struct SQueryCostInfo { uint32_t loadBlockStatis; uint32_t discardBlocks; uint64_t elapsedTime; - uint64_t computTime; + uint64_t firstStageMergeTime; uint64_t internalSupSize; uint64_t numOfTimeWindows; } SQueryCostInfo; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 98b0ed0011..6de3c7c0e5 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -38,8 +38,8 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf return &pWindowResInfo->pResult[slot]; } -#define curTimeWindowIndex(_winres) ((_winres)->curIndex) -#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval)} +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval - 1)} #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 1dafe15989..a443a43a23 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -138,11 +138,11 @@ typedef struct SInterpInfoDetail { typedef struct SResultInfo { int8_t hasResult; // result generated, not NULL value - bool initialized:1; // output buffer has been initialized - bool complete:1; // query has completed - bool superTableQ:1; // is super table query - int16_t numOfRes; // num of output result in current buffer - uint32_t bufLen; // buffer size + bool initialized; // output buffer has been initialized + bool complete; // query has completed + bool superTableQ; // is super table query + uint32_t bufLen; // buffer size + uint64_t numOfRes; // num of output result in current buffer void* interResultBuf; // output result buffer } SResultInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 67464c36eb..c98bac96ba 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -970,7 +970,6 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; - assert(tsCols != NULL); while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; @@ -2668,7 +2667,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } int32_t mergeIntoGroupResult(SQInfo *pQInfo) { - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampUs(); int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); @@ -2695,9 +2694,11 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { SET_STABLE_QUERY_OVER(pQInfo); } - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo, - pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + int64_t elapsedTime = taosGetTimestampUs() - st; + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, elapsedTime); + pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; return TSDB_CODE_SUCCESS; } @@ -2830,6 +2831,13 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { while (1) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); + + taosTFree(pTableList); + taosTFree(posList); + taosTFree(pTree); + taosTFree(pResultInfo); + taosTFree(buf); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -3996,9 +4004,9 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; - qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, total blocks:%d, load block statis:%d," - " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, + pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); qDebug("QInfo:%p :cost summary: internal size:%"PRId64", numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index a6fc3da05f..518bb4083b 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -232,7 +232,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); - UNUSED(r); + assert(r == 0); /* * format for output data: @@ -241,13 +241,14 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * * both side has the compressed length is used to support load data forwards/backwords. */ - fwrite(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - fwrite(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t metaLen = 0; + metaLen += fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - fwrite(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + metaLen += fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - fwrite(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + metaLen += fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); } fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); @@ -255,7 +256,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; pTSBuf->tsData.len = 0; From 91275a73b6bd6b5a1b1093d83470f28a68f4bc7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 13:54:11 +0800 Subject: [PATCH 12/55] [td-225] fix jdbc memory leaks. --- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 8 ++++++++ src/client/src/TSDBJNIConnector.c | 18 ++++++++++++++++++ src/connector/jdbc/.classpath | 17 ----------------- .../com/taosdata/jdbc/TSDBJNIConnector.java | 2 ++ 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index d81bbd9b6d..eaea91d1bf 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -89,6 +89,14 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp (JNIEnv *env, jobject jobj, jlong con, jlong tres); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: isUpdateQueryImp + * Signature: (J)J + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp + (JNIEnv *env, jobject jobj, jlong con, jlong tres); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: freeResultSetImp diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 34204f96bf..bc7d459a44 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -336,6 +336,24 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp( } } +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp(JNIEnv *env, jobject jobj, jlong con, + jlong tres) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection is closed", jobj); + return JNI_CONNECTION_NULL; + } + + if ((void *)tres == NULL) { + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); + return JNI_RESULT_SET_NULL; + } + + SSqlObj *pSql = (TAOS_RES *)tres; + + return (tscIsUpdateQuery(pSql)? 1:0); +} + JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(JNIEnv *env, jobject jobj, jlong con, jlong res) { TAOS *tscon = (TAOS *)con; diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath index 39abf1c5e9..a5d95095cc 100644 --- a/src/connector/jdbc/.classpath +++ b/src/connector/jdbc/.classpath @@ -28,22 +28,5 @@ - - - - - - - - - - - - - - - - - diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index bab3c79089..20a329074e 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -170,6 +170,8 @@ public class TSDBJNIConnector { } private native long getResultSetImp(long connection, long pSql); + + private native long isUpdateQueryImp(long connection, long pSql); /** * Free resultset operation from C to release resultset pointer by JNI From 61622a73b1abb2bb558453873f3c27ce8280785a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 14:20:31 +0800 Subject: [PATCH 13/55] [td-225] fix jdbc memory leaks. --- src/client/src/TSDBJNIConnector.c | 7 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 34 ++-- .../java/com/taosdata/jdbc/TSDBStatement.java | 31 ++-- src/query/inc/qExecutor.h | 23 +-- src/query/src/qExecutor.c | 153 +++++++++++------- src/tsdb/src/tsdbRead.c | 2 +- tests/script/general/parser/fill_stb.sim | 8 +- 7 files changed, 162 insertions(+), 96 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index bc7d459a44..b25f620508 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -327,13 +327,12 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp( STscObj *pObj = pSql->pTscObj; if (tscIsUpdateQuery(pSql)) { - // taos_free_result(pSql); // free result here - jniDebug("jobj:%p, conn:%p, no resultset, %p", jobj, pObj, (void *)tres); - return 0; + jniDebug("jobj:%p, conn:%p, update query, no resultset, %p", jobj, pObj, (void *)tres); } else { jniDebug("jobj:%p, conn:%p, get resultset, %p", jobj, pObj, (void *)tres); - return tres; } + + return tres; } JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp(JNIEnv *env, jobject jobj, jlong con, diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 20a329074e..0cd185de50 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -111,6 +111,8 @@ public class TSDBJNIConnector { * @throws SQLException */ public long executeQuery(String sql) throws SQLException { + // close previous result set if the user forgets to invoke the + // free method to close previous result set. if (!this.isResultsetClosed) { freeResultSet(taosResultSetPointer); } @@ -123,21 +125,20 @@ public class TSDBJNIConnector { this.freeResultSet(pSql); throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding")); } + int code = this.getErrCode(pSql); if (code != 0) { affectedRows = -1; - String err_msg = this.getErrMsg(pSql); + String msg = this.getErrMsg(pSql); + this.freeResultSet(pSql); - throw new SQLException(TSDBConstants.WrapErrMsg(err_msg), "", code); + throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code); } - // Try retrieving result set for the executed SQL using the current connection pointer. If the executed - // SQL is a DML/DDL which doesn't return a result set, then taosResultSetPointer should be 0L. Otherwise, - // taosResultSetPointer should be a non-zero value. + // Try retrieving result set for the executed SQL using the current connection pointer. taosResultSetPointer = this.getResultSetImp(this.taos, pSql); - if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - isResultsetClosed = false; - } + isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER); + return pSql; } @@ -171,6 +172,10 @@ public class TSDBJNIConnector { private native long getResultSetImp(long connection, long pSql); + public boolean isUpdateQuery(long pSql) { + return isUpdateQueryImp(this.taos, pSql) == 1? true:false; + } + private native long isUpdateQueryImp(long connection, long pSql); /** @@ -180,13 +185,14 @@ public class TSDBJNIConnector { int res = TSDBConstants.JNI_SUCCESS; if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { throw new RuntimeException("Invalid result set pointer"); - } else if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, result); - isResultsetClosed = true; // reset resultSetPointer to 0 after freeResultSetImp() return - taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - } else { - isResultsetClosed = true; } + + if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { + res = this.freeResultSetImp(this.taos, result); + taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; + } + + isResultsetClosed = true; return res; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index e828864313..075a351661 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -51,6 +51,8 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } + + // TODO make sure it is not a update query pSql = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); @@ -58,33 +60,40 @@ public class TSDBStatement implements Statement { if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { this.connecter.freeResultSet(pSql); throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { -// create/insert/update/del/alter + } + + // create/insert/update/delete/alter + if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { this.connecter.freeResultSet(pSql); return null; - } else { - return new TSDBResultSet(this.connecter, resultSetPointer); } + + if (!this.connecter.isUpdateQuery(pSql)) { + return new TSDBResultSet(this.connecter, resultSetPointer); + } else { + this.connecter.freeResultSet(pSql); + return null; + } + } public int executeUpdate(String sql) throws SQLException { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } + + // TODO check if current query is update query pSql = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { this.connecter.freeResultSet(pSql); throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - this.connecter.freeResultSet(); - throw new SQLException("The executed SQL is not a DML or a DDL"); - } else { - int num = this.connecter.getAffectedRows(pSql); - this.connecter.freeResultSet(pSql); - return num; } + + int num = this.connecter.getAffectedRows(pSql); + this.connecter.freeResultSet(pSql); + return num; } public String getErrorMsg(long pSql) { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 5126dd347e..b023de0dc9 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,6 +33,17 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct SPosInfo { + int32_t pageId:20; + int32_t rowId:12; +} SPosInfo; + +typedef struct SGroupResInfo { + int32_t groupId; + int32_t numOfDataPages; + SPosInfo pos; +} SGroupResInfo; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -41,11 +52,6 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SPosInfo { - int32_t pageId:20; - int32_t rowId:12; -} SPosInfo; - typedef struct SWindowResult { SPosInfo pos; // Position of current result in disk-based output buffer uint16_t numOfRows; // number of rows of current time window @@ -190,18 +196,15 @@ typedef struct SQInfo { STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; - int32_t groupIndex; - int32_t offset; // offset in group result set of subgroup, todo refactor SArray* arrTableIdInfo; + int32_t groupIndex; /* * the query is executed position on which meter of the whole list. * when the index reaches the last one of the list, it means the query is completed. - * We later may refactor to remove this attribution by using another flag to denote - * whether a multimeter query is completed or not. */ int32_t tableIndex; - int32_t numOfGroupResultPages; + SGroupResInfo groupResInfo; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2cab8cef71..3754276f93 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -155,9 +155,9 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQInfo *pQInfo); +static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); -bool doFilterData(SQuery *pQuery, int32_t elemPos) { + bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; @@ -231,7 +231,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static int32_t getGroupResultId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) { int32_t base = 20000000; return base + (groupIndex * 10000); } @@ -2686,11 +2686,12 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { break; } - assert(pQInfo->numOfGroupResultPages == 0); + assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - if (pQInfo->groupIndex == numOfGroups && pQInfo->offset == pQInfo->numOfGroupResultPages) { + SGroupResInfo* info = &pQInfo->groupResInfo; + if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -2703,8 +2704,12 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { } void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - if (pQInfo->offset == pQInfo->numOfGroupResultPages) { - pQInfo->numOfGroupResultPages = 0; + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + // all results have been return to client, try next group + if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { + pGroupResInfo->numOfDataPages = 0; + pGroupResInfo->pos.rowId = 0; // current results of group has been sent to client, try next group if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { @@ -2713,7 +2718,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } @@ -2722,30 +2727,50 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t id = getGroupResultId(pQInfo->groupIndex - 1); - SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); - - int32_t size = (int32_t)(taosArrayGetSize(list)); + int32_t id = pQInfo->groupResInfo.groupId; + SIDList list = getDataBufPagesIdList(pResultBuf, id); int32_t offset = 0; - for (int32_t j = 0; j < size; ++j) { + int32_t numOfCopiedRows = 0; + + size_t size = taosArrayGetSize(list); + assert(size == pGroupResInfo->numOfDataPages); + + bool done = false; + for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage *pData = getResBufPage(pResultBuf, pi->pageId); + tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); + + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); + int32_t numOfRes = pData->num - pGroupResInfo->pos.rowId; + + if (numOfRes > pQuery->rec.capacity - offset) { + numOfCopiedRows = pQuery->rec.capacity - offset; + pGroupResInfo->pos.rowId += numOfCopiedRows; + done = true; + } else { + numOfCopiedRows = pData->num; + + pGroupResInfo->pos.pageId += 1; + pGroupResInfo->pos.rowId = 0; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, (size_t)(bytes * pData->num)); + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, + (size_t)bytes * numOfCopiedRows); } - offset += (int32_t)pData->num; + offset += numOfCopiedRows; + if (done) { + break; + } } assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; - pQInfo->offset += 1; } int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { @@ -2791,23 +2816,38 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { // todo opt for the case of one table per group int32_t numOfTables = 0; + SIDList pageList = NULL; + int32_t tid = -1; + for (int32_t i = 0; i < size; ++i) { STableQueryInfo *item = taosArrayGetP(pGroup, i); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); + pageList = list; + tid = TSDB_TABLEID(item->pTable)->tid; + if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables] = item; - numOfTables += 1; + pTableList[numOfTables++] = item; } } + // there is no data in current group if (numOfTables == 0) { taosTFree(posList); taosTFree(pTableList); - - assert(pQInfo->numOfGroupResultPages == 0); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group + taosTFree(posList); + taosTFree(pTableList); + + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + pGroupResInfo->numOfDataPages = taosArrayGetSize(pageList); + pGroupResInfo->groupId = tid; + pGroupResInfo->pos.pageId = 0; + pGroupResInfo->pos.rowId = 0; + + return pGroupResInfo->numOfDataPages; } SCompSupporter cs = {pTableList, posList, pQInfo}; @@ -2824,6 +2864,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); + // todo add windowRes iterator int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -2868,7 +2910,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { doMerge(pRuntimeEnv, ts, pWindowRes, true); } else { // copy data to disk buffer if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { return -1; } @@ -2905,7 +2947,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); taosTFree(pTree); @@ -2929,16 +2971,14 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(posList); taosTFree(pTree); - pQInfo->offset = 0; - taosTFree(pResultInfo); taosTFree(buf); - return pQInfo->numOfGroupResultPages; + + return pQInfo->groupResInfo.numOfDataPages; } -int32_t flushFromResultBuf(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -2946,32 +2986,32 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { int32_t pageId = -1; int32_t capacity = pResultBuf->numOfRowsPerPage; - int32_t remain = (int32_t)pQuery->sdata[0]->num; + int32_t remain = (int32_t) pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { - int32_t r = remain; - if (r > capacity) { - r = capacity; - } + int32_t rows = (remain > capacity)? capacity:remain; + assert(rows > 0); - int32_t id = getGroupResultId(pQInfo->groupIndex) + pQInfo->numOfGroupResultPages; - tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); + // get the output buffer page + tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); + buf->num = rows; - // pagewise copy to dest buffer + // pagewisely copy to dest buffer for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - buf->num = r; - memcpy(buf->data + pRuntimeEnv->offset[i] * buf->num, ((char *)pQuery->sdata[i]->data) + offset * bytes, - buf->num * bytes); + char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; + char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; + memcpy(output, src, buf->num * bytes); } - offset += r; - remain -= r; + offset += rows; + remain -= rows; + + pGroupResInfo->numOfDataPages += 1; } - pQInfo->numOfGroupResultPages += 1; return TSDB_CODE_SUCCESS; } @@ -3773,27 +3813,27 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ step = -1; } + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { if (result[i].numOfRows == 0) { - pQInfo->offset = 0; pQInfo->groupIndex += 1; + pGroupResInfo->pos.rowId = 0; continue; } - assert(pQInfo->offset <= 1); - - int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; - int32_t oldOffset = pQInfo->offset; + int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId; + int32_t oldOffset = pGroupResInfo->pos.rowId; /* - * current output space is not enough to keep all the result data of this group, only copy partial results - * to SQuery object's result buffer + * current output space is not enough to accommodate all data of this page, only partial results + * will be copied to SQuery object's result buffer */ if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { - numOfRowsToCopy = (int32_t)pQuery->rec.capacity - numOfResult; - pQInfo->offset += numOfRowsToCopy; + numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; + pGroupResInfo->pos.rowId += numOfRowsToCopy; } else { - pQInfo->offset = 0; + pGroupResInfo->pos.rowId = 0; pQInfo->groupIndex += 1; } @@ -4004,12 +4044,15 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; + // add the merge time + pSummary->elapsedTime += pSummary->firstStageMergeTime; + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64", numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, + qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, pSummary->numOfTimeWindows); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7db9937a6d..924288ea55 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2564,7 +2564,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDestroyHelper(&pQueryHandle->rhelper); SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", + tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); taosTFree(pQueryHandle); diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index 402b79d6b5..f0cd058352 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -133,12 +133,18 @@ endi if $data74 != -4.000000000 then return -1 endi + ## fill(value) + group by sql select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) group by t1 $val = $rowNum * 2 +print $rowNum, $val + $val = $val - 1 $val = $val * $tbNum -if $rows != $val then +print ==================== $val + +if $rows != 190 then + print expect 190, actual:$rows return -1 endi if $data06 != 0 then From 49f23d689d5580cdacdcaf61a3fd8e4373a7716b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 18:08:31 +0800 Subject: [PATCH 14/55] [td-1067] opt last_row queries --- src/query/src/qExecutor.c | 8 +- src/tsdb/src/tsdbRead.c | 394 +++++++++++++++++++------------------- 2 files changed, 195 insertions(+), 207 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3754276f93..1103b9b531 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4623,7 +4623,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); + assert(0); // last_row query switch to other routine to handle +// pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); } else { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } @@ -4984,7 +4985,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { el = scanMultiTableDataBlocks(pQInfo); qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); -// doCloseAllTimeWindowAfterScan(pQInfo); doRestoreContext(pQInfo); } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); @@ -5262,15 +5262,13 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol && - !isFirstLastRowQuery(pQuery))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol)) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); sequentialTableProcess(pQInfo); - } // record the total elapsed time diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 924288ea55..ba46427b1a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -28,6 +28,14 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ + .tid = (_checkInfo)->tableId.tid, \ + .uid = (_checkInfo)->tableId.uid}) + + enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, @@ -119,7 +127,14 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +typedef struct STableGroupSupporter { + int32_t numOfCols; + SColIndex* pCols; + STSchema* pTagSchema; +} STableGroupSupporter; + +static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); + static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); @@ -283,12 +298,10 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { + pCond->order = TSDB_ORDER_DESC; + pCond->twindow = changeTableGroupByLastrow(groupList); + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); - if (pQueryHandle != NULL) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; - pQueryHandle->order = TSDB_ORDER_DESC; - changeQueryHandleForLastrowQuery(pQueryHandle); - } return pQueryHandle; } @@ -672,14 +685,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return code; } -#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ - ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ - .numOfCols = (_block)->numOfCols, \ - .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) - - static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { STsdbRepo *pRepo = pQueryHandle->pTsdb; int64_t st = taosGetTimestampUs(); @@ -1660,6 +1665,128 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { return false; } +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { + // filter the queried time stamp in the first place + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + pQueryHandle->order = TSDB_ORDER_DESC; + + assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); + + // starts from the buffer in case of descending timestamp order check data blocks + // todo consider the query time window, current last_row does not apply the query time window + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + int32_t i = 0; + while(i < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && + pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { + break; + } + + i++; + } + + // there are no data in all the tables + if (i == numOfTables) { + return; + } + + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); + taosArrayClear(pQueryHandle->pTableCheckInfo); + + info.lastKey = pQueryHandle->window.skey; + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + + // update the query time window according to the chosen last timestamp + pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; +} + +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle) { + int numOfRows = 0; + int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + win->skey = TSKEY_INITIAL_VAL; + + int64_t st = taosGetTimestampUs(); + STable* pTable = pCheckInfo->pTableObj; + + do { + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + if (row == NULL) { + break; + } + + TSKEY key = dataRowKey(row); + if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + pQueryHandle->window.ekey); + + break; + } + + if (win->skey == INT64_MIN) { + win->skey = key; + } + + win->ekey = key; + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + + if (++numOfRows >= maxRowsToRead) { + moveToNextRowInMem(pCheckInfo); + break; + } + + } while(moveToNextRowInMem(pCheckInfo)); + + assert(numOfRows <= maxRowsToRead); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + int32_t emptySize = maxRowsToRead - numOfRows; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } + + int64_t elapsedTime = taosGetTimestampUs() - st; + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); + + return numOfRows; +} + +static int32_t getAllTableList(STable* pSuperTable, SArray* list) { + SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + while (tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + + STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(list, &info); + } + + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} + +static void destroyHelper(void* param) { + if (param == NULL) { + return; + } + + + tQueryInfo* pInfo = (tQueryInfo*)param; + if (pInfo->optr != TSDB_RELATION_IN) { + taosTFree(pInfo->q); + } + + free(param); +} + // handle data in cache situation bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; @@ -1796,6 +1923,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { } if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block bool exists = true; int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); @@ -1824,150 +1952,48 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return ret; } -void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; - assert(!ASCENDING_TRAVERSE(pQueryHandle->order)); - - // starts from the buffer in case of descending timestamp order check data blocks +STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { + STimeWindow window = {INT64_MAX, INT64_MIN}; + // NOTE: starts from the buffer in case of descending timestamp order check data blocks // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); + for(int32_t j = 0; j < numOfGroups; ++j) { + SArray* pGroup = taosArrayGetP(groupList->pGroupList, j); + TSKEY key = TSKEY_INITIAL_VAL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t index = -1; + STableKeyInfo keyInfo = {0}; - for(int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pCheckInfo->pTableObj->lastKey > key) { - key = pCheckInfo->pTableObj->lastKey; - index = i; + size_t numOfTables = taosArrayGetSize(pGroup); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + + if (key < lastKey) { + key = lastKey; + + keyInfo.pTable = pKeyInfo->pTable; + keyInfo.lastKey = key; + pKeyInfo->lastKey = key; + + if (key < window.skey) { + window.skey = key; + } + + if (key > window.ekey) { + window.ekey = key; + } + } + } + + // more than one table in each group, only one table left for each group + if (numOfTables > 1) { + taosArrayClear(pGroup); + taosArrayPush(pGroup, &keyInfo); } } - if (index == -1) { - // todo add failure test cases - return; - } - - // erase all other elements in array list - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - if (i == index) { - continue; - } - - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - tSkipListDestroyIter(pTableCheckInfo->iter); - - if (pTableCheckInfo->pDataCols != NULL) { - taosTFree(pTableCheckInfo->pDataCols->buf); - } - - taosTFree(pTableCheckInfo->pDataCols); - taosTFree(pTableCheckInfo->pCompInfo); - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = key; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {key, key}; -} - -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { - // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - pQueryHandle->order = TSDB_ORDER_DESC; - - assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); - - // starts from the buffer in case of descending timestamp order check data blocks - // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - - int32_t i = 0; - while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && - pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { - break; - } - - i++; - } - - // there are no data in all the tables - if (i == numOfTables) { - return; - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; -} - -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { - int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - win->skey = TSKEY_INITIAL_VAL; - - int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; - - do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); - if (row == NULL) { - break; - } - - TSKEY key = dataRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); - - break; - } - - if (win->skey == INT64_MIN) { - win->skey = key; - } - - win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); - - if (++numOfRows >= maxRowsToRead) { - moveToNextRowInMem(pCheckInfo); - break; - } - - } while(moveToNextRowInMem(pCheckInfo)); - - assert(numOfRows <= maxRowsToRead); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { - int32_t emptySize = maxRowsToRead - numOfRows; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); - } - } - - int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); - - return numOfRows; + return window; } void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { @@ -2104,36 +2130,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } -static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); - while (tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); - - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(list, &info); - } - - tSkipListDestroyIter(iter); - return TSDB_CODE_SUCCESS; -} - -static void destroyHelper(void* param) { - if (param == NULL) { - return; - } - - - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - taosTFree(pInfo->q); - } - -// tVariantDestroy(&(pInfo->q)); - free(param); -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -2160,13 +2156,7 @@ void filterPrepare(void* expr, void* param) { } } -typedef struct STableGroupSupporter { - int32_t numOfCols; - SColIndex* pCols; - STSchema* pTagSchema; -} STableGroupSupporter; - -int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { +static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; STable* pTable1 = ((STableKeyInfo*) p1)->pTable; STable* pTable2 = ((STableKeyInfo*) p2)->pTable; @@ -2219,8 +2209,19 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, - __ext_compar_fn_t compareFn) { +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} + +void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, + STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); @@ -2293,7 +2294,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); @@ -2591,14 +2592,3 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } - -static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { - return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { - return 1; - } else { - ASSERT(false); - return 0; - } -} From c5cfeee49373c48dee545161cbe6eaff2211efbc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 23:40:39 +0800 Subject: [PATCH 15/55] [td-1067] fix bugs in lastrow query. --- src/query/src/qExecutor.c | 2 +- src/tsdb/src/tsdbRead.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1103b9b531..bf72f209d6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5262,7 +5262,7 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol)) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ba46427b1a..b189c7a406 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -301,6 +301,9 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab pCond->order = TSDB_ORDER_DESC; pCond->twindow = changeTableGroupByLastrow(groupList); + //descending order query, skey >= ekey + SWAP(pCond->twindow.skey, pCond->twindow.ekey, TSKEY); + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); return pQueryHandle; } From 2c4aff8f520c2dd351f3ef2ee8c838273a9335c8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 23:58:35 +0800 Subject: [PATCH 16/55] [td-1067] fix bugs in lastrow query. --- src/client/src/tscFunctionImpl.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index f1f235cd36..495136b393 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1811,23 +1811,23 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { * NOTE: last_row does not use the interResultBuf to keep the result */ static void last_row_function(SQLFunctionCtx *pCtx) { - assert(pCtx->size == 1); - + assert(pCtx->size >= 1); char *pData = GET_INPUT_CHAR(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + + // assign the last element in current data block + assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; - pInfo->ts = pCtx->ptsList[0]; - + pInfo->ts = pCtx->ptsList[pCtx->size - 1]; pInfo->hasResult = DATA_SET_FLAG; - // set the result to final result buffer + // set the result to final result buffer in case of super table query if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[0]; + pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); From 084cbca717607beadd77660997d759e4aa02c850 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Aug 2020 00:15:08 +0800 Subject: [PATCH 17/55] [td-1067] fix bugs in lastrow query. --- src/client/src/tscFunctionImpl.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 495136b393..e494446ce5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1820,10 +1820,6 @@ static void last_row_function(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; - pInfo->ts = pCtx->ptsList[pCtx->size - 1]; - pInfo->hasResult = DATA_SET_FLAG; - // set the result to final result buffer in case of super table query if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); From b28dde67ee79a85aba7f1fb03116e3d6f608d424 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Aug 2020 00:57:18 +0800 Subject: [PATCH 18/55] [td-1067] update the tablemeta life time --- src/client/src/tscServer.c | 4 ++-- src/util/inc/tcache.h | 12 +----------- src/util/src/tcache.c | 4 ++-- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 172f18af94..64562ef912 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1702,7 +1702,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(pTableMetaInfo->pTableMeta == NULL); pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer); + strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); // todo handle out of memory case if (pTableMetaInfo->pTableMeta == NULL) { @@ -1906,7 +1906,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer); + tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 11121fcf3b..3d6f9705ee 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -103,7 +103,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS); /** * get data from cache @@ -113,16 +113,6 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v */ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen); -/** - * update the expire time of data in cache - * @param pCacheObj cache object - * @param key key - * @param keyLen keyLen - * @param expireTime new expire time of data - * @return - */ -//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); - /** * Add one reference count for the exist data, and assign this data for a new owner. * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore. diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index bd903c8c23..dfa982b848 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -189,12 +189,12 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return pCacheObj; } -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { return NULL; } - SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); + SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); if (pNode1 == NULL) { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); return NULL; From 3050af21b297616b9485eea124e6a437c97bc3af Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Aug 2020 14:16:42 +0800 Subject: [PATCH 19/55] [td-225] add some logs. --- src/query/src/qExecutor.c | 6 ++++-- src/tsdb/src/tsdbRead.c | 20 +++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bf72f209d6..8cd6a12322 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4501,8 +4501,10 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - qDebug("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64, - pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); + qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " + "lastKey:%" PRId64, + pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, + pQuery->current->lastKey); } if (terrno != TSDB_CODE_SUCCESS) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b189c7a406..6d0966e52a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -409,7 +409,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, pCheckInfo->lastKey, pHandle->qinfo); @@ -744,6 +744,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; + tsdbDebug("%p key in mem:%"PRId64", %p", pQueryHandle, key, pQueryHandle->qinfo); + cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || @@ -755,7 +757,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); + TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; // update the last key value @@ -1170,6 +1173,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->mixBlock = true; } + tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + "end:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pQueryHandle->qinfo); + // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; int32_t pos = cur->pos; @@ -1199,6 +1207,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveDataToFront(pQueryHandle, numOfRows, numOfCols); updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); + + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, + cur->win.ekey, cur->rows, pQueryHandle->qinfo); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; @@ -1279,6 +1291,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; + cur->mixBlock = true; } } } @@ -1295,7 +1308,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey, + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pQueryHandle->qinfo); } From 3ff538c6cc71faef1dafe14990bd5d95c6f2e8c5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 10:52:00 +0800 Subject: [PATCH 20/55] [td-225] merge develop --- src/client/inc/tscLocalMerge.h | 1 - src/client/inc/tsclient.h | 1 + src/client/src/tscLocalMerge.c | 9 +++--- src/client/src/tscServer.c | 28 +++++++++++------ src/client/src/tscSql.c | 14 ++++----- src/client/src/tscSubquery.c | 56 ++++++--------------------------- src/client/src/tscUtil.c | 2 ++ src/kit/shell/src/shellEngine.c | 10 ++---- 8 files changed, 45 insertions(+), 76 deletions(-) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 397a60d140..4e579b0729 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -87,7 +87,6 @@ typedef struct SRetrieveSupport { SSqlObj * pParentSql; tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to uint32_t numOfRetry; // record the number of retry times - pthread_mutex_t queryMutex; } SRetrieveSupport; int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pDesc, diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d02bc7fbd..65a868cf40 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -323,6 +323,7 @@ typedef struct SSqlObj { SSqlRes res; uint16_t numOfSubs; struct SSqlObj **pSubs; + tsem_t subReadySem; struct SSqlObj * prev, *next; } SSqlObj; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 156bc1de9e..cabf2a6a11 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -1377,13 +1377,12 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tscResetForNextRetrieve(pRes); if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed - tscDebug("%p %s call the drop local reducer", pSql, __FUNCTION__); - tscDestroyLocalReducer(pSql); - return 0; + tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); + return pRes->code; } SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // set the data merge in progress int32_t prevStatus = @@ -1478,8 +1477,8 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * so the processing of previous group is completed. */ int32_t numOfRes = finalizeRes(pQueryInfo, pLocalReducer); + bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); tFilePage *pResBuf = pLocalReducer->pResultBuf; /* diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 05264a2665..5a7a858ba1 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -314,10 +314,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { pRes->rspLen = 0; - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { tscDebug("%p query is cancelled, code:%s", pSql, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; } if (pRes->code == TSDB_CODE_SUCCESS) { @@ -460,11 +460,10 @@ void tscKillSTableQuery(SSqlObj *pSql) { continue; } - /* - * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause - * sub-queries not correctly released and master sql object of super table query reaches an abnormal state. - */ - rpcCancelRequest(pSub->pRpcCtx); + if (pSub->pRpcCtx != NULL) { + rpcCancelRequest(pSub->pRpcCtx); + } + pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; tscQueueAsyncRes(pSub); } @@ -1443,6 +1442,17 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return code; + } + + // all subquery have completed already + if (pRes->pLocalReducer == NULL) { + sem_wait(&pSql->subReadySem); + } + pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1453,7 +1463,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { pRes->row = 0; pRes->completed = (pRes->numOfRows == 0); - int32_t code = pRes->code; + code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1bd885466c..cfcc9f0fa6 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -295,6 +295,8 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { } tsem_init(&pSql->rspSem, 0, 0); + tsem_init(&pSql->subReadySem, 0, 0); + doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); tsem_wait(&pSql->rspSem); @@ -655,16 +657,13 @@ int* taos_fetch_lengths(TAOS_RES *res) { char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { - if (res == NULL) { + SSqlObj *pSql = (SSqlObj *)res; + if (pSql == NULL || pSql->signature != pSql) { return; } - SSqlObj *pSql = (SSqlObj *)res; - SSqlCmd *pCmd = &pSql->cmd; - - if (pSql->signature != pSql) return; tscDebug("%p start to cancel query", res); - + SSqlCmd *pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { @@ -674,9 +673,8 @@ void taos_stop_query(TAOS_RES *res) { if (pSql->cmd.command < TSDB_SQL_LOCAL) { rpcCancelRequest(pSql->pRpcCtx); } - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSql); + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; tscDebug("%p query is cancelled", res); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index dc3445680c..1aeaba650d 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1338,10 +1338,6 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState SRetrieveSupport* pSupport = pSub->param; taosTFree(pSupport->localBuffer); - - pthread_mutex_unlock(&pSupport->queryMutex); - pthread_mutex_destroy(&pSupport->queryMutex); - taosTFree(pSupport); tscFreeSqlObj(pSub); @@ -1414,13 +1410,6 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSql = pSql; trs->pFinalColModel = pModel; - pthread_mutexattr_t mutexattr; - memset(&mutexattr, 0, sizeof(pthread_mutexattr_t)); - - pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); - pthread_mutex_init(&trs->queryMutex, &mutexattr); - pthread_mutexattr_destroy(&mutexattr); - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); @@ -1461,6 +1450,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscDebug("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); tscProcessSql(pSub); } + + // set the command flag must be after the semaphore been correctly set. + pSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, 0); + } return TSDB_CODE_SUCCESS; } @@ -1469,12 +1464,8 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tscDebug("%p start to free subquery result", pSql); taos_free_result(pSql); - taosTFree(trsupport->localBuffer); - pthread_mutex_unlock(&trsupport->queryMutex); - pthread_mutex_destroy(&trsupport->queryMutex); - taosTFree(trsupport); } @@ -1498,8 +1489,6 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES pParentSql->res.code = code; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - pthread_mutex_unlock(&trsupport->queryMutex); tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); } @@ -1518,8 +1507,6 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - pthread_mutex_unlock(&trsupport->queryMutex); - tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); @@ -1602,15 +1589,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO tscFreeSubSqlObj(trsupport, pSql); // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); - - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { - (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); - } else { // regular super table query - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pParentSql); - } - } + tsem_post(&pParentSql->subReadySem); } static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { @@ -1682,14 +1661,9 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // only free once taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - - // set the command flag must be after the semaphore been correctly set. - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; - if (pParentSql->res.code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - tscQueueAsyncRes(pParentSql); - } + + // all subqueries are completed, retrieve from local can be proceeded. + tsem_post(&pParentSql->subReadySem); } static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { @@ -1707,9 +1681,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); - // query process and cancel query process may execute at the same time - pthread_mutex_lock(&trsupport->queryMutex); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; @@ -1783,7 +1754,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR return; } else { // continue fetch data from dnode - pthread_mutex_unlock(&trsupport->queryMutex); taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } @@ -2083,12 +2053,6 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { tsem_post(&pSql->rspSem); return; } - -// if (pSql->res.code == TSDB_CODE_SUCCESS) { -// (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); -// } else { -// tscQueueAsyncRes(pSql); -// } } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 716d48fa03..feae58090c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -388,6 +388,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); + tsem_destroy(&pSql->subReadySem); + free(pSql); } diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d82ddf8871..45a81c7b56 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -25,6 +25,8 @@ #include "taosdef.h" #include "taoserror.h" #include "tglobal.h" +#include "tsclient.h" + #include /**************** Global variables ****************/ @@ -64,11 +66,6 @@ TAOS *shellInit(SShellArguments *args) { } taos_init(); - /* - * set tsTableMetaKeepTimer = 3000ms - * means not save cache in shell - */ - tsTableMetaKeepTimer = 3000; // Connect to the database. TAOS *con = NULL; @@ -303,8 +300,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { return; } - int num_fields = taos_field_count(pSql); - if (num_fields != 0) { // select and show kinds of commands + if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { From 2ad3133465b4fac0e230d70bfb99d45f39c4b05d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 11:57:13 +0800 Subject: [PATCH 21/55] [td-225] refactor shell code. --- src/kit/shell/src/shellEngine.c | 11 +++++------ src/kit/shell/src/shellMain.c | 9 ++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 45a81c7b56..8f7f4d9324 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -284,7 +284,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); TAOS_RES* pSql = taos_query(con, command); - result = pSql; // set it into the global variable + atomic_store_ptr(&result, pSql); // set the global TAOS_RES pointer if (taos_errno(pSql)) { taos_error(pSql); @@ -295,7 +295,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } @@ -304,7 +304,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } @@ -327,7 +327,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { wordfree(&full_path); } - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); } @@ -493,7 +493,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { } while( row != NULL); result = NULL; - //taos_free_result(tres); fclose(fp); return numOfRows; @@ -798,8 +797,8 @@ void write_history() { } void taos_error(TAOS_RES *tres) { + atomic_store_ptr(&result, 0); fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); - result = NULL; taos_free_result(tres); } diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 8481f498dd..44de6641f6 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -18,11 +18,10 @@ pthread_t pid; -// TODO: IMPLEMENT INTERRUPT HANDLER. -void interruptHandler(int signum) { +void shellQueryInterruptHandler(int signum) { #ifdef LINUX - taos_stop_query(result); - result = NULL; + void* pResHandle = atomic_val_compare_exchange_64(&result, result, 0); + taos_stop_query(pResHandle); #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); @@ -86,7 +85,7 @@ int main(int argc, char* argv[]) { struct sigaction act; memset(&act, 0, sizeof(struct sigaction)); - act.sa_handler = interruptHandler; + act.sa_handler = shellQueryInterruptHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); From b16fa964d795051816337cbef8b69a3bfc407687 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 11:59:48 +0800 Subject: [PATCH 22/55] [td-225] refactor shell code. --- src/client/src/tscSql.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index cfcc9f0fa6..4c9439f0a7 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -665,6 +665,8 @@ void taos_stop_query(TAOS_RES *res) { tscDebug("%p start to cancel query", res); SSqlCmd *pCmd = &pSql->cmd; + // TODO there are multi-thread problem. + // It may have been released by the other thread already. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { tscKillSTableQuery(pSql); From f22098b6ddefd0e0bc4413f5f5adce2252a9f045 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 12:03:57 +0800 Subject: [PATCH 23/55] [td-225] refactor shell code. --- src/client/src/tscSql.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 4c9439f0a7..e243002ff6 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -667,6 +667,7 @@ void taos_stop_query(TAOS_RES *res) { // TODO there are multi-thread problem. // It may have been released by the other thread already. + // The ref count may fix this problem. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { tscKillSTableQuery(pSql); From 74fdb1c2aa9b390c81d305d1c8e4ede0bf8a03d5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 13:24:33 +0800 Subject: [PATCH 24/55] [td-225] refactor shell code to support ctrl+c. --- src/client/src/tscServer.c | 19 ++----------------- src/client/src/tscSql.c | 18 +++++++++++------- src/client/src/tscSubquery.c | 27 +++++++++++---------------- 3 files changed, 24 insertions(+), 40 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5a7a858ba1..3b380af76c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -455,35 +455,20 @@ void tscKillSTableQuery(SSqlObj *pSql) { } for (int i = 0; i < pSql->numOfSubs; ++i) { + // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; if (pSub == NULL) { continue; } + pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; if (pSub->pRpcCtx != NULL) { rpcCancelRequest(pSub->pRpcCtx); } - pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; tscQueueAsyncRes(pSub); } - /* - * 1. if the subqueries are not launched or partially launched, we need to waiting the launched - * query return to successfully free allocated resources. - * 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage, - * set the res.code, and return. - */ - const int64_t MAX_WAITING_TIME = 10000; // 10 Sec. - int64_t stime = taosGetTimestampMs(); - - while (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - taosMsleep(100); - if (taosGetTimestampMs() - stime > MAX_WAITING_TIME) { - break; - } - } - tscDebug("%p super table query cancelled", pSql); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index e243002ff6..2906e5f4e2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -669,15 +669,19 @@ void taos_stop_query(TAOS_RES *res) { // It may have been released by the other thread already. // The ref count may fix this problem. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillSTableQuery(pSql); - } - - if (pSql->cmd.command < TSDB_SQL_LOCAL) { - rpcCancelRequest(pSql->pRpcCtx); - } + // set the error code for master pSqlObj firstly pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + assert(pSql->pRpcCtx == NULL); + tscKillSTableQuery(pSql); + } else { + if (pSql->cmd.command < TSDB_SQL_LOCAL) { + rpcCancelRequest(pSql->pRpcCtx); + } + } + tscDebug("%p query is cancelled", res); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1aeaba650d..e99f5ab153 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1462,10 +1462,15 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tscDebug("%p start to free subquery result", pSql); - + + int32_t index = trsupport->subqueryIndex; + SSqlObj *pParentSql = trsupport->pParentSql; + + assert(pSql == pParentSql->pSubs[index]); + pParentSql->pSubs[index] = NULL; + taos_free_result(pSql); taosTFree(trsupport->localBuffer); - taosTFree(trsupport); } @@ -1474,17 +1479,7 @@ static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, i static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { // set no disk space error info -#ifdef WINDOWS - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); - LocalFree(lpMsgBuf); -#else tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code)); -#endif - SSqlObj* pParentSql = trsupport->pParentSql; pParentSql->res.code = code; @@ -1548,14 +1543,14 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO */ pSql->res.numOfRows = 0; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscDebug("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pParentSql, pSql, numOfRows, subqueryIndex); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { From 8ba7387e5394152edb7bbd586920eae369ece177 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 16:00:22 +0800 Subject: [PATCH 25/55] [td-225] refactor codes. --- cmake/define.inc | 2 +- src/tsdb/src/tsdbRead.c | 68 +++++++++++++++++++---------------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 0a6e70f375..005ec29fdd 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -32,7 +32,7 @@ ENDIF () IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -pg -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6d0966e52a..690bbc2cc7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -257,10 +257,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab for (int32_t j = 0; j < gsize; ++j) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); - STableCheckInfo info = { - .lastKey = pKeyInfo->lastKey, - .pTableObj = pKeyInfo->pTable, - }; + STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; info.tableId = ((STable*)(pKeyInfo->pTable))->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || @@ -432,7 +429,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -448,47 +445,46 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { } } - if (rmem != NULL && rimem != NULL) { - TSKEY r1 = dataRowKey(rmem); - TSKEY r2 = dataRowKey(rimem); - - if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; - } else { - if (ASCENDING_TRAVERSE(order)) { - if (r1 < r2) { - pCheckInfo->chosen = 0; - return rmem; - } else { - pCheckInfo->chosen = 1; - return rimem; - } - } else { - if (r1 < r2) { - pCheckInfo->chosen = 1; - return rimem; - } else { - pCheckInfo->chosen = 0; - return rmem; - } - } - } + if (rmem == NULL && rimem == NULL) { + return NULL; } - // at least one (rmem or rimem) is absent here - if (rmem != NULL) { + if (rmem != NULL && rimem == NULL) { pCheckInfo->chosen = 0; return rmem; } - if (rimem != NULL) { + if (rmem == NULL && rimem != NULL) { pCheckInfo->chosen = 1; return rimem; } - return NULL; + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } + } } static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { From 0fc1c8cdb502f84ce3bc091e3ea396b2499cf711 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 17:50:12 +0800 Subject: [PATCH 26/55] [td-837] check mem after malloc --- cmake/define.inc | 2 +- src/client/src/tscSubquery.c | 21 ++--- src/query/src/qExecutor.c | 160 +++++++++++++++++++++++++---------- src/tsdb/src/tsdbRead.c | 34 ++++++-- src/vnode/src/vnodeMain.c | 5 ++ 5 files changed, 161 insertions(+), 61 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 005ec29fdd..0a6e70f375 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -32,7 +32,7 @@ ENDIF () IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -pg -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e99f5ab153..ef42398001 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1502,7 +1502,7 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscDebug("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); @@ -1521,9 +1521,10 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // if failed to process sql, let following code handle the pSql if (ret == TSDB_CODE_SUCCESS) { taos_free_result(pSql); + return ret; + } else { + return ret; } - - return code; } void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1681,7 +1682,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscDebug("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, numOfRows); @@ -1692,13 +1693,13 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(numOfRows == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscDebug("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); + tscDebug("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort } @@ -1792,7 +1793,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscError("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, code); @@ -1810,12 +1811,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscWarn("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); + tscError("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort } @@ -1823,7 +1824,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } - tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + tscDebug("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 10963ea26c..c612837f5f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -113,6 +113,15 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { } } +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5 <= 1) { + return NULL; + } else { + return realloc(p, __size); + } +} + #define calloc u_calloc #define malloc u_malloc #endif @@ -430,7 +439,10 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { - createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } } pWindowResInfo->capacity = (int32_t)newCap; @@ -1465,7 +1477,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY } // set the output buffer for the selectivity + tag query -static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { SQuery* pQuery = pRuntimeEnv->pQuery; if (isSelectivityWithTagsQuery(pQuery)) { @@ -1474,6 +1486,9 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p SQLFunctionCtx *p = NULL; SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; @@ -1499,6 +1514,8 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p taosTFree(pTagCtx); } } + + return TSDB_CODE_SUCCESS; } static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { @@ -1600,7 +1617,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order resetCtxOutputBuf(pRuntimeEnv); } - setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { + goto _clean; + } qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; @@ -2232,7 +2251,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { pQuery->sdata[i] = (tFilePage *)tmp; } @@ -2263,7 +2282,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; @@ -2803,7 +2822,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { size_t size = taosArrayGetSize(pGroup); tFilePage **buffer = pQuery->sdata; - int32_t* posList = calloc(size, sizeof(int32_t)); + int32_t *posList = calloc(size, sizeof(int32_t)); STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { @@ -2861,6 +2880,10 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } char* buf = calloc(1, pRuntimeEnv->interBufSize); + if (buf == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); @@ -4307,6 +4330,10 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { int32_t offset = 0; SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); + if (pFillCol == NULL) { + return NULL; + } + for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; @@ -5433,6 +5460,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + if (pColInfo->filters == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } } for (int32_t f = 0; f < numOfFilters; ++f) { @@ -5447,6 +5478,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pColFilter->len = htobe64(pFilterMsg->len); pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + if (pColFilter->pz == 0) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + memcpy((void *)pColFilter->pz, pMsg, (size_t)pColFilter->len); pMsg += (pColFilter->len + 1); } else { @@ -5460,6 +5496,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (*pExpr == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { @@ -5546,6 +5587,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (*tagCols == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { SColumnInfo* pTagCol = (SColumnInfo*) pMsg; @@ -5562,6 +5608,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); + + if (*tagCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + + } memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } @@ -5752,6 +5804,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { } pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); + if (pQuery->pFilterInfo == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { if (pQuery->colList[i].numOfFilters > 0) { @@ -5762,6 +5817,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); + if (pFilterInfo->pFilters == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; @@ -5911,6 +5969,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQuery == NULL) { goto _cleanup_query; } + pQInfo->runtimeEnv.pQuery = pQuery; pQuery->numOfCols = numOfCols; @@ -5996,6 +6055,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pQInfo->pBuf == NULL) { + goto _cleanup; + } + int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { @@ -6010,8 +6073,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - STableId* id = TSDB_TABLEID(info->pTable); + STableId* id = TSDB_TABLEID(info->pTable); STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); if (pTableId != NULL ) { window.skey = pTableId->key; @@ -6140,35 +6203,59 @@ static void freeQInfo(SQInfo *pQInfo) { return; } - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; qDebug("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - taosTFree(pQuery->sdata[col]); - } teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { - SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; - if (pColFilter->numOfFilters > 0) { - taosTFree(pColFilter->pFilters); + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (pQuery != NULL) { + if (pQuery->sdata != NULL) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + taosTFree(pQuery->sdata[col]); + } + taosTFree(pQuery->sdata); } - } - if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; + if (pQuery->fillVal != NULL) { + taosTFree(pQuery->fillVal); + } - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(&pExprInfo->pExpr, NULL); + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; + if (pColFilter->numOfFilters > 0) { + taosTFree(pColFilter->pFilters); } } - taosTFree(pQuery->pSelectExpr); - } + if (pQuery->pSelectExpr != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SExprInfo *pExprInfo = &pQuery->pSelectExpr[i]; - if (pQuery->fillVal != NULL) { - taosTFree(pQuery->fillVal); + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(&pExprInfo->pExpr, NULL); + } + } + + taosTFree(pQuery->pSelectExpr); + } + + if (pQuery->pGroupbyExpr != NULL) { + taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); + taosTFree(pQuery->pGroupbyExpr); + } + + taosTFree(pQuery->tagColList); + taosTFree(pQuery->pFilterInfo); + + if (pQuery->colList != NULL) { + for (int32_t i = 0; i < pQuery->numOfCols; i++) { + SColumnInfo *column = pQuery->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + taosTFree(pQuery->colList); + } + + taosTFree(pQuery); } // todo refactor, extract method to destroytableDataInfo @@ -6193,24 +6280,7 @@ static void freeQInfo(SQInfo *pQInfo) { tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - if (pQuery->pGroupbyExpr != NULL) { - taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - taosTFree(pQuery->pGroupbyExpr); - } - taosTFree(pQuery->tagColList); - taosTFree(pQuery->pFilterInfo); - - if (pQuery->colList != NULL) { - for (int32_t i = 0; i < pQuery->numOfCols; i++) { - SColumnInfo* column = pQuery->colList + i; - freeColumnFilterInfo(column->filters, column->numOfFilters); - } - taosTFree(pQuery->colList); - } - - taosTFree(pQuery->sdata); - taosTFree(pQuery); pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -6786,12 +6856,16 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 60; // every 30 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + if (pQueryMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); pQueryMgmt->closed = false; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 690bbc2cc7..c89241a76b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1840,6 +1840,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return true; } else { STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + if (pSecQueryHandle == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return false; + } + pSecQueryHandle->order = TSDB_ORDER_ASC; pSecQueryHandle->window = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX}; pSecQueryHandle->pTsdb = pQueryHandle->pTsdb; @@ -1851,6 +1856,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock; if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; free(pSecQueryHandle); return false; } @@ -1862,6 +1868,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pSecQueryHandle->statis == NULL || pSecQueryHandle->pColumns == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; @@ -1869,6 +1880,12 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { colInfo.info = pCol->info; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes); + if (colInfo.pData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } + taosArrayPush(pSecQueryHandle->pColumns, &colInfo); } @@ -2280,6 +2297,10 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); + if (sa == NULL) { + taosArrayDestroy(pTableGroup); + return NULL; + } for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); @@ -2294,14 +2315,13 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosArrayPush(pTableGroup, &sa); tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { - STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); - pSupp->numOfCols = numOfOrderCols; - pSupp->pTagSchema = pTagSchema; - pSupp->pCols = pCols; + STableGroupSupporter sup = {0}; + sup.numOfCols = numOfOrderCols; + sup.pTagSchema = pTagSchema; + sup.pCols = pCols; - taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, skey, pSupp, tableGroupComparFn); - taosTFree(pSupp); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e5536bfaaf..456f2be83b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -295,6 +295,11 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); + if (pVnode->qMgmt == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } + pVnode->events = NULL; pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); From 51290fd1aa7f5eb08a146546045740b07e643653 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 19:02:58 +0800 Subject: [PATCH 27/55] [td-1067] --- src/query/inc/qExecutor.h | 2 +- src/query/src/qExecutor.c | 62 ++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b023de0dc9..7093495763 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -193,7 +193,7 @@ typedef struct SQInfo { int64_t owner; // if it is in execution void* tsdb; int32_t vgId; - STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> + STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; SArray* arrTableIdInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c612837f5f..79bbeb65a2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -124,6 +124,7 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define calloc u_calloc #define malloc u_malloc +#define realloc u_realloc #endif #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) @@ -3146,7 +3147,8 @@ static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - // update the last key in tableKeyInfo list + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); pTableKeyInfo->lastKey = pCheckInfo->lastKey; @@ -3607,12 +3609,6 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo) { cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } -#define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \ - do { \ - assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_q)) || \ - (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_q))); \ - } while (0) - /** * set output buffer for different group * @param pRuntimeEnv @@ -4316,6 +4312,34 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + + // update the query time window + pQuery->window = cond.twindow; + + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); + + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); + SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); + + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle + STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); + pCheckInfo->win.skey = pTableKeyInfo->lastKey; + pCheckInfo->win.ekey = pTableKeyInfo->lastKey; + + pCheckInfo->lastKey = pTableKeyInfo->lastKey; + + assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + } + } + + } else if (isPointInterpoQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } else { @@ -4361,6 +4385,9 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); setScanLimitationByResultBuffer(pQuery); + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + // TODO fixme changeExecuteScanOrder(pQInfo, false); code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); @@ -4512,7 +4539,17 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - CHECK_QUERY_TIME_RANGE(pQuery, *pTableQueryInfo); + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert( + ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); + } else { + assert( + ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); + } if (!pRuntimeEnv->groupbyNormalCol) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); @@ -4653,7 +4690,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (isFirstLastRowQuery(pQuery)) { assert(0); // last_row query switch to other routine to handle -// pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); } else { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } @@ -6069,6 +6105,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (p1 == NULL) { goto _cleanup; } + taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { @@ -6076,13 +6113,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, STableId* id = TSDB_TABLEID(info->pTable); STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); - if (pTableId != NULL ) { - window.skey = pTableId->key; - } else { - window.skey = pQueryMsg->window.skey; - } + window.skey = (pTableId != NULL)? pTableId->key:pQueryMsg->window.skey; void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; From e4e4c7fcd3b76c7b68cdaebf6ff468ce6a8b89cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 23:02:12 +0800 Subject: [PATCH 28/55] [td-1067] --- src/query/src/qExecutor.c | 37 +++++++++++-------------------------- src/tsdb/src/tsdbRead.c | 6 +----- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 79bbeb65a2..511658c3ce 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -97,7 +97,8 @@ typedef struct { #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + + if (v % 1000 <= 0) { return NULL; } else { return malloc(__size); @@ -106,7 +107,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + if (v % 1000 <= 0) { return NULL; } else { return calloc(num, __size); @@ -1887,16 +1888,12 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), - pQuery->order.order, TSDB_ORDER_DESC); - - pQuery->order.order = TSDB_ORDER_DESC; - - int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); - int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); - - pQuery->window.skey = ekey; - pQuery->window.ekey = skey; + pQuery->order.order, TSDB_ORDER_ASC); + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } return; } @@ -4316,30 +4313,18 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) // update the query time window pQuery->window = cond.twindow; - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); for(int32_t i = 0; i < numOfGroups; ++i) { SArray *group = GET_TABLEGROUP(pQInfo, i); - SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide - // the start check timestamp of tsdbQueryHandle - STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); - pCheckInfo->win.skey = pTableKeyInfo->lastKey; - pCheckInfo->win.ekey = pTableKeyInfo->lastKey; - - pCheckInfo->lastKey = pTableKeyInfo->lastKey; - - assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; } } - - } else if (isPointInterpoQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } else { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c89241a76b..f8ff25ddab 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -295,12 +295,8 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { - pCond->order = TSDB_ORDER_DESC; + pCond->order = TSDB_ORDER_ASC; pCond->twindow = changeTableGroupByLastrow(groupList); - - //descending order query, skey >= ekey - SWAP(pCond->twindow.skey, pCond->twindow.ekey, TSKEY); - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); return pQueryHandle; } From ba9ef1377df724578eb0aa87d06a9256f53ac78c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Aug 2020 15:36:06 +0800 Subject: [PATCH 29/55] [td-1151] add test cases and fix the found bugs. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 5 +- src/client/src/tscFunctionImpl.c | 21 ++- src/client/src/tscSQLParser.c | 96 +++++----- src/client/src/tscServer.c | 11 +- src/client/src/tscSubquery.c | 3 +- src/client/src/tscUtil.c | 9 +- src/common/inc/tname.h | 3 +- src/common/src/tname.c | 17 +- src/inc/taosdef.h | 2 +- src/inc/taosmsg.h | 2 +- src/query/inc/qSqlparser.h | 1 + src/query/src/qExecutor.c | 17 +- src/query/src/qParserImpl.c | 2 +- tests/script/general/parser/bug.sim | 274 ++++++++++++++++++++++++++++ 15 files changed, 376 insertions(+), 89 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 7c406d1676..afbbd1d24b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -198,7 +198,7 @@ int32_t tscValidateName(SSQLToken* pToken); void tscIncStreamExecutionCount(void* pStream); -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 65a868cf40..47f8732de9 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -195,9 +195,9 @@ typedef struct STableDataBlocks { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert/import type + uint32_t type; // query/insert type char slidingTimeUnit; - STimeWindow window; + STimeWindow window; // query time window int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info @@ -216,6 +216,7 @@ typedef struct SQueryInfo { char * msg; // pointer to the pCmd->payload to keep error message temporarily int64_t clauseLimit; // limit for current sub clause int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX } SQueryInfo; typedef struct { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index e494446ce5..b5c17e57b9 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2902,12 +2902,12 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { - if (pCtx->numOfParams == 1) { // the number of output rows should not affect the final number of rows, so set it to be 1 - INC_INIT_VAL(pCtx, 1); + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + SET_VAL(pCtx, pCtx->size, 1); char* output = pCtx->aOutputBuf; for(int32_t i = 0; i < pCtx->size; ++i) { - tVariantDump(&pCtx->param[0], output, pCtx->outputType, true); + tVariantDump(&pCtx->param[1], output, pCtx->outputType, true); output += pCtx->outputBytes; } @@ -2936,11 +2936,16 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - - INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - + + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + SET_VAL(pCtx, pCtx->size, 1); + tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->outputType, true); + } else { + INC_INIT_VAL(pCtx, 1); + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + } + pCtx->aOutputBuf += pCtx->inputBytes; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 204c91bc52..e43b0d16d0 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1094,18 +1094,6 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } -static void extractColumnNameFromString(tSQLExprItem* pItem) { - if (pItem->pNode->nSQLOptr == TK_STRING) { - pItem->pNode->val.nLen = strdequote(pItem->pNode->val.pz); - pItem->pNode->nSQLOptr = TK_ID; - - SSQLToken* pIdToken = &pItem->pNode->colInfo; - pIdToken->type = TK_ID; - pIdToken->z = pItem->pNode->val.pz; - pIdToken->n = pItem->pNode->val.nLen; - } -} - static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { const char* msg1 = "invalid column name, or illegal column type"; const char* msg2 = "invalid arithmetic expression in select clause"; @@ -1234,6 +1222,11 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t return TSDB_CODE_SUCCESS; } +static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnListInsert(pQueryInfo->colList, &tsCol); +} + int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { assert(pSelection != NULL && pCmd != NULL); @@ -1254,15 +1247,12 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // project on all fields int32_t optr = pItem->pNode->nSQLOptr; - if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || (optr == TK_INTEGER || optr == TK_FLOAT)) { + if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - // if the name of column is quoted, remove it and set the right information for later process - extractColumnNameFromString(pItem); - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -1292,6 +1282,13 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } + // there is only one user-defined column in the final result field, add the timestamp column. + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + SColumnIndex index = {0}; + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + } + if (!functionCompatibleCheck(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1431,17 +1428,11 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnListInsert(pQueryInfo->colList, &tsCol); -} - int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; - - int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); int32_t optr = pItem->pNode->nSQLOptr; if (optr == TK_ALL) { // project on all fields @@ -1465,40 +1456,43 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t // add the primary timestamp column even though it is not required by user tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); - } else if (optr == TK_ID || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query + } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; // user-specified constant value as a new result column - if ((optr == TK_INTEGER || optr == TK_FLOAT) || (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { - index.columnIndex = TSDB_UD_COLUMN_INDEX; - index.tableIndex = 0; + index.columnIndex = (pQueryInfo->udColumnId--); + index.tableIndex = 0; - SSchema colSchema = tGetUserSpecifiedColumnSchema(pItem->pNode->val.pz, pItem->pNode->val.nType, pItem->aliasName); - SSqlExpr* pExpr = tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); - pExpr->numOfParams = 1; - tVariantAssign(&pExpr->param[0], &pItem->pNode->val); + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, pItem->aliasName); + SSqlExpr* pExpr = + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); - } else { // columns from the queried table - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + // NOTE: the first parameter is reserved for the tag column id during join query process. + pExpr->numOfParams = 2; + tVariantAssign(&pExpr->param[1], &pItem->pNode->val); + } else if (optr == TK_ID) { + SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = tGetTableNameColumnSchema(); - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); - } else { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - addProjectQueryCol(pQueryInfo, startPos, &index, pItem); - } - - // add the primary timestamp column even though it is not required by user - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + SSchema colSchema = tGetTableNameColumnSchema(); + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); + } else { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + } + + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2069,7 +2063,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo refactor static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && columnIndex >= -2 && tableIndex >= 0); + assert(num == 1 && tableIndex >= 0); SColumnList columnList = {0}; columnList.num = num; @@ -4706,7 +4700,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu } } - if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF || + if ((pExpr->functionId == TSDB_FUNC_PRJ && pExpr->numOfParams == 0) || pExpr->functionId == TSDB_FUNC_DIFF || pExpr->functionId == TSDB_FUNC_ARITHM) { isProjectionFunction = true; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3b380af76c..b0c66a22cd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -628,9 +628,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - - if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); + + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + tscError("%p illegal value of numOfCols in query msg: %"PRIu64", table cols:%d", pSql, numOfSrcCols, + tscGetNumOfColumns(pTableMeta)); + return TSDB_CODE_TSC_INVALID_SQL; } @@ -728,7 +731,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return TSDB_CODE_TSC_INVALID_SQL; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index ef42398001..c50b663e02 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -353,11 +353,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr = tscSqlExprGet(pQueryInfo, 0); } - // set the join condition tag column info, to do extract method + // set the join condition tag column info, todo extract method if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pQueryInfo->tagCond.joinInfo.hasJoin); int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + // set the tag column id for executor to extract correct tag value pExpr->param[0].i64Key = colId; pExpr->numOfParams = 1; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index feae58090c..d8f030938e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -937,8 +937,8 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; - } else if (pColIndex->columnIndex == TSDB_UD_COLUMN_INDEX) { - pExpr->colInfo.colId = TSDB_UD_COLUMN_INDEX; + } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { + pExpr->colInfo.colId = pColIndex->columnIndex; } else { if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); @@ -1290,12 +1290,12 @@ void tscIncStreamExecutionCount(void* pStream) { ps->num += 1; } -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) { if (pTableMetaInfo->pTableMeta == NULL) { return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_UD_COLUMN_INDEX) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -1511,6 +1511,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 41d6fe7280..65f3d8c3f9 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -4,6 +4,7 @@ #include "os.h" #include "taosmsg.h" #include "tstoken.h" +#include "tvariant.h" typedef struct SDataStatis { int16_t colId; @@ -28,7 +29,7 @@ void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); SSchema tGetTableNameColumnSchema(); -SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name); +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name); bool tscValidateTableNameLength(size_t len); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 6aa308cb45..e59d20b9ce 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -4,6 +4,7 @@ #include "tname.h" #include "tstoken.h" #include "ttokendef.h" +#include "tvariant.h" // todo refactor UNUSED_FUNC static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { @@ -47,19 +48,23 @@ SSchema tGetTableNameColumnSchema() { return s; } -SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name) { +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name) { SSchema s = {0}; - s.type = type; + s.type = pVal->nType; if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { - size_t len = strlen(v); - s.bytes = len + VARSTR_HEADER_SIZE; + s.bytes = pVal->nLen + VARSTR_HEADER_SIZE; } else { - s.bytes = tDataTypeDesc[type].nSize; + s.bytes = tDataTypeDesc[pVal->nType].nSize; } s.colId = TSDB_UD_COLUMN_INDEX; - tstrncpy(s.name, name, sizeof(s.name)); + if (name != NULL) { + tstrncpy(s.name, name, sizeof(s.name)); + } else { + tVariantToString(pVal, s.name); + strdequote(s.name); + } return s; } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 12c4699665..d3c744b684 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -287,7 +287,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_REPLICA 5 #define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-2) +#define TSDB_UD_COLUMN_INDEX (-100) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 7a3fc6b050..54b302760f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -169,7 +169,7 @@ enum _mgmt_table { #define TSDB_COL_NORMAL 0x0u // the normal column of the table #define TSDB_COL_TAG 0x1u // the tag column type -#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column extern char *taosMsg[]; diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index cd9618b223..2cfee344aa 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -210,6 +210,7 @@ typedef struct tSQLExprItem { char * aliasName; // alias name, null-terminated string } tSQLExprItem; +// todo refactor by using SArray typedef struct tSQLExprList { int32_t nExpr; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 511658c3ce..205d7aab7d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -168,7 +168,7 @@ static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); - bool doFilterData(SQuery *pQuery, int32_t elemPos) { +bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; @@ -899,7 +899,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } assert(dataBlock != NULL); - sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/; // start from the offset + sas->data[i] = dataBlock; // start from the offset } } else { // other type of query function @@ -1583,7 +1583,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int16_t type = pSqlFuncMsg->arg[j].argType; int16_t bytes = pSqlFuncMsg->arg[j].argBytes; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg->argValue.pz, bytes, type); + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); } @@ -5728,11 +5728,12 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * SSchema s = tGetTableNameColumnSchema(); type = s.type; bytes = s.bytes; - } else if (pExprs[i].base.colInfo.colId == TSDB_UD_COLUMN_INDEX) { + } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX) { + // it is a user-defined constant value column assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); - type = pExprs[i].base.arg[0].argType; - bytes = pExprs[i].base.arg[0].argBytes; + type = pExprs[i].base.arg[1].argType; + bytes = pExprs[i].base.arg[1].argBytes; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { bytes += VARSTR_HEADER_SIZE; @@ -5927,8 +5928,8 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } assert(f < pQuery->numOfCols); - } else if (pColIndex->colId == TSDB_UD_COLUMN_INDEX) { - // do nothing + } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { + // do nothing for user-defined constant value result columns } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index d8f12da642..0537726ad5 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -78,7 +78,7 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken pList = calloc(1, sizeof(tSQLExprList)); } - if (pList->nAlloc <= pList->nExpr) { // + if (pList->nAlloc <= pList->nExpr) { pList->nAlloc = (pList->nAlloc << 1) + 4; pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); if (pList->a == 0) { diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/bug.sim index 2a46ad1fd6..889fe83042 100644 --- a/tests/script/general/parser/bug.sim +++ b/tests/script/general/parser/bug.sim @@ -40,4 +40,278 @@ sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql +print ==============select with user-defined columns +sql select 'abc' as f, ts,f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select 'abc', ts, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select 'abc' from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 'abc' as f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select 1 as f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select 1 as f, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1.123 as f, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.123000000 then + print expect 1.123000000 , actual:$data00 + return -1 +endi + +sql select 1, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1.2391, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +print ===================== user-defined columns with agg functions +sql select 1 as t, count(*) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1, sum(f1) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1,2,3, sum(f1)*99, 4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +if $data02 != 3 then + return -1 +endi + +if $data03 != 99.000000000 then + print expect 99.000000000, actual:$data03 + return -1 +endi + +sql select sum(f1)*avg(f1)+12, 1,2,3,4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 13.000000000 then + print expect 13.000000000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +sql select 1.2987, f1, 'k' from t1 where f1=1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.298700000 then + print expect 1.298700000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != @k@ then + return -1 +endi + +print ====================user-defined columns with union +sql select f1, 'f1' from t1 union all select f1, 'f1' from t1; +if $rows != 2 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != @f1@ then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data11 != @f1@ then + return -1 +endi + +print =====================udc with join +sql select st1.ts, st1.f1, st2.f2, 'abc', 1.9827 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts +if $rows != 1 then + return -1 +endi + +if $data00 != @19-12-09 16:27:35.000@ then + print expect @19-12-09 16:27:35.000@ actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +if $data03 != @abc@ then + return -1 +endi + +if $data04 != 1.982700000 then + print expect 1.982700000 actual:$data04 + return -1 +endi + +print ======================udc with interval +sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; +if $rows != 1 then + return -1 +endi + +print ======================udc with tags +sql select t1,'abc',tbname from st1 +if $rows != 1 then + return -1 +endi + +print ======================udc with arithmetic +sql select 1+1 from t1 +if $rows != 1 then + return -1 +endi + +sql_error select from t1 +sql_error select abc from t1 +sql_error select abc as tu from t1 + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5b5edf00264bf313ea6bec535614fad7634058c9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Aug 2020 16:13:31 +0800 Subject: [PATCH 30/55] [td-1010] add check for max interval time range. --- src/client/src/tscSQLParser.c | 2 +- src/inc/taoserror.h | 1 + src/query/inc/tsqlfunction.h | 8 ++++---- src/query/src/qExecutor.c | 5 +++++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e43b0d16d0..995a64ebe8 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6144,7 +6144,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (pQueryInfo->intervalTime > 0) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { + if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_INTERVAL_TIME_WINDOW) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 57c2b322fa..e58c24b531 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -220,6 +220,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag condit TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index a443a43a23..65ab82883b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -91,16 +91,16 @@ extern "C" { #define QUERY_COND_REL_PREFIX_IN "IN|" #define QUERY_COND_REL_PREFIX_LIKE "LIKE|" -#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_IN_LEN 3 #define QUERY_COND_REL_PREFIX_LIKE_LEN 5 -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -#define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 -#define TOP_BOTTOM_QUERY_LIMIT 100 +#define MAX_INTERVAL_TIME_WINDOW 10000000 +#define TOP_BOTTOM_QUERY_LIMIT 100 enum { MASTER_SCAN = 0x0u, diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 205d7aab7d..5baa3f2bf8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -455,6 +455,11 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } + // too many time window in query + if (pWindowResInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); } From dac0a985c2bd9aaf723ec70a4520fd9e9fac7b3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Aug 2020 22:50:16 +0800 Subject: [PATCH 31/55] [td-1213] --- src/client/inc/tscUtil.h | 4 +- src/client/src/tscParseInsert.c | 28 ++-- src/client/src/tscPrepare.c | 2 +- src/client/src/tscSQLParser.c | 139 +++++++++--------- src/client/src/tscServer.c | 8 +- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 6 +- src/common/inc/tname.h | 4 +- src/common/inc/tvariant.h | 2 +- src/common/src/tname.c | 8 +- src/common/src/tvariant.c | 14 +- src/query/inc/qAst.h | 2 +- src/query/inc/qSqlparser.h | 84 +++++------ src/query/inc/sql.y | 30 ++-- src/query/src/qAst.c | 10 +- src/query/src/qParserImpl.c | 94 ++++++------ src/query/src/qTokenizer.c | 4 +- src/query/src/sql.c | 10 +- src/query/tests/unitTest.cpp | 8 +- src/util/inc/tstoken.h | 8 +- .../general/parser/{bug.sim => constCol.sim} | 0 tests/script/general/parser/testSuite.sim | 2 +- 22 files changed, 239 insertions(+), 230 deletions(-) rename tests/script/general/parser/{bug.sim => constCol.sim} (100%) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index afbbd1d24b..42959ade3c 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -141,7 +141,7 @@ bool tscQueryTags(SQueryInfo* pQueryInfo); SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); @@ -194,7 +194,7 @@ SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); -int32_t tscValidateName(SSQLToken* pToken); +int32_t tscValidateName(SStrToken* pToken); void tscIncStreamExecutionCount(void* pStream); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cb49bd80b7..939ebb6abe 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -40,7 +40,7 @@ enum { static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); -static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { +static int32_t tscToInteger(SStrToken *pToken, int64_t *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -73,7 +73,7 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { return pToken->type; } -static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { +static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -89,9 +89,9 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { return pToken->type; } -int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { +int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int64_t interval; int64_t useconds = 0; char * pTokenEnd = *next; @@ -128,7 +128,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 * time expression: * e.g., now+12a, now-5h */ - SSQLToken valueToken; + SStrToken valueToken; index = 0; sToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); pTokenEnd += index; @@ -163,7 +163,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } -int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, char *msg, char **str, bool primaryKey, +int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec) { int64_t iv; int32_t numType; @@ -409,7 +409,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken = {0}; + SStrToken sToken = {0}; char * payload = pDataBlocks->pData + pDataBlocks->size; // 1. set the parsed value from sql string @@ -524,7 +524,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, SParsedDataColInfo *spd, char *error, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int16_t numOfRows = 0; @@ -734,8 +734,8 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { int32_t index = 0; - SSQLToken sToken = {0}; - SSQLToken tableToken = {0}; + SStrToken sToken = {0}; + SStrToken tableToken = {0}; int32_t code = TSDB_CODE_SUCCESS; const int32_t TABLE_INDEX = 0; @@ -993,7 +993,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } -int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { +int validateTableName(char *tblName, int len, SStrToken* psTblToken) { tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); psTblToken->n = len; @@ -1057,7 +1057,7 @@ int tsParseInsertSql(SSqlObj *pSql) { while (1) { int32_t index = 0; - SSQLToken sToken = tStrGetToken(str, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(str, &index, false, 0, NULL); // no data in the sql string anymore. if (sToken.n == 0) { @@ -1083,7 +1083,7 @@ int tsParseInsertSql(SSqlObj *pSql) { pCmd->curSql = sToken.z; char buf[TSDB_TABLE_FNAME_LEN]; - SSQLToken sTblToken; + SStrToken sTblToken; sTblToken.z = buf; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { @@ -1285,7 +1285,7 @@ int tsInsertInitialCheck(SSqlObj *pSql) { int32_t index = 0; SSqlCmd *pCmd = &pSql->cmd; - SSQLToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); pCmd->count = 0; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index b996dd958a..620e8ea57a 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -155,7 +155,7 @@ static int normalStmtPrepare(STscStmt* stmt) { uint32_t i = 0, start = 0; while (sql[i] != 0) { - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(sql + i, &token.type); if (token.type == TK_QUESTION) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 995a64ebe8..bc2b2672d3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -51,12 +51,12 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name); -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken); -static bool hasSpecifyDB(SSQLToken* pTableName); +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken); +static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd); -static int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* len); +static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); @@ -100,11 +100,11 @@ static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); -static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); -static int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); @@ -151,7 +151,7 @@ static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo return TSDB_CODE_SUCCESS; } -static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) { +static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { const char* msg1 = "password can not be empty"; const char* msg2 = "name or password too long"; const char* msg3 = "password needs single quote marks enclosed"; @@ -206,7 +206,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; - SSQLToken* pzName = &pInfo->pDCLInfo->a[0]; + SStrToken* pzName = &pInfo->pDCLInfo->a[0]; if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -241,7 +241,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_USE_DB: { const char* msg = "invalid db name"; - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); @@ -296,7 +296,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0]; + SStrToken* pIpAddr = &pInfo->pDCLInfo->a[0]; pIpAddr->n = strdequote(pIpAddr->z); break; } @@ -307,8 +307,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; - SSQLToken* pName = &pInfo->pDCLInfo->user.user; - SSQLToken* pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken* pName = &pInfo->pDCLInfo->user.user; + SStrToken* pPwd = &pInfo->pDCLInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -337,7 +337,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_DESCRIBE_TABLE: { - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg2 = "table name is too long"; const char* msg1 = "invalid table name"; @@ -400,8 +400,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { // tDCLSQL* pDCL = pInfo->pDCLInfo; SUserInfo* pUser = &pInfo->pDCLInfo->user; - SSQLToken* pName = &pUser->user; - SSQLToken* pPwd = &pUser->passwd; + SStrToken* pName = &pUser->user; + SStrToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -423,7 +423,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); - SSQLToken* pPrivilege = &pUser->privilege; + SStrToken* pPrivilege = &pUser->privilege; if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { pCmd->count = 1; @@ -581,7 +581,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // interval is not null - SSQLToken* t = &pQuerySql->interval; + SStrToken* t = &pQuerySql->interval; if (getTimestampInUsFromStr(t->z, t->n, &pQueryInfo->intervalTime) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -667,7 +667,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SSQLToken* pSliding = &pQuerySql->sliding; + SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n != 0) { getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { @@ -692,7 +692,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -709,7 +709,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableNa // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); } else { // get current DB name first, then set it into path - SSQLToken t = {0}; + SStrToken t = {0}; getCurrentDBName(pSql, &t); code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); @@ -1027,13 +1027,13 @@ static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken) { +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken) { pDBToken->z = pSql->pTscObj->db; pDBToken->n = (uint32_t)strlen(pSql->pTscObj->db); } /* length limitation, strstr cannot be applied */ -static bool hasSpecifyDB(SSQLToken* pTableName) { +static bool hasSpecifyDB(SStrToken* pTableName) { for (uint32_t i = 0; i < pTableName->n; ++i) { if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { return true; @@ -1043,7 +1043,7 @@ static bool hasSpecifyDB(SSQLToken* pTableName) { return false; } -int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* xlen) { +int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { int32_t totalLen = 0; if (account != NULL) { @@ -1463,7 +1463,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t index.columnIndex = (pQueryInfo->udColumnId--); index.tableIndex = 0; - SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, pItem->aliasName); + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); SSqlExpr* pExpr = tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); @@ -1489,6 +1489,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } // add the primary timestamp column even though it is not required by user @@ -1579,7 +1580,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - SSQLToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; + SStrToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; if (pToken->z == NULL || pToken->n == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1588,7 +1589,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* // check if the table name is valid or not - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1786,7 +1787,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -2088,16 +2089,16 @@ void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, c snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); } -static bool isTablenameToken(SSQLToken* token) { - SSQLToken tmpToken = *token; - SSQLToken tableToken = {0}; +static bool isTablenameToken(SStrToken* token) { + SStrToken tmpToken = *token; + SStrToken tableToken = {0}; extractTableNameFromToken(&tmpToken, &tableToken); return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken* pToken) { +static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); @@ -2119,7 +2120,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken return columnIndex; } -int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { const char* msg0 = "ambiguous column name"; const char* msg1 = "invalid column name"; @@ -2161,7 +2162,7 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQu } } -int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pTableToken->n == 0) { // only one table and no table name prefix in column name if (pQueryInfo->numOfTables == 1) { pIndex->tableIndex = 0; @@ -2187,8 +2188,8 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum return TSDB_CODE_SUCCESS; } -int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SSQLToken tableToken = {0}; +int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { @@ -2198,12 +2199,12 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn return TSDB_CODE_SUCCESS; } -int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken tmpToken = *pToken; + SStrToken tmpToken = *pToken; if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2315,7 +2316,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { // db prefix in tagCond, show table conds in payload - SSQLToken* pDbPrefixToken = &pShowInfo->prefix; + SStrToken* pDbPrefixToken = &pShowInfo->prefix; if (pDbPrefixToken->type != 0) { if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long @@ -2337,7 +2338,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show table/stable like 'xxxx', set the like pattern for show tables - SSQLToken* pPattern = &pShowInfo->pattern; + SStrToken* pPattern = &pShowInfo->pattern; if (pPattern->type != 0) { pPattern->n = strdequote(pPattern->z); @@ -2355,7 +2356,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show vnodes may be ip addr of dnode in payload - SSQLToken* pDnodeIp = &pShowInfo->prefix; + SStrToken* pDnodeIp = &pShowInfo->prefix; if (pDnodeIp->n >= TSDB_IPv4ADDR_LEN) { // ip addr is too long return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2376,7 +2377,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SSqlCmd* pCmd = &pSql->cmd; pCmd->command = pInfo->type; - SSQLToken* idStr = &(pInfo->pDCLInfo->ip); + SStrToken* idStr = &(pInfo->pDCLInfo->ip); if (idStr->n > TSDB_KILL_MSG_LEN) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2597,7 +2598,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; - SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &token, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -3623,7 +3624,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* return; } - SSQLToken t = {0}; + SStrToken t = {0}; extractTableNameFromToken(&pLeft->colInfo, &t); *pOut = *pExpr; @@ -3703,7 +3704,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, num = j; char* name = extractDBName(pTableMetaInfo->name, db); - SSQLToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; + SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; for (int32_t i = 0; i < num; ++i) { if (i >= 1) { @@ -3712,7 +3713,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; int32_t xlen = (int32_t)strlen(segments[i]); - SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; + SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { @@ -4033,7 +4034,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t return TSDB_CODE_TSC_INVALID_SQL; } } else { - SSQLToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; + SStrToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; int32_t len = tSQLGetToken(pRight->val.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { @@ -4279,7 +4280,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } - SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query @@ -4348,7 +4349,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu } tVariant* pVar2 = &pSortorder->a[1].pVar; - SSQLToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; + SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4477,7 +4478,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; + SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4515,12 +4516,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } - SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; + SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); } @@ -4544,7 +4545,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t numOfTags = tscGetNumOfTags(pTableMeta); SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; + SStrToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -4637,7 +4638,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pItem = &pAlterSQL->varList->a[0]; SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; + SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } @@ -4757,7 +4758,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, }; - SSQLToken* pOptionToken = &pOptions->a[1]; + SStrToken* pOptionToken = &pOptions->a[1]; if (pOptions->nTokens == 2) { // reset log and reset query cache does not need value @@ -4769,7 +4770,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t vnodeId = 0; int32_t dnodeId = 0; strdequote(pValToken->z); @@ -4780,14 +4781,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } return TSDB_CODE_SUCCESS; } else { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { @@ -4816,7 +4817,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - SSQLToken* pOptionToken = &pOptions->a[0]; + SStrToken* pOptionToken = &pOptions->a[0]; if (pOptions->nTokens == 1) { // reset log does not need value @@ -4827,7 +4828,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { } } } else { - SSQLToken* pValToken = &pOptions->a[1]; + SStrToken* pValToken = &pOptions->a[1]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 131 || val > 199) { @@ -4852,7 +4853,7 @@ int32_t validateColumnName(char* name) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken token = {.z = name}; + SStrToken token = {.z = name}; token.n = tSQLGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { @@ -5031,7 +5032,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SSQLToken* pToken = &pCreateDbInfo->precision; + SStrToken* pToken = &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -5702,7 +5703,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p assert(pFieldList != NULL); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5757,7 +5758,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst - SSQLToken* pToken = &(pCreateTable->usingInfo.stableName); + SStrToken* pToken = &(pCreateTable->usingInfo.stableName); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5850,7 +5851,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); SQuerySQL* pQuerySql = pCreateTable->pSelect; if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { @@ -5863,7 +5864,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } tVariant* pVar = &pSrcMeterName->a[0].pVar; - SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -6004,7 +6005,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pTableItem->nLen = strdequote(pTableItem->pz); - SSQLToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; + SStrToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -6015,13 +6016,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); - SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; + SStrToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar; - SSQLToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; + SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); } @@ -6243,7 +6244,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SSQLToken t = {.type = pSqlExpr->nSQLOptr}; + SStrToken t = {.type = pSqlExpr->nSQLOptr}; (*pExpr)->_node.optr = getBinaryExprOptr(&t); assert((*pExpr)->_node.optr != 0); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b0c66a22cd..e33bfe44d1 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -931,8 +931,8 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload; - SSQLToken *pName = &pInfo->pDCLInfo->user.user; - SSQLToken *pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken *pName = &pInfo->pDCLInfo->user.user; + SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; strncpy(pAlterMsg->user, pName->z, pName->n); strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); @@ -1134,13 +1134,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SSQLToken *pPattern = &pShowInfo->pattern; + SStrToken *pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query strncpy(pShowMsg->payload, pPattern->z, pPattern->n); pShowMsg->payloadLen = htons(pPattern->n); } } else { - SSQLToken *pEpAddr = &pShowInfo->prefix; + SStrToken *pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 2906e5f4e2..1a774e8784 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -855,7 +855,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t str = nextStr + 1; len = (int32_t)strtrim(tblName); - SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName}; + SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; tSQLGetToken(tblName, &sToken.type); // Check if the table name available or not diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d8f030938e..788283a1ad 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -744,7 +744,7 @@ bool tscIsInsertData(char* sqlstr) { int32_t index = 0; do { - SSQLToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); + SStrToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); if (t0.type != TK_LP) { return t0.type == TK_INSERT || t0.type == TK_IMPORT; } @@ -1187,7 +1187,7 @@ void tscColumnListDestroy(SArray* pColumnList) { * 'first_part.second_part' * */ -static int32_t validateQuoteToken(SSQLToken* pToken) { +static int32_t validateQuoteToken(SStrToken* pToken) { strdequote(pToken->z); pToken->n = (uint32_t)strtrim(pToken->z); @@ -1203,7 +1203,7 @@ static int32_t validateQuoteToken(SSQLToken* pToken) { return TSDB_CODE_SUCCESS; } -int32_t tscValidateName(SSQLToken* pToken) { +int32_t tscValidateName(SStrToken* pToken) { if (pToken->type != TK_STRING && pToken->type != TK_ID) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 65f3d8c3f9..2a4ac3fc40 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -25,11 +25,11 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); -void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); +void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); SSchema tGetTableNameColumnSchema(); -SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name); +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); bool tscValidateTableNameLength(size_t len); diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 211635b4af..e9973bcb95 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -36,7 +36,7 @@ typedef struct tVariant { }; } tVariant; -void tVariantCreate(tVariant *pVar, SSQLToken *token); +void tVariantCreate(tVariant *pVar, SStrToken *token); void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index e59d20b9ce..2eda5953a3 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -48,7 +48,7 @@ SSchema tGetTableNameColumnSchema() { return s; } -SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name) { +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) { SSchema s = {0}; s.type = pVal->nType; @@ -62,8 +62,8 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name) { if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - tVariantToString(pVal, s.name); - strdequote(s.name); + size_t len = MIN(sizeof(s.name), exprStr->n + 1); + tstrncpy(s.name, exprStr->z, len); } return s; @@ -132,7 +132,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in * tablePrefix.columnName * extract table name and save it in pTable, with only column name in pToken */ -void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) { +void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) { const char sep = TS_PATH_DELIMITER[0]; if (pToken == pTable || pToken == NULL || pTable == NULL) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 6e8111aa72..4036e578b4 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -23,7 +23,7 @@ #include "tutil.h" // todo support scientific expression number and oct number -void tVariantCreate(tVariant *pVar, SSQLToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } +void tVariantCreate(tVariant *pVar, SStrToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { memset(pVar, 0, sizeof(tVariant)); @@ -270,7 +270,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -319,7 +319,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -478,7 +478,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { - SSQLToken stoken = {.z = pStr, .n = len}; + SStrToken stoken = {.z = pStr, .n = len}; if (TK_ILLEGAL == isValidNumber(&stoken)) { return -1; @@ -504,7 +504,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -521,7 +521,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result return 0; } - SSQLToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; + SStrToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; if (TK_ILLEGAL == isValidNumber(&sToken)) { return -1; } @@ -557,7 +557,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index ec568a6cdb..547616dee6 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -80,7 +80,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); -uint8_t getBinaryExprOptr(SSQLToken *pToken); +uint8_t getBinaryExprOptr(SStrToken *pToken); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 2cfee344aa..69fc0bc7ef 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -26,7 +26,7 @@ extern "C" { #include "tstoken.h" #include "tvariant.h" -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken extern char tTokenTypeSwitcher[13]; #define toTSDBType(x) \ @@ -71,16 +71,16 @@ typedef struct SQuerySQL { struct tSQLExpr * pWhere; // where clause [optional] tVariantList * pGroupby; // groupby clause, only for tags[optional] tVariantList * pSortOrder; // orderby [optional] - SSQLToken interval; // interval [optional] - SSQLToken sliding; // sliding window [optional] + SStrToken interval; // interval [optional] + SStrToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] SLimitVal slimit; // group limit offset [optional] tVariantList * fillType; // fill type[optional] - SSQLToken selectToken; // sql string + SStrToken selectToken; // sql string } SQuerySQL; typedef struct SCreateTableSQL { - struct SSQLToken name; // meter name, create table [meterName] xxx + struct SStrToken name; // meter name, create table [meterName] xxx bool existCheck; int8_t type; // create normal table/from super table/ stream @@ -90,7 +90,7 @@ typedef struct SCreateTableSQL { } colInfo; struct { - SSQLToken stableName; // super table name, for using clause + SStrToken stableName; // super table name, for using clause tVariantList *pTagVals; // create by using metric, tag value STagData tagdata; } usingInfo; @@ -99,7 +99,7 @@ typedef struct SCreateTableSQL { } SCreateTableSQL; typedef struct SAlterTableSQL { - SSQLToken name; + SStrToken name; int16_t type; STagData tagData; @@ -108,7 +108,7 @@ typedef struct SAlterTableSQL { } SAlterTableSQL; typedef struct SCreateDBInfo { - SSQLToken dbname; + SStrToken dbname; int32_t replica; int32_t cacheBlockSize; int32_t maxTablesPerVnode; @@ -121,7 +121,7 @@ typedef struct SCreateDBInfo { int32_t walLevel; int32_t quorum; int32_t compressionLevel; - SSQLToken precision; + SStrToken precision; bool ignoreExists; tVariantList *keep; @@ -136,33 +136,33 @@ typedef struct SCreateAcctSQL { int64_t maxStorage; int64_t maxQueryTime; int32_t maxConnections; - SSQLToken stat; + SStrToken stat; } SCreateAcctSQL; typedef struct SShowInfo { uint8_t showType; - SSQLToken prefix; - SSQLToken pattern; + SStrToken prefix; + SStrToken pattern; } SShowInfo; typedef struct SUserInfo { - SSQLToken user; - SSQLToken passwd; - SSQLToken privilege; + SStrToken user; + SStrToken passwd; + SStrToken privilege; int16_t type; } SUserInfo; typedef struct tDCLSQL { int32_t nTokens; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ - SSQLToken *a; /* one entry for element */ + SStrToken *a; /* one entry for element */ bool existsCheck; union { SCreateDBInfo dbOpt; SCreateAcctSQL acctOpt; SShowInfo showOpt; - SSQLToken ip; + SStrToken ip; }; SUserInfo user; @@ -194,14 +194,14 @@ typedef struct tSQLExpr { // the full sql string of function(col, param), which is actually the raw // field name, since the function name is kept in nSQLOptr already - SSQLToken operand; - SSQLToken colInfo; // field id + SStrToken operand; + SStrToken colInfo; // field id tVariant val; // value only for string, float, int struct tSQLExpr *pLeft; // left child struct tSQLExpr *pRight; // right child - struct tSQLExprList *pParam; // function parameters + SStrToken token; // original sql expr string } tSQLExpr; // used in select clause. select from xxx @@ -243,7 +243,7 @@ tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t so tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder); +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder); void tVariantListDestroy(tVariantList *pList); tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField); @@ -254,61 +254,61 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType); void tSQLExprDestroy(tSQLExpr *); -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken); +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); void tSQLExprListDestroy(tSQLExprList *pList); -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pMetricName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type); void tSQLExprNodeDestroy(tSQLExpr *pExpr); tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr); -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); tSQLExprListList *tSQLListListAppend(tSQLExprListList *pList, tSQLExprList *pExprList); void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); -SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type); +SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists); void SQLInfoDestroy(SSqlInfo *pInfo); void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck); -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns); +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken); +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists); +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo); -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd); -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip); -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege); +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip); +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); // prefix show db.tables; -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB); +void setDBName(SStrToken *pCpxName, SStrToken *pDB); -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pToken, int32_t optType); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optType); -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType); +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType); +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *pToken); +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken); void *ParseAlloc(void *(*mallocProc)(size_t)); diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index edf59327ab..7500dcba56 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -2,8 +2,8 @@ //usage: lemon sql.y %token_prefix TK_ -%token_type {SSQLToken} -%default_type {SSQLToken} +%token_type {SStrToken} +%default_type {SStrToken} %extra_argument {SSqlInfo* pInfo} %fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. @@ -80,11 +80,11 @@ cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } cmd ::= SHOW VNODES IPTOKEN(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } -%type dbPrefix {SSQLToken} +%type dbPrefix {SStrToken} dbPrefix(A) ::=. {A.n = 0; A.type = 0;} dbPrefix(A) ::= ids(X) DOT. {A = X; } -%type cpxName {SSQLToken} +%type cpxName {SStrToken} cpxName(A) ::= . {A.n = 0; } cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } @@ -101,19 +101,19 @@ cmd ::= SHOW dbPrefix(X) STABLES. { } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } @@ -145,7 +145,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} @@ -153,15 +153,15 @@ cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(p // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. // And "ids" is an identifer-or-string. -%type ids {SSQLToken} +%type ids {SStrToken} ids(A) ::= ID(X). {A = X; } ids(A) ::= STRING(X). {A = X; } -%type ifexists {SSQLToken} +%type ifexists {SStrToken} ifexists(X) ::= IF EXISTS. {X.n = 1;} ifexists(X) ::= . {X.n = 0;} -%type ifnotexists {SSQLToken} +%type ifnotexists {SStrToken} ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} ifnotexists(X) ::= . {X.n = 0;} @@ -416,7 +416,7 @@ selcollist(A) ::= sclp(P) STAR. { // An option "AS " phrase that can follow one of the expressions that // define the result set, or one of the tables in the FROM clause. // -%type as {SSQLToken} +%type as {SStrToken} as(X) ::= AS ids(Y). { X = Y; } as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } @@ -458,10 +458,10 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SSQLToken} +%type tmvar {SStrToken} tmvar(A) ::= VARIABLE(X). {A = X;} -%type interval_opt {SSQLToken} +%type interval_opt {SStrToken} interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N = E; } interval_opt(N) ::= . {N.n = 0; N.z = NULL; N.type = 0; } @@ -482,7 +482,7 @@ fill_opt(N) ::= FILL LP ID(Y) RP. { N = tVariantListAppendToken(NULL, &Y, -1); } -%type sliding_opt {SSQLToken} +%type sliding_opt {SStrToken} sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index afa3618a96..634f014d97 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -44,7 +44,7 @@ * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14 * */ -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken); static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); static void destroySyntaxTree(tExprNode *); @@ -103,7 +103,7 @@ static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, u } } -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken) { +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) { /* if the token is not a value, return false */ if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID && pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) { @@ -117,7 +117,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken int32_t i = 0; if (pToken->type == TK_ID) { do { - SSQLToken tableToken = {0}; + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); size_t len = strlen(pSchema[i].name); @@ -157,7 +157,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken return pNode; } -uint8_t getBinaryExprOptr(SSQLToken *pToken) { +uint8_t getBinaryExprOptr(SStrToken *pToken) { switch (pToken->type) { case TK_LT: return TSDB_RELATION_LESS; @@ -234,7 +234,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef } static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); + SStrToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 0537726ad5..0b5408f793 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -33,7 +33,7 @@ SSqlInfo qSQLParse(const char *pStr) { int32_t i = 0; while (1) { - SSQLToken t0 = {0}; + SStrToken t0 = {0}; if (pStr[i] == 0) { Parse(pParser, 0, t0, &sqlInfo); @@ -73,7 +73,7 @@ abort_parse: return sqlInfo; } -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken) { +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { if (pList == NULL) { pList = calloc(1, sizeof(tSQLExprList)); } @@ -117,41 +117,45 @@ void tSQLExprListDestroy(tSQLExprList *pList) { free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pAliasToken, int32_t optrType) { - tSQLExpr *nodePtr = calloc(1, sizeof(tSQLExpr)); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { + tSQLExpr *pSQLExpr = calloc(1, sizeof(tSQLExpr)); + + if (pToken != NULL) { + pSQLExpr->token = *pToken; + } if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pAliasToken->type); + toTSDBType(pToken->type); - tVariantCreate(&nodePtr->val, pAliasToken); - nodePtr->nSQLOptr = optrType; + tVariantCreate(&pSQLExpr->val, pToken); + pSQLExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // default use microsecond - nodePtr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = getTimestampInUsFromStr(pAliasToken->z, pAliasToken->n, &nodePtr->val.i64Key); + int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &pSQLExpr->val.i64Key); UNUSED(ret); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; } else { // it must be the column name (tk_id) if it is not the number assert(optrType == TK_ID || optrType == TK_ALL); - if (pAliasToken != NULL) { - nodePtr->colInfo = *pAliasToken; + if (pToken != NULL) { + pSQLExpr->colInfo = *pToken; } - nodePtr->nSQLOptr = optrType; + pSQLExpr->nSQLOptr = optrType; } - return nodePtr; + return pSQLExpr; } /* * pList is the parameters for function with id(optType) * function name is denoted by pFunctionToken */ -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType) { +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { if (pFuncToken == NULL) return NULL; tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); @@ -173,8 +177,12 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || - optrType == TK_REM) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = endPos - pExpr->token.z; + pExpr->token.type = pLeft->token.type; + + if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { /* * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. @@ -373,7 +381,7 @@ void tVariantListDestroy(tVariantList *pList) { free(pList); } -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder) { +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) { if (pList == NULL) { pList = calloc(1, sizeof(tVariantList)); } @@ -382,9 +390,9 @@ tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToke return pList; } - if (pAliasToken) { + if (pToken) { tVariant t = {0}; - tVariantCreate(&t, pAliasToken); + tVariantCreate(&t, pToken); tVariantListItem *pItem = &pList->a[pList->nExpr++]; memcpy(pItem, &t, sizeof(tVariant)); @@ -420,7 +428,7 @@ void tFieldListDestroy(tFieldList *pList) { free(pList); } -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB) { +void setDBName(SStrToken *pCpxName, SStrToken *pDB) { pCpxName->type = pDB->type; pCpxName->z = pDB->z; pCpxName->n = pDB->n; @@ -464,7 +472,7 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { return 0; } -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); // truncate the column name @@ -479,7 +487,7 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) pField->bytes = pType->bytes; } -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = -1; int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar. @@ -517,9 +525,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL)); @@ -611,7 +619,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { taosTFree(pClause->pClause); } -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pStableName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) { SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); @@ -644,7 +652,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLTo return pCreate; } -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); pAlterTable->name = *pMeterName; @@ -716,7 +724,7 @@ SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { return pSubclause; } -SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type) { +SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type) { pInfo->type = type; if (type == TSDB_SQL_SELECT) { @@ -745,7 +753,7 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists) { +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists) { pInfo->pCreateTableInfo->name = *pMeterName; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } @@ -760,7 +768,7 @@ void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { } } -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken) { +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken) { if (pToken == NULL) return NULL; if (pTokenList == NULL) pTokenList = calloc(1, sizeof(tDCLSQL)); @@ -781,19 +789,19 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { va_start(va, nParam); while (nParam-- > 0) { - SSQLToken *pToken = va_arg(va, SSQLToken *); + SStrToken *pToken = va_arg(va, SStrToken *); pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); } va_end(va); } -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck) { +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { pInfo->type = type; pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); } -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns) { +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); } @@ -816,7 +824,7 @@ void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* } } -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists) { +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -827,7 +835,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -843,7 +851,7 @@ void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken } } -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->type = TSDB_SQL_CREATE_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -855,7 +863,7 @@ void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { pInfo->pDCLInfo->user.passwd = *pPasswd; } -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege) { +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { pInfo->type = TSDB_SQL_ALTER_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -880,7 +888,7 @@ void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* } } -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip) { +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -909,5 +917,5 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; - memset(&pDBInfo->precision, 0, sizeof(SSQLToken)); + memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 6fb63c4a20..5fdabca166 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -580,8 +580,8 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return 0; } -SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { - SSQLToken t0 = {0}; +SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { + SStrToken t0 = {0}; // here we reach the end of sql string, null-terminated string if (str[*i] == 0) { diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 0b977977af..f3f55791b0 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -99,7 +99,7 @@ #define YYCODETYPE unsigned short int #define YYNOCODE 274 #define YYACTIONTYPE unsigned short int -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; @@ -2074,21 +2074,21 @@ static void yy_reduce( break; case 22: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; case 23: /* cmd ::= SHOW dbPrefix VGROUPS */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; case 24: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } @@ -2139,7 +2139,7 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index b59e0783a2..b6e1170d0e 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -16,7 +16,7 @@ namespace { int32_t testValidateName(char* name) { - SSQLToken token = {0}; + SStrToken token = {0}; token.z = name; token.n = strlen(name); token.type = 0; @@ -720,8 +720,8 @@ TEST(testCase, tGetToken_Test) { EXPECT_FALSE(type == TK_HEX); } -static SSQLToken createStrToken(char* s) { - SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; +static SStrToken createStrToken(char* s) { + SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; t.type = TK_STRING; t.z = s; t.n = strlen(s); @@ -730,7 +730,7 @@ static SSQLToken createStrToken(char* s) { } TEST(testCase, isValidNumber_test) { - SSQLToken t1 = createStrToken("123abc"); + SStrToken t1 = createStrToken("123abc"); EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index c1c6f2de7a..7aeb2da6b6 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -28,11 +28,11 @@ extern "C" { #define TSQL_TBNAME_L "tbname" // used to denote the minimum unite in sql parsing -typedef struct SSQLToken { +typedef struct SStrToken { uint32_t n; uint32_t type; char * z; -} SSQLToken; +} SStrToken; /** * tokenizer for sql string @@ -52,7 +52,7 @@ uint32_t tSQLGetToken(char *z, uint32_t *tokenType); * @param ignoreTokenTypes * @return */ -SSQLToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); +SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); /** * check if it is a keyword or not @@ -76,7 +76,7 @@ bool isKeyWord(const char *z, int32_t len); * @param pToken * @return token type, if it is not a number, TK_ILLEGAL will return */ -static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) { +static FORCE_INLINE int32_t isValidNumber(const SStrToken* pToken) { const char* z = pToken->z; int32_t type = TK_ILLEGAL; diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/constCol.sim similarity index 100% rename from tests/script/general/parser/bug.sim rename to tests/script/general/parser/constCol.sim diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index aafba2d328..4e26d14cfd 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -97,7 +97,7 @@ run general/parser/topbot.sim sleep 2000 run general/parser/union.sim sleep 2000 -run general/parser/bug.sim +run general/parser/constCol.sim sleep 2000 run general/parser/sliding.sim From a7f58f94378ffdaa34c9bdd026e70eb73dca26df Mon Sep 17 00:00:00 2001 From: Amos Kong Date: Tue, 25 Aug 2020 07:17:09 +0800 Subject: [PATCH 32/55] install.sh: create empty history file after installation There is a error message in first use of taos, because the history file doesn't exist. This patch tried to create an empty history file after installation, then users won't see the error message. | [root@045a9307c53a ~]# taos | | Welcome to the TDengine shell from Linux, Client Version:2.0.1.1 | Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. | | Failed to open file /root/.taos_history | taos> Signed-off-by: Amos Kong --- packaging/tools/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 64de09df6d..ee500a727b 100644 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -713,6 +713,7 @@ function install_TDengine() { echo echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" fi + touch ~/.taos_history rm -rf $(tar -tf taos.tar.gz) } From a3e77f23fa943a56a57ed52c00725831b54c32ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 10:56:53 +0800 Subject: [PATCH 33/55] [td-1213] --- src/client/src/tscSQLParser.c | 36 ++++++++++------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index bc2b2672d3..3b91b381ef 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1111,42 +1111,31 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } int32_t tableIndex = columnList.ids[0].tableIndex; - - // todo potential data overflow - char* arithmeticExprStr = malloc(1024*1024); - char* p = arithmeticExprStr; - if (arithmeticType == NORMAL_ARITHMETIC) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; // all columns in arithmetic expression must belong to the same table for (int32_t f = 1; f < columnList.num; ++f) { if (columnList.ids[f].tableIndex != tableIndex) { - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } } - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double), false); - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); + char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; + size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1); + tstrncpy(pExpr->aliasName, name, len); tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); if (ret != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); @@ -1157,7 +1146,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t SColIndex* pIndex = taosArrayGet(colList, k); if (TSDB_COL_IS_TAG(pIndex->flag)) { tExprTreeDestroy(&pNode, NULL); - taosTFree(arithmeticExprStr); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1174,7 +1162,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t // TODO: other error handling } END_TRY - size_t len = tbufTell(&bw); + len = tbufTell(&bw); char* c = tbufGetData(&bw, true); // set the serialized binary string as the parameter of arithmetic expression @@ -1185,16 +1173,14 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); } else { - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, name, NULL); + char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; + size_t len = MIN(sizeof(pItem->aliasName), pItem->pNode->token.n + 1); + tstrncpy(pItem->aliasName, name, len); + + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pItem->aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); @@ -1210,7 +1196,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } @@ -1218,7 +1203,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } } - taosTFree(arithmeticExprStr); return TSDB_CODE_SUCCESS; } @@ -3170,7 +3154,7 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { +static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { char* start = *str; int32_t code = doArithmeticExprToString(pExpr, str); From 77816db719215dae11b7d6d1f75bd071c2abf7bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 10:57:35 +0800 Subject: [PATCH 34/55] [td-225] --- src/inc/taoserror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index e58c24b531..eda5fd0f3c 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -220,7 +220,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag condit TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") From 2271ac926014683f6eef1c95817827521fc0d622 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 15:53:39 +0800 Subject: [PATCH 35/55] [td-1227] --- src/mnode/src/mnodeProfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 01a824baa7..85457d7a26 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -35,8 +35,8 @@ #include "mnodeVgroup.h" #include "mnodeWrite.h" -#define CONN_KEEP_TIME (tsShellActivityTimer * 3000) -#define CONN_CHECK_TIME (tsShellActivityTimer * 2000) +#define CONN_KEEP_TIME (tsShellActivityTimer * 3) +#define CONN_CHECK_TIME (tsShellActivityTimer * 2) #define QUERY_ID_SIZE 20 #define QUERY_STREAM_SAVE_SIZE 20 From 73fcf1e1ce2c5e4d2eebe8bc7681c41e374d652f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 15:56:51 +0800 Subject: [PATCH 36/55] [td-1151] --- src/client/inc/tsclient.h | 39 +++++++++++++++------------ src/client/src/tscFunctionImpl.c | 45 +++++++++++++------------------- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 47f8732de9..aecbdd8207 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -433,31 +433,36 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField int32_t bytes = pInfo->pSqlExpr->resBytes; char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t realLen = varDataLen(pData); - assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + // user defined constant value output columns + if (pInfo->pSqlExpr->colInfo.flag == TSDB_COL_UDC) { + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + pData = pInfo->pSqlExpr->param[1].pz; + pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; + pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : pData; } else { - pRes->tsrow[columnIndex] = ((tstr*)pData)->data; - } + assert(bytes == tDataTypeDesc[type].nSize); - if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor - *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : &pInfo->pSqlExpr->param[1].i64Key; + pRes->length[columnIndex] = bytes; } - - pRes->length[columnIndex] = realLen; } else { - assert(bytes == tDataTypeDesc[type].nSize); + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : ((tstr *)pData)->data; + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor + *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + } + + pRes->length[columnIndex] = realLen; } else { - pRes->tsrow[columnIndex] = pData; - } + assert(bytes == tDataTypeDesc[type].nSize); - pRes->length[columnIndex] = bytes; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : pData; + pRes->length[columnIndex] = bytes; + } } } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index b5c17e57b9..a0ef40d367 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2902,27 +2902,20 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { - if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 - SET_VAL(pCtx, pCtx->size, 1); + // the number of output rows should not affect the final number of rows, so set it to be 0 + if (pCtx->numOfParams == 2) { + return; + } - char* output = pCtx->aOutputBuf; - for(int32_t i = 0; i < pCtx->size; ++i) { - tVariantDump(&pCtx->param[1], output, pCtx->outputType, true); - output += pCtx->outputBytes; - } + INC_INIT_VAL(pCtx, pCtx->size); + char *pData = GET_INPUT_CHAR(pCtx); + if (pCtx->order == TSDB_ORDER_ASC) { + memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { - - INC_INIT_VAL(pCtx, pCtx->size); - - char *pData = GET_INPUT_CHAR(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); - } else { - for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); - } + for(int32_t i = 0; i < pCtx->size; ++i) { + memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, + pCtx->inputBytes); } } @@ -2931,20 +2924,18 @@ static void col_project_function(SQLFunctionCtx *pCtx) { static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + return; + } + // only one output if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 - SET_VAL(pCtx, pCtx->size, 1); - tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->outputType, true); - } else { - INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - } + INC_INIT_VAL(pCtx, 1); + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pCtx->aOutputBuf += pCtx->inputBytes; } From 18637381a43e30779a028cabc7d96e0c75edbaf0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 11:03:42 +0800 Subject: [PATCH 37/55] [td-1151] --- src/client/src/tscSQLParser.c | 104 ++++++++++++++--------- src/common/src/tname.c | 6 +- src/query/src/qExecutor.c | 15 +++- src/query/src/qParserImpl.c | 2 + tests/script/general/parser/constCol.sim | 77 ++++++++++++----- 5 files changed, 138 insertions(+), 66 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3b91b381ef..e9fc27bb4e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -65,11 +65,11 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int3 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t changeFunctionID(int32_t optr, int16_t* functionId); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); @@ -1176,11 +1176,15 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; - char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; - size_t len = MIN(sizeof(pItem->aliasName), pItem->pNode->token.n + 1); - tstrncpy(pItem->aliasName, name, len); + char aliasName[TSDB_COL_NAME_LEN] = {0}; + if (pItem->aliasName != NULL) { + tstrncpy(aliasName, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + int32_t nameLen = MIN(TSDB_COL_NAME_LEN, pItem->pNode->token.n + 1); + tstrncpy(aliasName, pItem->pNode->token.z, nameLen); + } - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pItem->aliasName, NULL); + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); @@ -1211,7 +1215,30 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* tscColumnListInsert(pQueryInfo->colList, &tsCol); } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { + SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); + + char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; + tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); + + SColumnList ids = {0}; + ids.num = 1; + ids.ids[0] = *pIndex; + + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || + pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { + ids.num = 0; + } + + insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); +} + +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { assert(pSelection != NULL && pCmd != NULL); const char* msg2 = "functions can not be mixed up"; @@ -1252,12 +1279,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (code != TSDB_CODE_SUCCESS) { return code; } - } else { - /* - * not support such expression - * e.g., select 12+5 from table_name - */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1270,10 +1292,22 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { SColumnIndex index = {0}; - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + + // set the constant column value always attached to first table. + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, clauseIndex, 0); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); + + // add the timestamp column into the output columns + int32_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); + tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + + SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); + pSupInfo->visible = false; + + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } - if (!functionCompatibleCheck(pQueryInfo)) { + if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1336,29 +1370,6 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { - SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - - char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); - - SColumnList ids = {0}; - ids.num = 1; - ids.ids[0] = *pIndex; - - if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || - pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { - ids.num = 0; - } - - insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); -} - SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, @@ -1837,10 +1848,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i + j, &index) != - 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } + + colIndex++; } numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -2513,7 +2525,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t startIdx = 0; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); @@ -2545,6 +2557,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { if (functionCompatList[functionId] != factor) { return false; } + + if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { + return false; + } } return true; @@ -5863,7 +5879,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6046,7 +6062,8 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6080,6 +6097,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } } else { // set the time rang pQueryInfo->window = TSWINDOW_INITIALIZER; + if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } } // user does not specified the query time window, twa is not allowed in such case. diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 2eda5953a3..960cc7d725 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -62,8 +62,10 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t len = MIN(sizeof(s.name), exprStr->n + 1); - tstrncpy(s.name, exprStr->z, len); + size_t len = strdequote(exprStr->z); + size_t tlen = MIN(sizeof(s.name), len + 1); + + tstrncpy(s.name, exprStr->z, tlen); } return s; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5baa3f2bf8..19324bc2cb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1770,13 +1770,20 @@ static bool needReverseScan(SQuery *pQuery) { return false; } +/** + * The following 4 kinds of query are treated as the tags query + * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query + */ static bool onlyQueryTags(SQuery* pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; int32_t functionId = pExprInfo->base.functionId; - if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG && - (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX))) { + + if (functionId != TSDB_FUNC_TAGPRJ && + functionId != TSDB_FUNC_TID_TAG && + (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == TSDB_FUNC_PRJ && pExprInfo->base.colInfo.flag == TSDB_COL_UDC))) { return false; } } @@ -6835,6 +6842,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) { char *data = NULL, *dst = NULL; int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (pExprInfo[j].base.colInfo.flag == TSDB_COL_UDC) { + continue; + } if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { bytes = tbnameSchema.bytes; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 0b5408f793..733459c6fe 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -167,6 +167,8 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt pExpr->operand.n = len; // raw field name pExpr->operand.type = pFuncToken->type; + + pExpr->token = pExpr->operand; return pExpr; } diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim index 889fe83042..a196ba2b50 100644 --- a/tests/script/general/parser/constCol.sim +++ b/tests/script/general/parser/constCol.sim @@ -34,6 +34,9 @@ sql create table t1 using st1 tags(1); sql create table t2 using st2 tags(1); sql insert into t1 values(1575880055000, 1); +sql insert into t1 values(1575880059000, 1); +sql insert into t1 values(1575880069000, 1); + sql insert into t2 values(1575880055000, 2); sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts @@ -42,7 +45,7 @@ system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl print ==============select with user-defined columns sql select 'abc' as f, ts,f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -59,7 +62,7 @@ if $data02 != 1 then endi sql select 'abc', ts, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -71,8 +74,12 @@ if $data02 != 1 then return -1 endi +if $data10 != @abc@ then + return -1 +endi + sql select 'abc' from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -81,7 +88,7 @@ if $data00 != @abc@ then endi sql select 'abc' as f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -90,7 +97,7 @@ if $data00 != @abc@ then endi sql select 1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -98,8 +105,16 @@ if $data00 != 1 then return -1 endi +if $data10 != 1 then + return -1 +endi + +if $data20 != 1 then + return -1 +endi + sql select 1 as f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -108,7 +123,7 @@ if $data00 != 1 then endi sql select 1 as f, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -121,7 +136,7 @@ if $data01 != 1 then endi sql select 1.123 as f, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -130,8 +145,13 @@ if $data00 != 1.123000000 then return -1 endi +if $data10 != 1.123000000 then + print expect 1.123000000 , actual:$data10 + return -1 +endi + sql select 1, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -143,8 +163,12 @@ if $data01 != 1 then return -1 endi +if $data10 != 1 then + return -1 +endi + sql select 1.2391, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -153,6 +177,11 @@ if $data00 != 1.239100000 then return -1 endi +if $data10 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + if $data01 != 1 then return -1 endi @@ -167,7 +196,7 @@ if $data00 != 1 then return -1 endi -if $data01 != 1 then +if $data01 != 3 then return -1 endi @@ -180,7 +209,7 @@ if $data00 != 1 then return -1 endi -if $data01 != 1 then +if $data01 != 3 then return -1 endi @@ -201,8 +230,8 @@ if $data02 != 3 then return -1 endi -if $data03 != 99.000000000 then - print expect 99.000000000, actual:$data03 +if $data03 != 297.000000000 then + print expect 297.000000000, actual:$data03 return -1 endi @@ -211,8 +240,8 @@ if $rows != 1 then return -1 endi -if $data00 != 13.000000000 then - print expect 13.000000000 actual:$data00 +if $data00 != 15.000000000 then + print expect 15.000000000 actual:$data00 return -1 endi @@ -225,7 +254,7 @@ if $data02 != 2 then endi sql select 1.2987, f1, 'k' from t1 where f1=1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -244,7 +273,7 @@ endi print ====================user-defined columns with union sql select f1, 'f1' from t1 union all select f1, 'f1' from t1; -if $rows != 2 then +if $rows != 6 then return -1 endi @@ -294,7 +323,7 @@ endi print ======================udc with interval sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -304,9 +333,17 @@ if $rows != 1 then return -1 endi +if $data01 != @abc@ then + return -1 +endi + +if $data02 != @t1@ then + return -1 +endi + print ======================udc with arithmetic sql select 1+1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi From 5e17894fbae8932ba8a70c0caad1d2ff0f5847e8 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 26 Aug 2020 14:33:03 +0800 Subject: [PATCH 38/55] Update architecture-ch.md --- documentation20/webdocs/markdowndocs/architecture-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index bfe3b55bd2..7ab4b5d096 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -82,7 +82,7 @@ TDengine 分布式架构的逻辑结构图如下： ### 节点之间的通讯 **通讯方式：**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。 -**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过选项“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接使用IP地址作为FQDN，但不建议使用，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。 +**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。 **端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP链接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 From 53a7e3d62bc26fd66e190ebead996f4894e4b6df Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 15:51:52 +0800 Subject: [PATCH 39/55] [td-225] add test case for join query. --- src/client/inc/tsclient.h | 1 - src/client/src/tscAsync.c | 3 +- src/client/src/tscServer.c | 9 ++-- src/client/src/tscSql.c | 2 - src/client/src/tscSubquery.c | 68 +++++++++++++++------------- src/client/src/tscUtil.c | 4 +- src/util/inc/tlockfree.h | 8 ++++ tests/pytest/crash_gen.sh | 2 +- tests/script/general/parser/join.sim | 5 +- 9 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index aecbdd8207..004c5bc21c 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -324,7 +324,6 @@ typedef struct SSqlObj { SSqlRes res; uint16_t numOfSubs; struct SSqlObj **pSubs; - tsem_t subReadySem; struct SSqlObj * prev, *next; } SSqlObj; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b05aef76eb..4643d255dc 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -433,7 +433,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); goto _error; } else { - tscDebug("%p get tableMeta successfully", pSql); + const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; + tscDebug("%p get %s successfully", pSql, msg); } if (pSql->pStream == NULL) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e33bfe44d1..81be0625ea 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -454,6 +454,8 @@ void tscKillSTableQuery(SSqlObj *pSql) { return; } + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + for (int i = 0; i < pSql->numOfSubs; ++i) { // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; @@ -466,7 +468,7 @@ void tscKillSTableQuery(SSqlObj *pSql) { rpcCancelRequest(pSub->pRpcCtx); } - tscQueueAsyncRes(pSub); + tscQueueAsyncRes(pSub); // async res? not other functions? } tscDebug("%p super table query cancelled", pSql); @@ -1436,11 +1438,6 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { return code; } - // all subquery have completed already - if (pRes->pLocalReducer == NULL) { - sem_wait(&pSql->subReadySem); - } - pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1a774e8784..8d33368280 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -295,8 +295,6 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { } tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->subReadySem, 0, 0); - doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); tsem_wait(&pSql->rspSem); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c50b663e02..4e2edde31e 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1146,7 +1146,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { ///////////////////////////////////////////////////////////////////////////////////////// static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; @@ -1411,7 +1411,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSql = pSql; trs->pFinalColModel = pModel; - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); + SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); taosTFree(trs->localBuffer); @@ -1451,18 +1451,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscDebug("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); tscProcessSql(pSub); } - - // set the command flag must be after the semaphore been correctly set. - pSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, 0); - } return TSDB_CODE_SUCCESS; } static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { - tscDebug("%p start to free subquery result", pSql); + tscDebug("%p start to free subquery obj", pSql); int32_t index = trsupport->subqueryIndex; SSqlObj *pParentSql = trsupport->pParentSql; @@ -1503,10 +1497,10 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - tscDebug("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscError("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); - SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); + SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); @@ -1584,9 +1578,17 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - tsem_post(&pParentSql->subReadySem); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); + + if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { + (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); + } else { // regular super table query + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pParentSql); + } + } } static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { @@ -1659,8 +1661,13 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - // all subqueries are completed, retrieve from local can be proceeded. - tsem_post(&pParentSql->subReadySem); + // set the command flag must be after the semaphore been correctly set. + pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; + if (pParentSql->res.code == TSDB_CODE_SUCCESS) { + (*pParentSql->fp)(pParentSql->param, pParentSql, 0); + } else { + tscQueueAsyncRes(pParentSql); + } } static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { @@ -1669,21 +1676,22 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t idx = trsupport->subqueryIndex; SSqlObj * pParentSql = trsupport->pParentSql; + assert(tres != NULL); SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // sql object has been released in error process, return immediately - tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); - return; - } - +// if (pSql == NULL) { // sql object has been released in error process, return immediately +// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); +// return; +// } + SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscDebug("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscDebug("%p query cancelled/failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, numOfRows); @@ -1694,7 +1702,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(numOfRows == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscDebug("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; @@ -1745,11 +1753,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR (int32_t)pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); if (ret != 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - } else if (pRes->completed) { tscAllDataRetrievedFromDnode(trsupport, pSql); - return; - } else { // continue fetch data from dnode taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } @@ -1759,15 +1764,15 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } } -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { const int32_t table_index = 0; SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1); + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->numOfSubs); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); @@ -1784,7 +1789,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj* pParentSql = trsupport->pParentSql; SSqlObj* pSql = (SSqlObj *) tres; - + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); @@ -1812,7 +1817,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscWarn("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } @@ -2099,7 +2104,6 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlRes *pRes = &pSql->res; assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker taosTFree(pRes->tsrow); return pRes->tsrow; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 788283a1ad..9f23ac2bdc 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -373,7 +373,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } - + tscDebug("%p start to free sql object", pSql); tscPartiallyFreeSqlObj(pSql); @@ -388,7 +388,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); - tsem_destroy(&pSql->subReadySem); free(pSql); } @@ -1759,6 +1758,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; + SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); diff --git a/src/util/inc/tlockfree.h b/src/util/inc/tlockfree.h index a81f597832..e960b601ca 100644 --- a/src/util/inc/tlockfree.h +++ b/src/util/inc/tlockfree.h @@ -36,6 +36,14 @@ typedef void (*_ref_fn_t)(const void* pObj); _ref_fn_t end; \ } _ref_func = {.begin = (s), .end = (e)}; +// set the initial reference count value +#define T_REF_INIT_VAL(x, _v) \ + do { \ + assert(_v >= 0); \ + atomic_store_32(&((x)->_ref.val), (_v)); \ + } while (0) + +// increase the reference count by 1 #define T_REF_INC(x) (atomic_add_fetch_32(&((x)->_ref.val), 1)) #define T_REF_INC_WITH_CB(x, p) \ diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index f6be6aae49..df1a9f595b 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -49,4 +49,4 @@ export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR # Now we are all let, and let's see if we can find a crash. Note we pass all params -python3 ./crash_gen.py $@ +python3.8 ./crash_gen.py $@ diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 882f561ae1..c4e490190d 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -330,7 +330,10 @@ sql_error select join_tb1.* from $tb1 , $tb2 where join_tb1.ts != join_tb0.ts an sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts and join_tb1.ts >= 100000; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by ts; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by join_tb1.c7; - +sql_error select * from $tb1, $tb2; +sql_error select last_row(*) from $tb1, $tb2 +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts < now +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts = join_tb2.ts print ==================================super table join ============================== # select duplicate columns From 3326df48736da2a86c0246617778ed278372be2e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 16:01:53 +0800 Subject: [PATCH 40/55] [td-225] change python script version. --- tests/pytest/crash_gen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index df1a9f595b..f6be6aae49 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -49,4 +49,4 @@ export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR # Now we are all let, and let's see if we can find a crash. Note we pass all params -python3.8 ./crash_gen.py $@ +python3 ./crash_gen.py $@ From 6e43fe5217e9aea45ece66d659dd1e61c0c7a761 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 18:24:25 +0800 Subject: [PATCH 41/55] [td-621] --- src/client/src/tscSubquery.c | 10 +++++++++- src/client/src/tscUtil.c | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1c7047996c..e222c9748c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1912,6 +1912,10 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { assert(size > 0); pSql->pSubs = calloc(size, POINTER_BYTES); + if (pSql->pSubs == NULL) { + goto _error; + } + pSql->numOfSubs = (uint16_t)size; tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); @@ -1925,6 +1929,10 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); + if (pSupporter == NULL) { + goto _error; + } + pSupporter->pSql = pSql; pSupporter->pState = pState; pSupporter->index = numOfSub; @@ -1957,7 +1965,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { if (numOfSub < pSql->numOfSubs) { tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; // free all allocated resource + goto _error; } pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9f23ac2bdc..6dd8973f65 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1526,8 +1526,11 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + if (pQueryInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tscInitQueryInfo(pQueryInfo); - pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; From a393cc7524a56a2e16ac6935116d034a426506f7 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 27 Aug 2020 09:26:31 +0800 Subject: [PATCH 42/55] fix td-1242 --- src/dnode/src/dnodeSystem.c | 4 ++-- src/sync/src/tarbitrator.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 6f32bc0f7a..b286c0f219 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -120,8 +120,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { - if (res != EINTR) { - syslog(LOG_ERR, "failed to wait exit semphore: %d", res); + if (errno != EINTR) { + syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); break; } } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 625c0d6838..79ddb073de 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort); for (int res = tsem_wait(&tsArbSem); res != 0; res = tsem_wait(&tsArbSem)) { - if (res != EINTR) break; + if (errno != EINTR) break; } taosCloseTcpThreadPool(tsArbTcpPool); From 829caf39d38be113774ae236ba737c3d902cff52 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 27 Aug 2020 10:07:58 +0800 Subject: [PATCH 43/55] td-1242: wait again if sem_wait returns EINTR --- src/dnode/src/dnodeSystem.c | 7 ++----- src/os/inc/osSemphone.h | 2 +- src/os/src/detail/osSemphone.c | 12 ++++++++++++ src/sync/src/tarbitrator.c | 4 +--- src/util/src/tsched.c | 13 ++----------- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index b286c0f219..56316e9619 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -119,11 +119,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); - for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { - if (errno != EINTR) { - syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); - break; - } + if (tsem_wait(&exitSem) != 0) { + syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); } dnodeCleanUpSystem(); diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index fd88d2d798..4280b458a6 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -23,7 +23,7 @@ extern "C" { #ifndef TAOS_OS_FUNC_SEMPHONE #define tsem_t sem_t #define tsem_init sem_init - #define tsem_wait sem_wait + int tsem_wait(tsem_t* sem); #define tsem_post sem_post #define tsem_destroy sem_destroy #endif diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 74f8859029..b91888845e 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -16,6 +16,18 @@ #define _DEFAULT_SOURCE #include "os.h" +#ifndef TAOS_OS_FUNC_SEMPHONE + +int tsem_wait(tsem_t* sem) { + int ret = 0; + do { + ret = sem_wait(sem); + } while (ret != 0 && errno == EINTR); + return ret; +} + +#endif + #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 79ddb073de..eea46a2495 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -103,9 +103,7 @@ int main(int argc, char *argv[]) { sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort); - for (int res = tsem_wait(&tsArbSem); res != 0; res = tsem_wait(&tsArbSem)) { - if (errno != EINTR) break; - } + tsem_wait(&tsArbSem); taosCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down\n"); diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index cf7f5c10d4..f014dd0fab 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -123,11 +123,6 @@ void *taosProcessSchedQueue(void *param) { while (1) { if (tsem_wait(&pSched->fullSem) != 0) { - if (errno == EINTR) { - /* sem_wait is interrupted by interrupt, ignore and continue */ - uDebug("wait %s fullSem was interrupted", pSched->label); - continue; - } uError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); } if (pSched->stop) { @@ -163,12 +158,8 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { return 0; } - while (tsem_wait(&pSched->emptySem) != 0) { - if (errno != EINTR) { - uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); - break; - } - uDebug("wait %s emptySem was interrupted", pSched->label); + if (tsem_wait(&pSched->emptySem) != 0) { + uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) From 65f750a087e18b4ffcffdc3379cacb97c0994e5a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 27 Aug 2020 06:47:59 +0000 Subject: [PATCH 44/55] fixbug connection killing led floating point exception --- src/client/src/tscProfile.c | 2 +- src/client/src/tscServer.c | 10 +++++++--- src/inc/taoserror.h | 1 + src/mnode/src/mnodeProfile.c | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f48e7b7691..6ff97e9d00 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -285,9 +285,9 @@ void tscKillConnection(STscObj *pObj) { SSqlObj *pSql = pObj->sqlList; while (pSql) { - //taosStopRpcConn(pSql->thandle); pSql = pSql->next; } + SSqlStream *pStream = pObj->streamList; while (pStream) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9282fa74fb..4f179adf72 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -226,13 +226,17 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = &pSql->pRpcCtx, .code = 0 }; - // NOTE: the rpc context should be acquired before sending data to server. // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); - return TSDB_CODE_SUCCESS; + if (pObj != NULL && pObj->signature == pObj) { + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + return TSDB_CODE_SUCCESS; + } else { + //pObj->signature has been reset by other thread, ignore concurrency problem + return TSDB_CODE_TSC_CONN_KILLED; + } } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 57c2b322fa..e2ba7b03c1 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -96,6 +96,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Applicatio TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 85457d7a26..06f992c26a 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -100,7 +100,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { }; tstrncpy(connObj.user, user, sizeof(connObj.user)); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME * 1000); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; From 13cd54637b12aae0001dd69ba58c74ab5e42ef08 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 14:50:53 +0800 Subject: [PATCH 45/55] [td-255] fix compiler error --- src/client/src/tscSubquery.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e222c9748c..eaae026f06 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1911,13 +1911,12 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); + pSql->numOfSubs = (uint16_t)size; pSql->pSubs = calloc(size, POINTER_BYTES); if (pSql->pSubs == NULL) { goto _error; } - pSql->numOfSubs = (uint16_t)size; - tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); From d5abee89dde1bef4d3ff20dcfdc4966bc8154ab2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 15:04:19 +0800 Subject: [PATCH 46/55] [td-255] fix compiler error --- src/client/src/tscSubquery.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index eaae026f06..011940a0e6 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1911,6 +1911,9 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); + // the number of already initialized subqueries + int32_t numOfSub = 0; + pSql->numOfSubs = (uint16_t)size; pSql->pSubs = calloc(size, POINTER_BYTES); if (pSql->pSubs == NULL) { @@ -1924,7 +1927,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pState->numOfRemain = pSql->numOfSubs; pRes->code = TSDB_CODE_SUCCESS; - int32_t numOfSub = 0; while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); From 76ad0bc70e4bc6f74d09c843e765cfd56970fdbe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 18:32:34 +0800 Subject: [PATCH 47/55] [td-255] add check for memory allocation failure. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 34 +++++++++ src/client/src/tscParseInsert.c | 13 ++-- src/client/src/tscSQLParser.c | 39 +++++++--- src/client/src/tscServer.c | 15 ++-- src/client/src/tscSql.c | 7 +- src/client/src/tscSubquery.c | 13 ++++ src/client/src/tscUtil.c | 107 ++++++++++++++++++--------- src/query/src/qParserImpl.c | 10 ++- tests/script/general/parser/join.sim | 4 +- 10 files changed, 174 insertions(+), 70 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 42959ade3c..228e54a1c7 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -217,7 +217,7 @@ STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); -int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); +SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 004c5bc21c..c59f305b58 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -35,6 +35,40 @@ extern "C" { #include "qTsbuf.h" #include "tcmdtype.h" +#if 0 +static UNUSED_FUNC void *u_malloc (size_t __size) { + uint32_t v = rand(); + + if (v % 5000 <= 0) { + return NULL; + } else { + return malloc(__size); + } +} + +static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return calloc(num, __size); + } +} + +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return realloc(p, __size); + } +} + +#define calloc u_calloc +#define malloc u_malloc +#define realloc u_realloc +#endif + // forward declaration struct SSqlInfo; struct SLocalReducer; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 939ebb6abe..47bfe0fcdc 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1031,11 +1031,11 @@ int tsParseInsertSql(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); assert(pQueryInfo != NULL); - STableMetaInfo *pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = terrno; + return code; } if ((code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -1292,8 +1292,7 @@ int tsInsertInitialCheck(SSqlObj *pSql) { pCmd->command = TSDB_SQL_INSERT; pSql->res.numOfRows = 0; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e9fc27bb4e..5600ee3be9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -179,20 +179,24 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_TSC_APP_ERROR; } - SSqlCmd* pCmd = &(pSql->cmd); - SQueryInfo* pQueryInfo = NULL; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), pInfo->pzErrMsg); } - int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pRes->code = terrno; + return pRes->code; + } - STableMetaInfo* pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + STableMetaInfo* pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo) : pQueryInfo->pTableMetaInfo[0]; + if (pTableMetaInfo == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; } pCmd->command = pInfo->type; @@ -487,9 +491,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg1 = "columns in select clause not identical"; for (int32_t i = pCmd->numOfClause; i < pInfo->subclauseInfo.numOfClause; ++i) { - SQueryInfo* pqi = NULL; - if ((code = tscGetQueryInfoDetailSafely(pCmd, i, &pqi)) != TSDB_CODE_SUCCESS) { - return code; + SQueryInfo* pqi = tscGetQueryInfoDetailSafely(pCmd, i); + if (pqi == NULL) { + pRes->code = terrno; + return pRes->code; } } @@ -2678,9 +2683,12 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { } int32_t size = pColumn->numOfFilters + 1; - char* tmp = (char*)realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); + + char* tmp = (char*) realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; + } else { + return NULL; } pColumn->numOfFilters++; @@ -2964,9 +2972,16 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } else { // update the existed column filter information, find the filter info here pColFilter = &pColumn->filterInfo[0]; } + + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else if (sqlOptr == TK_OR) { // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" pColFilter = addColumnFilterInfo(pColumn); + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else { // error; return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7903ff9562..6cdb39040c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1956,8 +1956,12 @@ static void createHBObj(STscObj* pObj) { pSql->fp = tscProcessHeartBeatRsp; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pSql->cmd, 0, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return; + } + pQueryInfo->command = TSDB_SQL_HB; pSql->cmd.command = pQueryInfo->command; @@ -2142,8 +2146,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf tscAddSubqueryInfo(&pNew->cmd); - SQueryInfo *pNewQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo); + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { @@ -2246,8 +2249,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - SQueryInfo *pNewQueryInfo = NULL; - if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); + if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); return code; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 8d33368280..e5db752458 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -827,8 +827,11 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; char *str = (char *)tblNameList; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return terrno; + } STableMetaInfo *pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 011940a0e6..2fb264c756 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -437,6 +437,7 @@ int32_t tscCompareTidTags(const void* p1, const void* p2) { if (t1->vgId != t2->vgId) { return (t1->vgId > t2->vgId) ? 1 : -1; } + if (t1->tid != t2->tid) { return (t1->tid > t2->tid) ? 1 : -1; } @@ -543,6 +544,7 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, for(int32_t i = 1; i < p1->num; ++i) { STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); + assert(prev->vgId >= 1 && p->vgId >= 1); if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { tscError("%p join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj); @@ -579,6 +581,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar while(i < p1->num && j < p2->num) { STidTags* pp1 = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); STidTags* pp2 = (STidTags*) varDataVal(p2->pIdTagList + j * p2->tagSize); + assert(pp1->tid != 0 && pp2->tid != 0); int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes); if (ret == 0) { @@ -1221,6 +1224,16 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + // get the tag colId column index + int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + for(int32_t i = 0; i < numOfTags; ++i) { + if (pSchema[i].colId == tagColId) { + index.columnIndex = i; + break; + } + } + int16_t bytes = 0; int16_t type = 0; int32_t inter = 0; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6dd8973f65..5bb6d3743d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -486,15 +486,6 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { return TSDB_CODE_SUCCESS; } -//void tscFreeUnusedDataBlocks(SDataBlockList* pList) { -// /* release additional memory consumption */ -// for (int32_t i = 0; i < pList->nSize; ++i) { -// STableDataBlocks* pDataBlock = pList->pData[i]; -// pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size); -// pDataBlock->nAllocSize = (uint32_t)pDataBlock->size; -// } -//} - /** * create the in-memory buffer for each table to keep the submitted data block * @param initialSize @@ -519,6 +510,11 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff } dataBuf->pData = calloc(1, dataBuf->nAllocSize); + if (dataBuf->pData == NULL) { + tscError("failed to allocated memory, reason:%s", strerror(errno)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; @@ -931,8 +927,12 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); + if (pExpr == NULL) { + return NULL; + } + pExpr->functionId = functionId; - + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; @@ -1063,8 +1063,11 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) if (deepcopy) { SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + if (p1 == NULL) { + assert(0); + } + *p1 = *pExpr; - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { tVariantAssign(&p1->param[j], &pExpr->param[j]); } @@ -1100,16 +1103,22 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { if (i >= numOfCols || numOfCols == 0) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } else { SColumn* pCol = taosArrayGetP(pColumnList, i); if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } } @@ -1131,7 +1140,10 @@ SColumn* tscColumnClone(const SColumn* src) { assert(src != NULL); SColumn* dst = calloc(1, sizeof(SColumn)); - + if (dst == NULL) { + return NULL; + } + dst->colIndex = src->colIndex; dst->numOfFilters = src->numOfFilters; dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); @@ -1341,6 +1353,10 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { if (pCond->len > 0) { assert(pCond->cond != NULL); c.cond = malloc(c.len); + if (c.cond == NULL) { + assert(0); + } + memcpy(c.cond, pCond->cond, c.len); } @@ -1466,20 +1482,20 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { return pQueryInfo->pTableMetaInfo[tableIndex]; } -int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo) { +SQueryInfo* tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); int32_t ret = TSDB_CODE_SUCCESS; - *pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); - - while ((*pQueryInfo) == NULL) { + while ((pQueryInfo) == NULL) { if ((ret = tscAddSubqueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - return ret; + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; } - (*pQueryInfo) = tscGetQueryInfoDetail(pCmd, subClauseIndex); + pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); } - return TSDB_CODE_SUCCESS; + return pQueryInfo; } STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { @@ -1591,14 +1607,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST SVgroupsInfo* vgroupList, SArray* pTagCols) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } pQueryInfo->pTableMetaInfo = pAlloc; - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = calloc(1, sizeof(STableMetaInfo)); + STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables]; - assert(pTableMetaInfo != NULL); + pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; if (name != NULL) { tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name)); @@ -1609,10 +1629,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST if (vgroupList != NULL) { size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; pTableMetaInfo->vgroupList = malloc(size); + if (pTableMetaInfo->vgroupList == NULL) { + return NULL; + } + memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + if (pTableMetaInfo->tagColList == NULL) { + return NULL; + } + if (pTagCols != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } @@ -1678,8 +1706,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return NULL; } - SQueryInfo* pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); @@ -1777,10 +1804,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - - free(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SSqlCmd* pnCmd = &pNew->cmd; @@ -1797,9 +1822,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->parseFinished = 1; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pnCmd, 0); @@ -1824,20 +1848,28 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } } tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->fillType != TSDB_FILL_NONE) { pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + if (pNewQueryInfo->fillVal == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); @@ -1880,16 +1912,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pFinalInfo->pTableMeta == NULL) { tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); - tscFreeSqlObj(pNew); - if (pPrevSql != NULL) { + if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); terrno = pPrevSql->res.code; } else { terrno = TSDB_CODE_TSC_APP_ERROR; } - return NULL; + goto _error; } assert(pNewQueryInfo->numOfTables == 1); @@ -1914,6 +1945,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } return pNew; + +_error: + tscFreeSqlObj(pNew); + return NULL; } /** diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 733459c6fe..c09e244643 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -179,10 +179,12 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - char* endPos = pRight->token.z + pRight->token.n; - pExpr->token.z = pLeft->token.z; - pExpr->token.n = endPos - pExpr->token.z; - pExpr->token.type = pLeft->token.type; + if (pRight != NULL && pLeft != NULL) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = endPos - pExpr->token.z; + pExpr->token.type = pLeft->token.type; + } if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { /* diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index c4e490190d..1bb0ff5448 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -330,8 +330,8 @@ sql_error select join_tb1.* from $tb1 , $tb2 where join_tb1.ts != join_tb0.ts an sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts and join_tb1.ts >= 100000; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by ts; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by join_tb1.c7; -sql_error select * from $tb1, $tb2; -sql_error select last_row(*) from $tb1, $tb2 +sql_error select * from join_tb0, join_tb1 +sql_error select last_row(*) from join_tb0, join_tb1 sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts < now sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts = join_tb2.ts From fdbd8fe09970d7261247a478686361e0b5502fff Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 18:40:23 +0800 Subject: [PATCH 48/55] [td-255] update the sim --- tests/script/general/parser/nchar.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 8d4e468f0e..bdac5ace55 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -223,6 +223,7 @@ if $rows != 2 then endi print data00 = $data00 if $data00 != 5 then + print expect 5, actual: $data00 return -1 endi if $data10 != 5 then From df013fc336336230391013fdc635a9dbdeca809d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 27 Aug 2020 11:13:20 +0000 Subject: [PATCH 49/55] TD-857 --- src/util/src/tcompare.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 9564588254..ba711ced8f 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -220,8 +220,14 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { char pattern[128] = {0}; memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); assert(varDataLen(pRight) < 128); - - int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } From 3c45dcdee9481ff813bf6d5beb52606881634fac Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 22:26:37 +0800 Subject: [PATCH 50/55] [td-255] replace wcsncpy with memcpy --- src/common/src/tvariant.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 4036e578b4..069d8eb251 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -100,10 +100,9 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; + pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); - - wcsncpy(pVar->wpz, (wchar_t *)pz, lenInwchar); - pVar->wpz[lenInwchar] = 0; + memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; From 5242848a8280883c0f33b8441956733cb470b150 Mon Sep 17 00:00:00 2001 From: zwllxs Date: Fri, 28 Aug 2020 10:06:01 +0800 Subject: [PATCH 51/55] Update administrator-ch.md --- documentation20/webdocs/markdowndocs/administrator-ch.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index c5ecb862e3..64cadf69cd 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -153,10 +153,10 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下： ``` -CREATE USER PASS <‘password’>; +CREATE USER PASS <'password'>; ``` -创建用户，并指定用户名和密码，密码需要用单引号引起来 +创建用户，并指定用户名和密码，密码需要用单引号引起来,单引号为英文半角 ``` DROP USER ; @@ -165,10 +165,10 @@ DROP USER ; 删除用户，限root用户使用 ``` -ALTER USER PASS <‘password’>; +ALTER USER PASS <'password'>; ``` -修改用户密码, 为避免被转换为小写，密码需要用单引号引用 +修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 ``` SHOW USERS; From 9f158b9d482de7a49a2adf4fa8c1ed5686b71cc0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 10:28:12 +0800 Subject: [PATCH 52/55] minor changes --- tests/script/jenkins/basic.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a48584b0ed..c9e081a70b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -147,7 +147,6 @@ cd ../../../debug; make ./test.sh -f general/parser/join.sim ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/bug.sim ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim From ae8fbb6b212d3d760b90ac638eab92a2f6d7456f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 11:57:40 +0800 Subject: [PATCH 53/55] TD-1090 compile error in windows --- src/client/src/tscFunctionImpl.c | 6 +++--- src/client/src/tscLocalMerge.c | 2 +- src/client/src/tscSQLParser.c | 2 +- src/common/src/tname.c | 2 +- src/query/src/qExecutor.c | 18 +++++++++--------- src/query/src/qParserImpl.c | 2 +- src/query/src/qResultbuf.c | 2 +- src/query/src/qTsbuf.c | 8 ++++---- tests/script/wtest.bat | 6 +++--- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index bd4aa70ee4..e74fcba246 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2034,7 +2034,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { tValuePair **tvp = pRes->res; int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = GET_RES_INFO(pCtx)->numOfRes; + int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); switch (type) { case TSDB_DATA_TYPE_INT: { @@ -2408,10 +2408,10 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } GET_TRUE_DATA_TYPE(); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index cabf2a6a11..563c9fa84e 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -966,7 +966,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); } pRes->numOfRowsGroup += pRes->numOfRows; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5600ee3be9..5ce4c7125f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1303,7 +1303,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); // add the timestamp column into the output columns - int32_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 960cc7d725..01945dbb00 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -53,7 +53,7 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const s.type = pVal->nType; if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { - s.bytes = pVal->nLen + VARSTR_HEADER_SIZE; + s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); } else { s.bytes = tDataTypeDesc[pVal->nType].nSize; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 19324bc2cb..4e2e31d269 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2771,14 +2771,14 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); - int32_t numOfRes = pData->num - pGroupResInfo->pos.rowId; + int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId); if (numOfRes > pQuery->rec.capacity - offset) { - numOfCopiedRows = pQuery->rec.capacity - offset; + numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); pGroupResInfo->pos.rowId += numOfCopiedRows; done = true; } else { - numOfCopiedRows = pData->num; + numOfCopiedRows = (int32_t)pData->num; pGroupResInfo->pos.pageId += 1; pGroupResInfo->pos.rowId = 0; @@ -2871,7 +2871,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - pGroupResInfo->numOfDataPages = taosArrayGetSize(pageList); + pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); pGroupResInfo->groupId = tid; pGroupResInfo->pos.pageId = 0; pGroupResInfo->pos.rowId = 0; @@ -3036,7 +3036,7 @@ int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupR char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; - memcpy(output, src, buf->num * bytes); + memcpy(output, src, (size_t)(buf->num * bytes)); } offset += rows; @@ -3211,7 +3211,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity)); } initCtxOutputBuf(pRuntimeEnv); @@ -3926,7 +3926,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } } @@ -4789,7 +4789,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } @@ -6356,7 +6356,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, s); + size_t sz = read(fd, data, (uint32_t)s); if(sz < s) { // todo handle error assert(0); } diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index c09e244643..9629f24cc2 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -182,7 +182,7 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { if (pRight != NULL && pLeft != NULL) { char* endPos = pRight->token.z + pRight->token.n; pExpr->token.z = pLeft->token.z; - pExpr->token.n = endPos - pExpr->token.z; + pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); pExpr->token.type = pLeft->token.type; } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 8d7730a75a..b3e97459d3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -142,7 +142,7 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { } - ret = fwrite(t, size, 1, pResultBuf->file); + ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); if (ret != size) { // todo handle the error case } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 518bb4083b..b264f6cdc9 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -242,13 +242,13 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * both side has the compressed length is used to support load data forwards/backwords. */ int32_t metaLen = 0; - metaLen += fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); - metaLen += fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); } fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 1574b5013e..6cdd63b42d 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,8 +6,8 @@ echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\build\bin" -set "TSIM=%~dp0..\..\debug\build\bin\tsim" +set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" +set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" echo BUILD_DIR: %BUILD_DIR% set "SIM_DIR=%~dp0..\..\sim" @@ -47,7 +47,7 @@ echo qdebugFlag 143 >> %TAOS_CFG% echo udebugFlag 143 >> %TAOS_CFG% set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=localhost" +set "FIRSTEP=192.168.1.182" if "%1" == "-f" set "FILE_NAME=%2" if "%1" == "-h" set "FIRSTEP=%2" if "%3" == "-f" set "FILE_NAME=%4" From 89744f4ba80c6d13433b6657adf2751c7001fef5 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 28 Aug 2020 13:00:53 +0800 Subject: [PATCH 54/55] [release version modify] --- cmake/version.inc | 79 ++++++++---- packaging/release.sh | 261 ++++++++++++++------------------------ src/util/src/version.c.in | 4 +- 3 files changed, 150 insertions(+), 194 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 8c02520142..ad399bfa86 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -1,42 +1,65 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -SET(TD_VER_1 "2") -SET(TD_VER_2 "0") -SET(TD_VER_3 "2") -SET(TD_VER_4 "3") -SET(TD_VER_GIT "d711657139620f6c50f362597020705b8ad26bd2") -SET(TD_VER_GIT_INTERNAL "1d74ae24c541ffbb280e8630883c0236cd45f8c7") +IF (DEFINED VERNUMBER) + SET(TD_VER_NUMBER ${VERNUMBER}) +ELSE () + SET(TD_VER_NUMBER "2.0.2.0") +ENDIF () -SET(TD_VER_VERTYPE "stable") -SET(TD_VER_CPUTYPE "x64") -SET(TD_VER_OSTYPE "Linux") +IF (DEFINED VERCOMPATIBLE) + SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) +ELSE () + SET(TD_VER_COMPATIBLE "2.0.0.0") +ENDIF () -SET(TD_VER_COMPATIBLE "2.0.0.0") -STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +IF (DEFINED GITINFO) + SET(TD_VER_GIT ${GITINFO}) +ELSE () + SET(TD_VER_GIT "community") +ENDIF () -IF (TD_LINUX_64) +IF (DEFINED GITINFOI) + SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSE () + SET(TD_VER_GIT_INTERNAL "internal") +ENDIF () + +IF (DEFINED VERDATE) + SET(TD_VER_DATE ${VERDATE}) +ELSE () + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +ENDIF () + +IF (DEFINED VERTYPE) + SET(TD_VER_VERTYPE ${VERTYPE}) +ELSE () + SET(TD_VER_VERTYPE "stable") +ENDIF () + +IF (DEFINED CPUTYPE) + SET(TD_VER_CPUTYPE ${CPUTYPE}) +ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () -IF (TD_LINUX_32) - SET(TD_VER_CPUTYPE "x86") +IF (DEFINED OSTYPE) + SET(TD_VER_OSTYPE ${OSTYPE}) +ELSE () + SET(TD_VER_OSTYPE "Linux") ENDIF () -IF (TD_ARM_64) - SET(TD_VER_CPUTYPE "aarch64") -ENDIF () +MESSAGE(STATUS "============= compile version parameter information start ============= ") +MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) +MESSAGE(STATUS "internal commit id:" ${TD_VER_GIT_INTERNAL}) +MESSAGE(STATUS "build date:" ${TD_VER_DATE}) +MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) +MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "============= compile version parameter information end ============= ") -IF (TD_ARM_32) - SET(TD_VER_CPUTYPE "aarch32") -ENDIF () - -IF (TD_WINDOWS_64) - SET(TD_VER_CPUTYPE "x64") -ENDIF () - -IF (TD_WINDOWS_32) - SET(TD_VER_CPUTYPE "x86") -ENDIF () +STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/util/src/version.c.in" "${TD_COMMUNITY_DIR}/src/util/src/version.c") diff --git a/packaging/release.sh b/packaging/release.sh index bf355bdcd4..3c0378042d 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -12,6 +12,7 @@ set -e # -l [full | lite] # -s [static | dynamic] # -n [2.0.0.3] +# -m [2.0.0.0] # set parameters by default value verMode=edge # [cluster, edge] @@ -21,8 +22,9 @@ osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ni pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] verNumber="" +verNumberComp="2.0.0.0" -while getopts "hv:V:c:o:l:s:n:" arg +while getopts "hv:V:c:o:l:s:n:m:" arg do case $arg in v) @@ -49,6 +51,10 @@ do #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) ;; + m) + #echo "verNumberComp=$OPTARG" + verNumberComp=$(echo $OPTARG) + ;; o) #echo "osType=$OPTARG" osType=$(echo $OPTARG) @@ -61,6 +67,7 @@ do echo " -l [full | lite] " echo " -s [static | dynamic] " echo " -n [version number] " + echo " -m [compatible version number] " exit 0 ;; ?) #unknow option @@ -70,216 +77,142 @@ do esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber} verNumberComp=${verNumberComp}" curr_dir=$(pwd) if [ "$osType" != "Darwin" ]; then - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/..)" else - script_dir=`dirname $0` - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/.. fi -versioninfo="${top_dir}/src/util/src/version.c" - csudo="" #if command -v sudo > /dev/null; then -# csudo="sudo" +# csudo="sudo" #fi function is_valid_version() { - [ -z $1 ] && return 1 || : + [ -z $1 ] && return 1 || : - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 + rx='^([0-9]+\.){3}(\*|[0-9]+)$' + if [[ $1 =~ $rx ]]; then + return 0 + fi + return 1 } function vercomp () { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i=0; i<${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done + if [[ $1 == $2 ]]; then echo 0 -} + exit 0 + fi + + local IFS=. + local i ver1=($1) ver2=($2) -# 1. Read version information -version=$(cat ${versioninfo} | grep " version" | cut -d '"' -f2) -compatible_version=$(cat ${versioninfo} | grep " compatible_version" | cut -d '"' -f2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done -if [ -z ${verNumber} ]; then - while true; do - read -p "Do you want to release a new version? [y/N]: " is_version_change - - if [[ ( "${is_version_change}" == "y") || ( "${is_version_change}" == "Y") ]]; then - read -p "Please enter the new version: " tversion - while true; do - if (! is_valid_version $tversion) || [ "$(vercomp $tversion $version)" = '2' ]; then - read -p "Please enter a correct version: " tversion - continue - fi - version=${tversion} - break - done - - echo - - read -p "Enter the oldest compatible version: " tversion - while true; do - - if [ -z $tversion ]; then - break - fi - - if (! is_valid_version $tversion) || [ "$(vercomp $version $tversion)" = '2' ]; then - read -p "enter correct compatible version: " tversion - else - compatible_version=$tversion - break - fi - done - - break - elif [[ ( "${is_version_change}" == "n") || ( "${is_version_change}" == "N") ]]; then - echo "Use old version: ${version} compatible version: ${compatible_version}." - break - else - continue + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + echo 1 + exit 0 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + echo 2 + exit 0 fi done -else - echo "old version: $version, new version: $verNumber" - #if ( ! is_valid_version $verNumber ) || [[ "$(vercomp $version $verNumber)" == '2' ]]; then - # echo "please enter correct version" - # exit 0 - #else - version=${verNumber} - #fi -fi + echo 0 +} -echo "=======================new version number: ${version}======================================" +# 1. check version information +if (( ! is_valid_version $verNumber ) || ( ! is_valid_version $verNumberComp ) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then + echo "please enter correct version" + exit 0 +fi + +echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" -# output the version info to the buildinfo file. build_time=$(date +"%F %R") -echo "char version[12] = \"${version}\";" > ${versioninfo} -echo "char compatible_version[12] = \"${compatible_version}\";" >> ${versioninfo} -echo "char gitinfo[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} -if [ "$verMode" != "cluster" ]; then - echo "char gitinfoOfInternal[48] = \"\";" >> ${versioninfo} -else - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - echo "char gitinfoOfInternal[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} - cd ${curr_dir} -fi -echo "char buildinfo[64] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} -echo "" >> ${versioninfo} -tmp_version=$(echo $version | tr -s "." "_") -if [ "$verMode" == "cluster" ]; then - libtaos_info=${tmp_version}_${osType}_${cpuType} -else - libtaos_info=edge_${tmp_version}_${osType}_${cpuType} -fi -if [ "$verType" == "beta" ]; then - libtaos_info=${libtaos_info}_${verType} -fi -echo "void libtaos_${libtaos_info}() {};" >> ${versioninfo} + +# get commint id from git +gitinfo=$(git rev-parse --verify HEAD) +enterprise_dir="${top_dir}/../enterprise" +cd ${enterprise_dir} +gitinfoOfInternal=$(git rev-parse --verify HEAD) +cd ${curr_dir} # 2. cmake executable file compile_dir="${top_dir}/debug" if [ -d ${compile_dir} ]; then - ${csudo} rm -rf ${compile_dir} + ${csudo} rm -rf ${compile_dir} fi if [ "$osType" != "Darwin" ]; then - ${csudo} mkdir -p ${compile_dir} + ${csudo} mkdir -p ${compile_dir} else - mkdir -p ${compile_dir} + mkdir -p ${compile_dir} fi cd ${compile_dir} # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then - if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DOSTYPE=${osType} -DSOMODE=${soMode} - else - cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} - fi + if [ "$verMode" != "cluster" ]; then + cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} + else + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} + fi else - echo "input cpuType=${cpuType} error!!!" - exit 1 + echo "input cpuType=${cpuType} error!!!" + exit 1 fi make cd ${curr_dir} -# 3. judge the operating system type, then Call the corresponding script for packaging -#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -#osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) -#echo "osinfo: ${osinfo}" - +# 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} - - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} + if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then + echo "====do deb package for the ubuntu system====" + output_dir="${top_dir}/debs" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/deb + ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + + echo "====do rpm package for the centos system====" + output_dir="${top_dir}/rpms" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/rpm + ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + fi - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo} ./makepkg.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makearbi.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + echo "====do tar.gz package for all systems====" + cd ${script_dir}/tools + + ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else - cd ${script_dir}/tools - ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} + cd ${script_dir}/tools + ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} fi -# 4. Clean up temporary compile directories -#${csudo} rm -rf ${compile_dir} - diff --git a/src/util/src/version.c.in b/src/util/src/version.c.in index c7aea2afb1..21c78a0eb4 100644 --- a/src/util/src/version.c.in +++ b/src/util/src/version.c.in @@ -1,7 +1,7 @@ -char version[12] = "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}"; +char version[12] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; char buildinfo[64] = "Built at ${TD_VER_DATE}"; -void libtaos_${TD_VER_1}_${TD_VER_2}_${TD_VER_3}_${TD_VER_4}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; +void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; From 28cef33a72147ec61a620ef1febe698ec87170fb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 13:08:34 +0800 Subject: [PATCH 55/55] minor changes --- src/client/src/taos.rc.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/taos.rc.in b/src/client/src/taos.rc.in index 05dbd9bb7d..751be85fd0 100644 --- a/src/client/src/taos.rc.in +++ b/src/client/src/taos.rc.in @@ -1,6 +1,6 @@ 1 VERSIONINFO - FILEVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} - PRODUCTVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} + FILEVERSION ${TD_VER_NUMBER} + PRODUCTVERSION ${TD_VER_NUMBER} FILEFLAGSMASK 0x17L #ifdef _DEBUG FILEFLAGS 0x1L @@ -16,12 +16,12 @@ BEGIN BLOCK "040904b0" BEGIN VALUE "FileDescription", "Native C Driver for TDengine" - VALUE "FileVersion", "${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3}" + VALUE "FileVersion", "${TD_VER_NUMBER}" VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" VALUE "OriginalFilename", "" VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" - VALUE "ProductVersion", "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}" + VALUE "ProductVersion", "${TD_VER_NUMBER}" END END BLOCK "VarFileInfo"