From 8be56fb73c8573bd9113908607909988a2f5d3a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 11:35:04 +0800 Subject: [PATCH 001/111] [td-1148] enable the binary tag to be used as the join condition. --- src/client/src/tscFunctionImpl.c | 6 +- src/client/src/tscSQLParser.c | 5 -- src/client/src/tscSubquery.c | 9 +-- src/common/inc/tvariant.h | 2 + src/common/src/tvariant.c | 34 +++++++++- src/query/inc/qExecutor.h | 2 +- src/query/inc/qTsbuf.h | 21 ++++--- src/query/src/qExecutor.c | 10 ++- src/query/src/qTsbuf.c | 66 +++++++++++++------- src/query/tests/tsBufTest.cpp | 104 ++++++++++++++++++++++--------- 10 files changed, 176 insertions(+), 83 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 5c708dffee..205be5e3a6 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3900,11 +3900,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE); } } @@ -3923,7 +3923,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f933366711..43a43b6d31 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3064,7 +3064,6 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { const char* msg1 = "invalid join query condition"; - const char* msg2 = "join on binary/nchar not supported"; const char* msg3 = "type of join columns must be identical"; const char* msg4 = "invalid column name in join condition"; @@ -3108,10 +3107,6 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (pTagSchema1->type == TSDB_DATA_TYPE_BINARY || pTagSchema1->type == TSDB_DATA_TYPE_NCHAR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - pTagCond->joinInfo.hasJoin = true; return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 75644c355c..eedb229e1c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -93,13 +93,14 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); #endif - if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem1.ts, elem2.ts))) { + int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); + if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { if (!tsBufNextPos(pSupporter1->pTSBuf)) { break; } numOfInput1++; - } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem2.ts, elem1.ts))) { + } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { if (!tsBufNextPos(pSupporter2->pTSBuf)) { break; } @@ -119,8 +120,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ win->ekey = elem1.ts; } - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); } else { pLimit->offset -= 1; } diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 4fd6ea5541..211635b4af 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -46,6 +46,8 @@ void tVariantDestroy(tVariant *pV); void tVariantAssign(tVariant *pDst, const tVariant *pSrc); +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2); + int32_t tVariantToString(tVariant *pVar, char *dst); int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 5460d21252..33f8eb127f 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -12,12 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "os.h" #include "tvariant.h" #include "hash.h" -#include "hashfunc.h" -#include "os.h" -#include "hash.h" #include "taos.h" #include "taosdef.h" #include "tstoken.h" @@ -172,6 +170,36 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } } +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { + assert((p1->nType != TSDB_DATA_TYPE_NULL) || (p2->nType != TSDB_DATA_TYPE_NULL)); + + switch (p1->nType) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + if (p1->nLen == p2->nLen) { + return memcmp(p1->pz, p2->pz, p1->nLen); + } else { + return p1->nLen > p2->nLen? 1:-1; + } + }; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (p1->dKey == p2->dKey) { + return 0; + } else { + return p1->dKey > p2->dKey? 1:-1; + } + + default: + if (p1->i64Key == p2->i64Key) { + return 0; + } else { + return p1->i64Key > p2->i64Key? 1:-1; + } + } +} + int32_t tVariantToString(tVariant *pVar, char *dst) { if (pVar == NULL || dst == NULL) return 0; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b5487561b2..d824a269c9 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -98,7 +98,7 @@ typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query - int64_t tag; + tVariant tag; STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 59b224e096..f6d672b26d 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -22,6 +22,7 @@ extern "C" { #include "os.h" #include "taosdef.h" +#include "tvariant.h" #define MEM_BUF_SIZE (1 << 20) #define TS_COMP_FILE_MAGIC 0x87F5EC4C @@ -42,9 +43,9 @@ typedef struct STSRawBlock { } STSRawBlock; typedef struct STSElem { - TSKEY ts; - int64_t tag; - int32_t vnode; + TSKEY ts; + tVariant tag; + int32_t vnode; } STSElem; typedef struct STSCursor { @@ -55,11 +56,11 @@ typedef struct STSCursor { } STSCursor; typedef struct STSBlock { - int64_t tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed + tVariant tag; // tag value + int32_t numOfElem; // number of elements + int32_t compLen; // size after compressed + int32_t padding; // 0xFFFFFFFF by default, after the payload + char* payload; // actual data that is compressed } STSBlock; /* @@ -109,7 +110,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ void* tsBufDestroy(STSBuf* pTSBuf); -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); STSBuf* tsBufClone(STSBuf* pTSBuf); @@ -122,7 +123,7 @@ void tsBufResetPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); bool tsBufNextPos(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag); +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c8efee03cd..52408f78bf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1103,7 +1103,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first - if (pCtx[0].tag.i64Key != elem.tag) { + if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) { return TS_JOIN_TAG_NOT_EQUALS; } @@ -3644,9 +3644,8 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { if (pTableQueryInfo->cur.vgroupIndex == -1) { - pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; - - tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); + tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag); + tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag); // keep the cursor info of current meter pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; @@ -4501,8 +4500,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vgroupIndex == -1) { - int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag); // failed to find data with the specified tag value if (elem.vnode < 0) { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 25eb33ff7d..eb47a1b7a1 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -8,7 +8,6 @@ static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInf static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - /** * todo error handling * support auto closeable tmp file @@ -225,11 +224,12 @@ static void writeDataToDisk(STSBuf* pTSBuf) { } STSBlock* pBlock = &pTSBuf->block; - - pBlock->numOfElem = pTSBuf->tsData.len / TSDB_KEYSIZE; + STSList* pTsData = &pTSBuf->tsData; + + pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; pBlock->compLen = - tsCompressTimestamp(pTSBuf->tsData.rawBuf, pTSBuf->tsData.len, pTSBuf->tsData.len / TSDB_KEYSIZE, pBlock->payload, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, + TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); UNUSED(r); @@ -241,13 +241,18 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * * both side has the compressed length is used to support load data forwards/backwords. */ - fwrite(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); + fwrite(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + fwrite(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + fwrite(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + fwrite(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + } + fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; @@ -298,9 +303,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } - - size_t sz = fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); - UNUSED(sz); + + fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + + // NOTE: mix types tags are not supported + size_t sz = 0; + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + assert(tp != NULL); + + memset(tp, 0, pBlock->tag.nLen + 1); + pBlock->tag.pz = tp; + + sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + } + sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); UNUSED(sz); sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); @@ -361,7 +381,7 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { return TSDB_CODE_SUCCESS; } -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len) { +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) { STSVnodeBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; @@ -375,15 +395,15 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData } assert(pBlockInfo->info.vnode == vnodeId); - - if (pTSBuf->block.tag != tag && ptsData->len > 0) { + + if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first writeDataToDisk(pTSBuf); } else { expandBuffer(ptsData, len); } - pTSBuf->block.tag = tag; + tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); // todo check return value @@ -465,7 +485,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int return 0; } -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int64_t tag) { +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) { bool decomp = false; int64_t offset = 0; @@ -484,7 +504,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo return -1; } - if (pTSBuf->block.tag == tag) { + if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { return i; } } @@ -669,8 +689,8 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = pBlock->tag; - + tVariantAssign(&elem1.tag, &pBlock->tag); + return elem1; } @@ -800,7 +820,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ return pTSBuf; } -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { STSElem elem = {.vnode = -1}; if (pTSBuf == NULL) { @@ -881,7 +901,9 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag, elem.ts); + if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) { + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + } } pTSBuf->cur.order = old; diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f8738eec9c..b78c5314f2 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -32,14 +32,16 @@ void simpleTest() { // write 10 ts points int32_t num = 10; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(10, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -57,14 +59,16 @@ void largeTSTest() { // write 10 ts points int32_t num = 1000000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(num, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); // the data has been flush to disk, no data in cache EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -80,14 +84,18 @@ void multiTagsTest() { STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); int32_t num = 10000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int64_t start = 10000000; int32_t numOfTags = 50; int32_t step = 30; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -96,7 +104,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -118,9 +126,14 @@ void multiVnodeTagsTest() { for (int32_t j = 0; j < 20; ++j) { // vnodeId:0 start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -131,11 +144,11 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -157,9 +170,14 @@ void loadDataTest() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %" PRIu64 "\n", i, list[0]); free(list); @@ -172,11 +190,11 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -230,16 +248,21 @@ void TSTraverse() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); start += step * num; list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); @@ -272,12 +295,16 @@ void TSTraverse() { int32_t startVnode = 1; int32_t startTag = 2; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = startTag; + + tsBufGetElemStartPos(pTSBuf, startVnode, &t); int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -286,7 +313,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag == 0) { startVnode -= 1; @@ -316,13 +345,14 @@ void TSTraverse() { startVnode = 1; startTag = 2; + t.i64Key = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tsBufGetElemStartPos(pTSBuf, startVnode, &t); totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -331,7 +361,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag < 0) { startVnode -= 1; @@ -375,12 +407,17 @@ void mergeDiffVnodeBufferTest() { int32_t num = 1000; int32_t numOfTags = 10; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + // vnodeId:0 int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -403,6 +440,9 @@ void mergeIdenticalVnodeBufferTest() { STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int32_t step = 30; int32_t num = 1000; int32_t numOfTags = 10; @@ -411,17 +451,21 @@ void mergeIdenticalVnodeBufferTest() { int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); + t.i64Key = i; - tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -438,7 +482,7 @@ void mergeIdenticalVnodeBufferTest() { STSElem elem = tsBufGetElem(pTSBuf1); EXPECT_EQ(elem.vnode, 12); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); } tsBufDestroy(pTSBuf1); @@ -446,6 +490,8 @@ void mergeIdenticalVnodeBufferTest() { } } // namespace + +//TODO add binary tag value test case TEST(testCase, tsBufTest) { simpleTest(); largeTSTest(); @@ -453,7 +499,7 @@ TEST(testCase, tsBufTest) { multiVnodeTagsTest(); loadDataTest(); invalidFileTest(); - // randomIncTsTest(); +// randomIncTsTest(); TSTraverse(); mergeDiffVnodeBufferTest(); mergeIdenticalVnodeBufferTest(); From 9f940295d607fe709cf42c4eed6afe06df68e086 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 14:24:58 +0800 Subject: [PATCH 002/111] [td-225] fix bugs during stable query with the fill option. --- src/client/src/tscLocalMerge.c | 49 +++++-------------- tests/script/general/parser/lastrow_query.sim | 24 +++++++++ 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 186c2871a1..0a7bbac601 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -370,7 +370,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision); @@ -843,28 +843,6 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); } -static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, - SLocalReducer *pLocalReducer) { - assert(0); - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); - - char *src = pResPages[i]->data + (pRes->numOfRows - 1) * pField->bytes; - char *dst = pRes->data + pRes->numOfRows * offset; - - for (int32_t j = 0; j < pRes->numOfRows; ++j) { - memcpy(dst, src, (size_t)pField->bytes); - dst += pField->bytes; - src -= pField->bytes; - } - } -} - static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { assert(pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); @@ -925,7 +903,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SFillInfo *pFillInfo = pLocalReducer->pFillInfo; - int64_t actualETime = MAX(pQueryInfo->window.skey, pQueryInfo->window.ekey); + // todo extract function + int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -984,14 +963,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo); } - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); - } - } else { // todo bug?? - reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); } pRes->numOfRowsGroup += pRes->numOfRows; @@ -1249,15 +1224,15 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur // tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num); #endif - - // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { genFinalResWithoutFill(pRes, pLocalReducer, pQueryInfo); } else { SFillInfo* pFillInfo = pLocalReducer->pFillInfo; if (pFillInfo != NULL) { - taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, pQueryInfo->window.ekey); + TSKEY ekey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; + + taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, ekey); taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } @@ -1292,7 +1267,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->fillType != TSDB_FILL_NONE) { - TSKEY skey = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t newTime = taosGetIntervalStartTimestamp(skey, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); taosResetFillInfo(pLocalReducer->pFillInfo, newTime); @@ -1345,7 +1320,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { // if fillType == TSDB_FILL_NONE, return directly if (pQueryInfo->fillType != TSDB_FILL_NONE && ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { - int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; + int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey; int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 98eb5a8d6d..7954a8d228 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -128,3 +128,27 @@ if $rows != 86399 then return -1 endi +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 30 +if $rows != 48 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 2 +if $rows != 4 then + return -1 +endi + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 order by ts desc slimit 1 soffset 1 limit 250000 offset 1 +if $rows != 86399 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 1 +if $rows != 2 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 25 offset 1 +if $rows != 46 then + return -1 +endi From 5aff8962a45952cbd54377435cda13647d24714c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 14:52:50 +0800 Subject: [PATCH 003/111] [td-225] fix bugs in retrieve error. --- src/tsdb/src/tsdbRead.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c89ae0698a..bb3f7ca195 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1795,14 +1795,17 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { if (pQueryHandle->checkFiles) { bool exists = true; + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - return code; + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; + + return false; } if (exists) { - elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); return exists; } From 3e2ea659cba021a5186f38f695e9f97147b2aa3b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 18:17:46 +0800 Subject: [PATCH 004/111] [td-225] fix bug in "show configs"; --- src/mnode/src/mnodeDnode.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 261f63d5cf..f6a687d864 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -857,13 +857,13 @@ static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = TSDB_CFG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config name"); + tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CFG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config value"); + tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -898,27 +898,32 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v int32_t cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->option, TSDB_CFG_OPTION_LEN); + cols++; + int32_t t = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { case TAOS_CFG_VTYPE_INT16: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_INT32: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_FLOAT: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_STRING: case TAOS_CFG_VTYPE_IPSTR: case TAOS_CFG_VTYPE_DIRECTORY: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->ptr, TSDB_CFG_VALUE_LEN); numOfRows++; break; default: From ff20d564abf28b4307a43edd6a52e60ebf780852 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 15 Aug 2020 23:05:07 +0800 Subject: [PATCH 005/111] [td-1109] --- src/inc/taosmsg.h | 2 +- src/inc/ttokendef.h | 2 +- src/mnode/src/mnodeDnode.c | 34 +++++++++++++--------------------- src/mnode/src/mnodeShow.c | 2 +- src/query/inc/sql.y | 2 +- src/query/src/qTokenizer.c | 2 +- src/query/src/sql.c | 17 +++++++---------- 7 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 265bf47d6d..9ae3fd71e2 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -133,7 +133,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, - TSDB_MGMT_TABLE_CONFIGS, + TSDB_MGMT_TABLE_VARIABLES, TSDB_MGMT_TABLE_CONNS, TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_GRANTS, diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index e77f5741d3..c1579fe9e7 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -69,7 +69,7 @@ #define TK_QUERIES 51 #define TK_CONNECTIONS 52 #define TK_STREAMS 53 -#define TK_CONFIGS 54 +#define TK_VARIABLES 54 #define TK_SCORES 55 #define TK_GRANTS 56 #define TK_VNODES 57 diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index f6a687d864..772894376c 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -49,7 +49,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg); static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; -static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *rpcMsg); +static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg); static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -161,8 +161,8 @@ int32_t mnodeInitDnodes() { mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mnodeProcessDnodeStatusMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mnodeGetModuleMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mnodeRetrieveModules); - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeGetConfigMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeRetrieveConfigs); + mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeGetConfigMeta); + mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeRetrieveConfigs); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mnodeGetVnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); @@ -728,7 +728,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo } static bool mnodeCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { - uint32_t status = pDnode->moduleStatus & (1 << moduleType); + uint32_t status = pDnode->moduleStatus & (1u << moduleType); return status > 0; } @@ -753,7 +753,7 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); + strcpy(pSchema[cols].name, "end_point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -787,7 +787,9 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; - char * pWrite; + + char* pWrite; + char* moduleName[5] = { "MNODE", "HTTP", "MONITOR", "MQTT", "UNKNOWN" }; while (numOfRows < rows) { SDnodeObj *pDnode = NULL; @@ -802,28 +804,18 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - 1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - switch (moduleType) { - case TSDB_MOD_MNODE: - strcpy(pWrite, "mnode"); - break; - case TSDB_MOD_HTTP: - strcpy(pWrite, "http"); - break; - case TSDB_MOD_MONITOR: - strcpy(pWrite, "monitor"); - break; - default: - strcpy(pWrite, "unknown"); - } + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, moduleName[moduleType], pShow->bytes[cols]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; bool enable = mnodeCheckModuleInDnode(pDnode, moduleType); - strcpy(pWrite, enable ? "enable" : "disable"); + + char* v = enable? "enable":"disable"; + STR_TO_VARSTR(pWrite, v); cols++; numOfRows++; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 8a84b66a33..e553853658 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -98,7 +98,7 @@ static char *mnodeGetShowType(int32_t showType) { case TSDB_MGMT_TABLE_MODULE: return "show modules"; case TSDB_MGMT_TABLE_QUERIES: return "show queries"; case TSDB_MGMT_TABLE_STREAMS: return "show streams"; - case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; + case TSDB_MGMT_TABLE_VARIABLES: return "show configs"; case TSDB_MGMT_TABLE_CONNS: return "show connections"; case TSDB_MGMT_TABLE_SCORES: return "show scores"; case TSDB_MGMT_TABLE_GRANTS: return "show grants"; diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 2b2a967edf..edf59327ab 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -73,7 +73,7 @@ cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW CONFIGS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } +cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 227aded5f1..6fb63c4a20 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -85,7 +85,7 @@ static SKeyword keywordTable[] = { {"QUERIES", TK_QUERIES}, {"CONNECTIONS", TK_CONNECTIONS}, {"STREAMS", TK_STREAMS}, - {"CONFIGS", TK_CONFIGS}, + {"VARIABLES", TK_VARIABLES}, {"SCORES", TK_SCORES}, {"GRANTS", TK_GRANTS}, {"DOT", TK_DOT}, diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 75ef2f3218..0b977977af 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -488,7 +488,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* QUERIES => nothing */ 0, /* CONNECTIONS => nothing */ 0, /* STREAMS => nothing */ - 0, /* CONFIGS => nothing */ + 0, /* VARIABLES => nothing */ 0, /* SCORES => nothing */ 0, /* GRANTS => nothing */ 0, /* VNODES => nothing */ @@ -781,7 +781,7 @@ static const char *const yyTokenName[] = { /* 51 */ "QUERIES", /* 52 */ "CONNECTIONS", /* 53 */ "STREAMS", - /* 54 */ "CONFIGS", + /* 54 */ "VARIABLES", /* 55 */ "SCORES", /* 56 */ "GRANTS", /* 57 */ "VNODES", @@ -1017,7 +1017,7 @@ static const char *const yyRuleName[] = { /* 7 */ "cmd ::= SHOW QUERIES", /* 8 */ "cmd ::= SHOW CONNECTIONS", /* 9 */ "cmd ::= SHOW STREAMS", - /* 10 */ "cmd ::= SHOW CONFIGS", + /* 10 */ "cmd ::= SHOW VARIABLES", /* 11 */ "cmd ::= SHOW SCORES", /* 12 */ "cmd ::= SHOW GRANTS", /* 13 */ "cmd ::= SHOW VNODES", @@ -1702,7 +1702,7 @@ static const struct { { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW CONFIGS */ + { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ @@ -2029,8 +2029,8 @@ static void yy_reduce( case 9: /* cmd ::= SHOW STREAMS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } break; - case 10: /* cmd ::= SHOW CONFIGS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } + case 10: /* cmd ::= SHOW VARIABLES */ +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } break; case 11: /* cmd ::= SHOW SCORES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } @@ -2277,6 +2277,7 @@ static void yy_reduce( yymsp[-1].minor.yy268 = yylhsminor.yy268; break; case 95: /* db_optr ::= db_optr fsync */ + case 107: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==107); { yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy268 = yylhsminor.yy268; break; @@ -2297,10 +2298,6 @@ static void yy_reduce( case 99: /* alter_db_optr ::= */ { setDefaultCreateDbOption(&yymsp[1].minor.yy268);} break; - case 107: /* alter_db_optr ::= alter_db_optr fsync */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; case 108: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; From 39618becb74e3dcc5b2284d6501bf0fc6daa9c79 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 00:01:01 +0800 Subject: [PATCH 006/111] [td-1151] --- src/client/inc/tscUtil.h | 4 +- src/client/src/tscFunctionImpl.c | 30 ++++++++++----- src/client/src/tscSQLParser.c | 66 +++++++++++++++++++++----------- src/client/src/tscServer.c | 16 ++++---- src/client/src/tscUtil.c | 12 +++--- src/common/inc/tname.h | 2 + src/common/src/tname.c | 19 ++++++++- src/common/src/tvariant.c | 4 ++ src/inc/taosdef.h | 3 +- src/inc/taosmsg.h | 6 +-- src/query/src/qExecutor.c | 40 +++++++++++++------ 11 files changed, 140 insertions(+), 62 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2ca6ba6691..7c406d1676 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -138,8 +138,8 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, - SSchema* pColSchema, int16_t colType); +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, + SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 205be5e3a6..f1f235cd36 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2906,18 +2906,30 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size); - - char *pData = GET_INPUT_CHAR(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); - } else { + if (pCtx->numOfParams == 1) { // the number of output rows should not affect the final number of rows, so set it to be 1 + INC_INIT_VAL(pCtx, 1); + + char* output = pCtx->aOutputBuf; for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); + tVariantDump(&pCtx->param[0], output, pCtx->outputType, true); + output += pCtx->outputBytes; + } + + } else { + + INC_INIT_VAL(pCtx, pCtx->size); + + char *pData = GET_INPUT_CHAR(pCtx); + if (pCtx->order == TSDB_ORDER_ASC) { + memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); + } else { + for(int32_t i = 0; i < pCtx->size; ++i) { + memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, + pCtx->inputBytes); + } } } - + pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 43a43b6d31..fc33e22eff 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -33,8 +33,8 @@ #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" -// -1 is tbname column index, so here use the -2 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-2) +// -1 is tbname column index, so here use the -3 as the initial value +#define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } #define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) @@ -1248,7 +1248,9 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields - if (pItem->pNode->nSQLOptr == TK_ALL || pItem->pNode->nSQLOptr == TK_ID || pItem->pNode->nSQLOptr == TK_STRING) { + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || (optr == TK_INTEGER || optr == TK_FLOAT)) { // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); @@ -1256,7 +1258,6 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // if the name of column is quoted, remove it and set the right information for later process extractColumnNameFromString(pItem); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { @@ -1372,10 +1373,10 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); } -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, pColSchema->bytes, flag); + pColSchema->bytes, pColSchema->bytes, TSDB_COL_IS_TAG(flag)); tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName)); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1391,6 +1392,8 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex if (TSDB_COL_IS_TAG(flag)) { tscColumnListInsert(pTableMetaInfo->tagColList, pIndex); } + + return pExpr; } static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { @@ -1434,7 +1437,11 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL) { // project on all fields + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getTableIndexByName(&pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); @@ -1450,28 +1457,43 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } else { doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos); } - } else if (pItem->pNode->nSQLOptr == TK_ID) { // simple column projection query + + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + } else if (optr == TK_ID || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); - } + // user-specified constant value as a new result column + if ((optr == TK_INTEGER || optr == TK_FLOAT) || (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { + index.columnIndex = TSDB_UD_COLUMN_INDEX; + index.tableIndex = 0; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = tGetTableNameColumnSchema(); - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); - } else { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + SSchema colSchema = tGetUserSpecifiedColumnSchema(pItem->pNode->val.pz, pItem->pNode->val.nType, pItem->aliasName); + SSqlExpr* pExpr = tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + pExpr->numOfParams = 1; + tVariantAssign(&pExpr->param[0], &pItem->pNode->val); - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } else { // columns from the queried table + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + SSchema colSchema = tGetTableNameColumnSchema(); + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); + } else { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + addProjectQueryCol(pQueryInfo, startPos, &index, pItem); } - addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2037,7 +2059,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo refactor static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && columnIndex >= -1 && tableIndex >= 0); + assert(num == 1 && columnIndex >= -2 && tableIndex >= 0); SColumnList columnList = {0}; columnList.num = num; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6b75b680b1..03d696a8ee 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -622,7 +622,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); - return -1; // todo add test for this + return TSDB_CODE_TSC_INVALID_SQL; // todo add test for this } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -631,17 +631,17 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->intervalTime < 0) { tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; @@ -719,7 +719,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { tscError("invalid filter info"); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } } } @@ -731,7 +731,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); @@ -1279,7 +1279,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for alter table msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; @@ -1631,7 +1631,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); tscError("%p failed to malloc for heartbeat msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 64a871ff74..716d48fa03 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -926,7 +926,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize, bool isTagCol) { + int16_t size, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); @@ -935,8 +935,10 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else if (pColIndex->columnIndex == TSDB_UD_COLUMN_INDEX) { + pExpr->colInfo.colId = TSDB_UD_COLUMN_INDEX; } else { - if (isTagCol) { + if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name)); @@ -948,9 +950,9 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol } } - pExpr->colInfo.flag = isTagCol? TSDB_COL_TAG:TSDB_COL_NORMAL; - + pExpr->colInfo.flag = colType; pExpr->colInfo.colIndex = pColIndex->columnIndex; + pExpr->resType = type; pExpr->resBytes = size; pExpr->interBytes = interSize; @@ -1291,7 +1293,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_UD_COLUMN_INDEX) { return true; } diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 2263a5dae1..41d6fe7280 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -28,6 +28,8 @@ void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); SSchema tGetTableNameColumnSchema(); +SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name); + bool tscValidateTableNameLength(size_t len); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index d80ddb0ee2..6aa308cb45 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -43,7 +43,24 @@ SSchema tGetTableNameColumnSchema() { s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; s.type = TSDB_DATA_TYPE_BINARY; s.colId = TSDB_TBNAME_COLUMN_INDEX; - strncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + return s; +} + +SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name) { + SSchema s = {0}; + + s.type = type; + if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { + size_t len = strlen(v); + s.bytes = len + VARSTR_HEADER_SIZE; + } else { + s.bytes = tDataTypeDesc[type].nSize; + } + + s.colId = TSDB_UD_COLUMN_INDEX; + tstrncpy(s.name, name, sizeof(s.name)); + return s; } diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 33f8eb127f..6716a1827e 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -167,7 +167,11 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } + + return; } + + pDst->nLen = tDataTypeDesc[pDst->nType].nSize; } int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 3dea8da18a..12c4699665 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -286,7 +286,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-2) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 9ae3fd71e2..7a3fc6b050 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -167,9 +167,9 @@ enum _mgmt_table { #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column extern char *taosMsg[]; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 52408f78bf..b7434edadf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -35,6 +35,9 @@ */ #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f) == TSDB_COL_UDC) + #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -371,14 +374,14 @@ static bool hasTagValOutput(SQuery* pQuery) { * @return */ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (pStatis != NULL && !TSDB_COL_IS_TAG(pColIndex->flag)) { + if (pStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { *pColStatis = &pStatis[pColIndex->colIndex]; assert((*pColStatis)->colId == pColIndex->colId); } else { *pColStatis = NULL; } - if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } @@ -884,14 +887,14 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { - dataBlock = NULL; - } else { + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { SColIndex* pColIndex = &pQuery->pSelectExpr[col].base.colInfo; SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); assert(p->info.colId == pColIndex->colId); dataBlock = p->pData; + } else { + dataBlock = NULL; } } @@ -1536,7 +1539,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputBytes = pQuery->tagColList[index].bytes; pCtx->inputType = pQuery->tagColList[index].type; } - + } else if (TSDB_COL_IS_UD_COL(pIndex->flag)) { + pCtx->inputBytes = pSqlFuncMsg->arg[0].argBytes; + pCtx->inputType = pSqlFuncMsg->arg[0].argType; } else { pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; @@ -5231,6 +5236,8 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } + } else if (pExprMsg->colInfo.flag == TSDB_COL_UDC) { // user specified column data + return TSDB_UD_COLUMN_INDEX; } else { while (j < pQueryMsg->numOfCols) { if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { @@ -5590,9 +5597,18 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * bytes = tDataTypeDesc[type].nSize; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column SSchema s = tGetTableNameColumnSchema(); - type = s.type; + type = s.type; bytes = s.bytes; - } else{ + } else if (pExprs[i].base.colInfo.colId == TSDB_UD_COLUMN_INDEX) { + assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); + + type = pExprs[i].base.arg[0].argType; + bytes = pExprs[i].base.arg[0].argBytes; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + bytes += VARSTR_HEADER_SIZE; + } + } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); @@ -5765,7 +5781,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { // todo opt performance SColIndex *pColIndex = &pSqlExprMsg->colInfo; - if (!TSDB_COL_IS_TAG(pColIndex->flag)) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { int32_t f = 0; for (f = 0; f < pQuery->numOfCols; ++f) { if (pColIndex->colId == pQuery->colList[f].colId) { @@ -5773,8 +5789,10 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - - assert (f < pQuery->numOfCols); + + assert(f < pQuery->numOfCols); + } else if (pColIndex->colId == TSDB_UD_COLUMN_INDEX) { + // do nothing } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { From 58bad4cf38e9d7b02ac4b895cffe080ea6a5ed72 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 13:18:49 +0800 Subject: [PATCH 007/111] [td-1151] fix the coverity warnings --- src/client/src/tscSQLParser.c | 6 ++-- src/client/src/tscSubquery.c | 57 ++++++++++++----------------------- src/query/inc/qTsbuf.h | 4 +-- src/query/src/qExecutor.c | 40 +++++++++++++----------- src/query/src/qResultbuf.c | 11 +++++-- src/query/src/qTsbuf.c | 2 +- src/util/src/hash.c | 2 +- 7 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index fc33e22eff..b7c0c65c57 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1166,7 +1166,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t size_t numOfNode = taosArrayGetSize(colList); for(int32_t k = 0; k < numOfNode; ++k) { SColIndex* pIndex = taosArrayGet(colList, k); - if (pIndex->flag == 1) { + if (TSDB_COL_IS_TAG(pIndex->flag)) { + tExprTreeDestroy(&pNode, NULL); taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1366,7 +1367,8 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn ids.num = 1; ids.ids[0] = *pIndex; - if (pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta) || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || + pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { ids.num = 0; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index eedb229e1c..e9244fa968 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -685,6 +685,9 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; tscQueueAsyncRes(pParentSql); + + taosArrayDestroy(s1); + taosArrayDestroy(s2); return; } @@ -1284,8 +1287,14 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); - + + // todo add test SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + if (pState == NULL) { + pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pSql->res.code; + } + pState->numOfTotal = pQueryInfo->numOfTables; pState->numOfRemain = pState->numOfTotal; @@ -1297,7 +1306,8 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); pState->numOfRemain = i; pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - + taosTFree(pState); + return pSql->res.code; } @@ -1511,7 +1521,6 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in tstrerror(code), subqueryIndex, trsupport->numOfRetry); SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); - if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); @@ -1522,8 +1531,14 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in return pParentSql->res.code; } - taos_free_result(pSql); - return tscProcessSql(pNew); + int32_t ret = tscProcessSql(pNew); + + // if failed to process sql, let following code handle the pSql + if (ret == TSDB_CODE_SUCCESS) { + taos_free_result(pSql); + } + + return code; } void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -2063,39 +2078,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { doBuildResFromSubqueries(pSql); tsem_post(&pSql->rspSem); - return; - - // continue retrieve data from vnode -// if (!tscHasRemainDataInSubqueryResultSet(pSql)) { -// tscDebug("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1); -// SSubqueryState* pState = NULL; -// -// // free all sub sqlobj -// for (int32_t i = 0; i < pSql->numOfSubs; ++i) { -// SSqlObj* pChildObj = pSql->pSubs[i]; -// if (pChildObj == NULL) { -// continue; -// } -// -// SJoinSupporter* pSupporter = (SJoinSupporter*)pChildObj->param; -// pState = pSupporter->pState; -// -// tscDestroyJoinSupporter(pChildObj->param); -// taos_free_result(pChildObj); -// } -// -// free(pState); -// -// pRes->completed = true; // set query completed -// tsem_post(&pSql->rspSem); -// return; -// } - - tscFetchDatablockFromSubquery(pSql); - if (pRes->code != TSDB_CODE_SUCCESS) { - return; - } } if (pSql->res.code == TSDB_CODE_SUCCESS) { diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index f6d672b26d..46e6f79014 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -85,8 +85,8 @@ typedef struct STSBuf { uint32_t fileSize; STSVnodeBlockInfoEx* pData; - int32_t numOfAlloc; - int32_t numOfVnodes; + uint32_t numOfAlloc; + uint32_t numOfVnodes; char* assistBuf; int32_t bufSize; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b7434edadf..32525a2444 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -967,6 +967,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; + assert(tsCols != NULL); while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; @@ -2730,7 +2731,8 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num); + + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, (size_t)(bytes * pData->num)); } offset += (int32_t)pData->num; @@ -3298,7 +3300,8 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); - tsBufNextPos(pRuntimeEnv->pTSBuf); + bool ret = tsBufNextPos(pRuntimeEnv->pTSBuf); + assert(ret); } // reverse order time range @@ -5225,7 +5228,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { if (pExprMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - return -1; + return TSDB_TBNAME_COLUMN_INDEX; } while(j < pQueryMsg->numOfTags) { @@ -5644,18 +5647,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols); - - SColumnInfo *pCol = &pQueryMsg->colList[j]; - - int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, - &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); - assert(ret == TSDB_CODE_SUCCESS); + if (j < 0 || j >= pQueryMsg->numOfCols) { + assert(0); + } else { + SColumnInfo *pCol = &pQueryMsg->colList[j]; + int32_t ret = + getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); + assert(ret == TSDB_CODE_SUCCESS); + } } } - *pExprInfo = pExprs; + *pExprInfo = pExprs; return TSDB_CODE_SUCCESS; } @@ -6194,14 +6198,16 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // make sure file exist if (FD_VALID(fd)) { - int32_t s = lseek(fd, 0, SEEK_END); - UNUSED(s); - qDebug("QInfo:%p ts comp data return, file:%s, size:%d", pQInfo, pQuery->sdata[0]->data, s); + uint64_t s = lseek(fd, 0, SEEK_END); + + qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { size_t sz = read(fd, data, s); - UNUSED(sz); + if(sz < s) { // todo handle error + assert(0); + } } else { - // todo handle error + UNUSED(s); } close(fd); diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 51ff892b33..acedad0f0e 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -137,8 +137,15 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { } //3. write to disk. - fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - fwrite(t, size, 1, pResultBuf->file); + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + if (ret != 0) { // todo handle the error case + + } + + ret = fwrite(t, size, 1, pResultBuf->file); + if (ret != size) { // todo handle the error case + + } if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { pResultBuf->fileSize = pg->info.offset + pg->info.length; diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index eb47a1b7a1..a6fc3da05f 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -75,7 +75,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - if ((int32_t)header.numOfVnode > pTSBuf->numOfAlloc) { + if (header.numOfVnode > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfVnode; STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 714f36f1cb..7d10545ce7 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -657,7 +657,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; From 1892645002239726c3b3a6138865e85e6d00efbb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 13:24:04 +0800 Subject: [PATCH 008/111] [td-1151] fix the coverity warnings --- src/tsdb/src/tsdbRead.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index bb3f7ca195..7abf86c6bd 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1072,8 +1072,8 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, - int32_t numOfExisted, int32_t *start, int32_t *end) { +static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, + int32_t* start, int32_t* end) { *start = -1; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { @@ -1233,6 +1233,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); + assert(end != -1); + if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it moveToNextRowInMem(pCheckInfo); } From 8972ebc1013ad7373d4ba7f901845890844ef576 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 15:50:52 +0800 Subject: [PATCH 009/111] [td-1103] refactor the structure for reducing memory consumption during interval query. --- src/query/inc/qExecutor.h | 16 +-- src/query/inc/qUtil.h | 3 +- src/query/inc/tsqlfunction.h | 14 +- src/query/src/qExecutor.c | 271 ++++++++++++++++++----------------- src/query/src/qUtil.c | 44 +++--- 5 files changed, 177 insertions(+), 171 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index d824a269c9..320ae8d137 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -42,20 +42,16 @@ typedef struct SSqlGroupbyExpr { } SSqlGroupbyExpr; typedef struct SPosInfo { - int32_t pageId; - int32_t rowId; + int32_t pageId:20; + int32_t rowId:12; } SPosInfo; -typedef struct SWindowStatus { - bool closed; -} SWindowStatus; - typedef struct SWindowResult { - uint16_t numOfRows; // number of rows of current time window - SWindowStatus status; // this result status: closed or opened SPosInfo pos; // Position of current result in disk-based output buffer + uint16_t numOfRows; // number of rows of current time window + bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo - STimeWindow window; // The time window that current result covers. + TSKEY skey; // start key of current time window } SWindowResult; /** @@ -79,6 +75,7 @@ typedef struct SWindowResInfo { int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key int64_t threshold; // threshold to halt query and return the generated results. + int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { @@ -123,6 +120,7 @@ typedef struct SQueryCostInfo { uint64_t elapsedTime; uint64_t computTime; uint64_t internalSupSize; + uint64_t numOfTimeWindows; } SQueryCostInfo; typedef struct SQuery { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b7f57bf1b5..98b0ed0011 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -38,7 +38,8 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf return &pWindowResInfo->pResult[slot]; } -#define curTimeWindow(_winres) ((_winres)->curIndex) +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval)} #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 58d2ff260e..1dafe15989 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -137,13 +137,13 @@ typedef struct SInterpInfoDetail { } SInterpInfoDetail; typedef struct SResultInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - int32_t numOfRes; // num of output result in current buffer - int32_t bufLen; // buffer size - void* interResultBuf; // output result buffer + int8_t hasResult; // result generated, not NULL value + bool initialized:1; // output buffer has been initialized + bool complete:1; // query has completed + bool superTableQ:1; // is super table query + int16_t numOfRes; // num of output result in current buffer + uint32_t bufLen; // buffer size + void* interResultBuf; // output result buffer } SResultInfo; struct SQLFunctionCtx; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 32525a2444..15277ee228 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -27,7 +27,8 @@ #include "query.h" #include "queryLog.h" #include "tlosertree.h" -#include "tscompression.h" + +#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1); /** * check if the primary column is load by default, otherwise, the program will @@ -215,16 +216,16 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { */ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; - + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - + int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS_DUMMY) { continue; } - + assert(pResInfo->numOfRes > numOfRes); pResInfo->numOfRes = numOfRes; } @@ -318,10 +319,10 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.function static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery* pQuery = pRuntimeEnv->pQuery; - + if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; - + qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); assert(pQuery->rec.rows >= 0); @@ -415,6 +416,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -450,8 +452,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.skey = pWindowResInfo->prevSKey; w.ekey = w.skey + pQuery->intervalTime - 1; } else { - int32_t slot = curTimeWindow(pWindowResInfo); - w = getWindowResult(pWindowResInfo, slot)->window; + int32_t slot = curTimeWindowIndex(pWindowResInfo); + SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes); } if (w.skey > ts || w.ekey < ts) { @@ -552,15 +555,15 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes } // set time window for current result - pWindowRes->window = *win; + pWindowRes->skey = win->skey; setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; } -static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot].status; + return pWindowResInfo->pResult[slot].closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -602,7 +605,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe // no qualified results exist, abort check int32_t numOfClosed = 0; - + if (pWindowResInfo->size == 0) { return pWindowResInfo->size; } @@ -620,16 +623,17 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { + if (pResult->closed) { numOfClosed += 1; continue; } - if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + TSKEY ekey = pResult->skey + pWindowResInfo->interval; + if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { - skey = pResult->window.skey; + skey = pResult->skey; break; } } @@ -642,26 +646,26 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - + setQueryStatus(pQuery, QUERY_RESBUF_FULL); } else { qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed); } } - + // output has reached the limitation, set query completed if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && pRuntimeEnv->scanFlag == MASTER_SCAN) { setQueryStatus(pQuery, QUERY_COMPLETED); } - + assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); return numOfClosed; } @@ -675,7 +679,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); STableQueryInfo* item = pQuery->current; - + if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); @@ -706,12 +710,12 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; @@ -735,12 +739,11 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, - int32_t offset) { +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; @@ -825,14 +828,14 @@ static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) //todo binary search static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (colId == p->info.colId) { return p->pData; } } - + return NULL; } @@ -961,7 +964,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + bool pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } @@ -990,8 +993,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } pWindowResInfo->curIndex = index; @@ -1044,8 +1047,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return -1; } - pWindowRes->window.skey = v; - pWindowRes->window.ekey = v; + pWindowRes->skey = v; + assert(pRuntimeEnv->windowResInfo.interval == 0); if (pWindowRes->pos.pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); @@ -1144,7 +1147,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx if (functionId == TSDB_FUNC_TS) { return true; } - + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { return false; } @@ -1251,8 +1254,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; @@ -1275,8 +1278,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { - pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); + closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } } @@ -1331,10 +1334,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + STableQueryInfo* pTableQInfo = pQuery->current; SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { @@ -1377,10 +1380,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, SDataStatis *pStatis, void *param, int32_t colIndex) { - + int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId; int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; - + SDataStatis *tpField = NULL; pCtx->hasNull = hasNullValue(&pQuery->pSelectExpr[colIndex].base.colInfo, pStatis, &tpField); pCtx->aInputElemBuf = inputData; @@ -1436,7 +1439,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - + if (pQuery->fillVal != NULL) { if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) { pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; @@ -1469,13 +1472,13 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p if (isSelectivityWithTagsQuery(pQuery)) { int32_t num = 0; int16_t tagLen = 0; - + SQLFunctionCtx *p = NULL; SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; - + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1494,7 +1497,7 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - taosTFree(pTagCtx); + taosTFree(pTagCtx); } } } @@ -1547,7 +1550,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; } - + assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1809,7 +1812,7 @@ bool colIdCheck(SQuery *pQuery) { return false; } } - + return true; } @@ -1969,7 +1972,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i } pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); - + assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2123,21 +2126,21 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } - + pRuntimeEnv->summary.loadBlockStatis += 1; - + if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { assert(status == BLK_DATA_ALL_NEEDED); - + // load the data block statistics to perform further filter pRuntimeEnv->summary.loadBlockStatis += 1; if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { } - + if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; @@ -2145,7 +2148,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return BLK_DATA_DISCARD; } - + pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); @@ -2250,11 +2253,11 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB SQuery* pQuery = pRuntimeEnv->pQuery; if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; - + if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { int32_t remain = (int32_t)(pRec->capacity - pRec->rows); int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain)); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pQuery->pSelectExpr[i].bytes; assert(bytes > 0 && newSize > 0); @@ -2278,7 +2281,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), newSize, pRec->capacity, newSize - pRec->rows); - + pRec->capacity = newSize; } } @@ -2388,7 +2391,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { char* val = tsdbGetTableName(pTable); assert(val != NULL); - + tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY); } else { char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); @@ -2396,7 +2399,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa tag->nType = TSDB_DATA_TYPE_NULL; return; } - + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; @@ -2443,7 +2446,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx]; - + // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { continue; @@ -2493,14 +2496,14 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes // in case of tag column, the tag information should be extracted from input buffer if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { tVariantDestroy(&pCtx[i].tag); - + int32_t type = pCtx[i].outputType; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); } else { tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); } - + } } @@ -2590,7 +2593,7 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { int32_t type = pQuery->pSelectExpr[i].type; @@ -2839,7 +2842,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); - assert(ts == pWindowRes->window.skey); + assert(ts == pWindowRes->skey); int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -2979,10 +2982,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * if (pTableQueryInfo == NULL) { return; } - + // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - + // TODO validate the assertion // if (!QUERY_IS_ASC_QUERY(pQuery)) { // assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); @@ -2998,7 +3001,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - + SWITCH_ORDER(pTableQueryInfo->cur.order); pTableQueryInfo->cur.vgroupIndex = -1; @@ -3008,10 +3011,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i); - if (!pStatus->closed) { + bool closed = getTimeWindowResStatus(pWindowResInfo, i); + if (!closed) { continue; } @@ -3185,7 +3188,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { qDebug("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows, pQuery->limit.offset - pQuery->rec.rows); - + pQuery->limit.offset -= pQuery->rec.rows; pQuery->rec.rows = 0; @@ -3197,14 +3200,14 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { int64_t numOfSkip = pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; pQuery->limit.offset = 0; - + qDebug("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip, 0, pQuery->rec.rows); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - + memmove(pQuery->sdata[i]->data, (char*) pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes); pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; @@ -3237,7 +3240,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = getWindowResult(pWindowResInfo, i); - if (!pResult->status.closed) { + if (!pResult->closed) { continue; } @@ -3275,10 +3278,10 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) || (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery))); - + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, @@ -3359,7 +3362,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pTableQueryInfo->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; - + pTableQueryInfo->win = pStatus->w; pQuery->window = pTableQueryInfo->win; } @@ -3375,7 +3378,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position @@ -3428,7 +3431,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; - + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, cond.twindow.skey, cond.twindow.ekey); @@ -3678,7 +3681,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { @@ -3709,7 +3712,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (!QUERY_IS_ASC_QUERY(pQuery)) { assert(win.ekey == pQuery->window.ekey); } - + pWindowResInfo->prevSKey = w.skey; } @@ -3735,7 +3738,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { */ STimeWindow *w = &pDataBlockInfo->window; STableQueryInfo* pTableQueryInfo = pQuery->current; - + bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); @@ -3856,7 +3859,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; @@ -3948,10 +3951,10 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; - + while (1) { int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity); - + // todo apply limit output function /* reached the start position of according to offset value, return immediately */ if (pQuery->limit.offset == 0) { @@ -3962,7 +3965,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int if (pQuery->limit.offset < ret) { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); - + ret -= (int32_t)pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer @@ -3970,14 +3973,14 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, ret * pQuery->pSelectExpr[i].bytes); } - + pQuery->limit.offset = 0; return ret; } else { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, " "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0, pQuery->limit.offset - ret); - + pQuery->limit.offset -= ret; pQuery->rec.rows = 0; ret = 0; @@ -3998,13 +4001,14 @@ static void queryCostStatis(SQInfo *pQInfo) { pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64, pQInfo, pSummary->internalSupSize); + qDebug("QInfo:%p :cost summary: internal size:%"PRId64", numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, + pSummary->numOfTimeWindows); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed @@ -4094,7 +4098,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; - + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4143,21 +4147,21 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { // set the abort info pQuery->pos = startPos; - + // reset the query start timestamp pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; pQuery->window.skey = pTableQueryInfo->win.skey; *start = pTableQueryInfo->win.skey; - + pWindowResInfo->prevSKey = tw.skey; int32_t index = pRuntimeEnv->windowResInfo.curIndex; - + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock); pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index - + qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); - + return true; } else { // do nothing *start = tw.skey; @@ -4225,7 +4229,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) - && (cond.order == TSDB_ORDER_ASC) + && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) && (!isFixedOutputQuery(pRuntimeEnv)) @@ -4250,21 +4254,21 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { int32_t numOfCols = pQuery->numOfOutput; int32_t offset = 0; - + SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; - + pFillCol[i].col.bytes = pExprInfo->bytes; pFillCol[i].col.type = (int8_t)pExprInfo->type; pFillCol[i].col.offset = offset; pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query pFillCol[i].functionId = pExprInfo->base.functionId; pFillCol[i].fillVal.i = pQuery->fillVal[i]; - + offset += pExprInfo->bytes; } - + return pFillCol; } @@ -4285,7 +4289,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (code != TSDB_CODE_SUCCESS) { return code; } - + pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4558,7 +4562,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); taosArrayPush(g1, &tx); - + STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; // include only current table @@ -4566,7 +4570,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); pRuntimeEnv->pQueryHandle = NULL; } - + if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); } else { @@ -4580,10 +4584,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } initCtxOutputBuf(pRuntimeEnv); - + SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); - + setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); if (isFirstLastRowQuery(pQuery)) { assert(taosArrayGetSize(s) == 1); @@ -4596,13 +4600,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->current = taosArrayGetP(first, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - + int64_t numOfRes = getNumOfResult(pRuntimeEnv); if (numOfRes > 0) { pQuery->rec.rows += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } - + skipResults(pRuntimeEnv); pQInfo->groupIndex += 1; @@ -4661,14 +4665,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + taosArrayDestroy(s); if (pWindowResInfo->size <= 0) { continue; } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; - pStatus->closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { @@ -4822,11 +4825,11 @@ static void doSaveContext(SQInfo *pQInfo) { SET_REVERSE_SCAN_FLAG(pRuntimeEnv); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + STsdbQueryCond cond = { .order = pQuery->order.order, .colList = pQuery->colList, @@ -4968,14 +4971,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. return; } - + pQuery->current = pTableInfo; // set current query table info - + scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); finalizeQueryResult(pRuntimeEnv); @@ -4993,10 +4996,10 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - + // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); @@ -5087,7 +5090,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { int32_t numOfFilled = 0; TSKEY newStartKey = TSKEY_INITIAL_VAL; - + // skip blocks without load the actual data block from file if no filter condition present skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { @@ -5114,7 +5117,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata); numOfFilled = 0; - + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); @@ -5180,11 +5183,11 @@ static void tableQueryImpl(SQInfo *pQInfo) { // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); SArray* g = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* item = taosArrayGetP(g, 0); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); @@ -5384,7 +5387,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; pColFilter->filterstr = htons(pFilterMsg->filterstr); @@ -5805,7 +5808,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - + assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); } } @@ -6062,7 +6065,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } - + return code; _error: @@ -6140,7 +6143,7 @@ static void freeQInfo(SQInfo *pQInfo) { taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - + if (pQuery->pGroupbyExpr != NULL) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); taosTFree(pQuery->pGroupbyExpr); @@ -6217,7 +6220,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); if (fd != -1) { - close(fd); + close(fd); } } @@ -6236,7 +6239,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit); setQueryStatus(pQuery, QUERY_OVER); } - + return TSDB_CODE_SUCCESS; } @@ -6338,7 +6341,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi pExprs = NULL; pGroupbyExpr = NULL; pTagColumnInfo = NULL; - + if ((*pQInfo) == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _over; @@ -6353,7 +6356,7 @@ _over: if (pGroupbyExpr != NULL) { taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); - } + } free(pTagColumnInfo); free(pExprs); free(pExprMsg); @@ -6520,7 +6523,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = 0; (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } - + (*pRsp)->precision = htons(pQuery->precision); if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { doDumpQueryResult(pQInfo, (*pRsp)->data); @@ -6597,7 +6600,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { if (numOfGroup == 0) { return; } - + SArray* pa = GET_TABLEGROUP(pQInfo, 0); size_t num = taosArrayGetSize(pa); @@ -6698,7 +6701,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { } else { type = pExprInfo[j].type; bytes = pExprInfo[j].bytes; - + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a01eb33ec7..509362863c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -45,8 +45,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + + SQueryCostInfo* pSummary = &pRuntimeEnv->summary; // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); @@ -54,8 +54,11 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; - pRuntimeEnv->summary.internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pWindowResInfo->interval = pRuntimeEnv->pQuery->intervalTime; + + pSummary->internalSupSize += sizeof(SWindowResult) * threshold; + pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pSummary->numOfTimeWindows = threshold; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); @@ -126,8 +129,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { // remove the window slot from hash table - taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type); + if (pResult->closed) { // remove the window slot from hash table + taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type); } else { break; } @@ -149,12 +152,12 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize, + taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize, (char *)&v, sizeof(int32_t)); } @@ -173,7 +176,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].status.closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { ++i; } @@ -184,11 +187,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].status.closed) { + if (pWindowResInfo->pResult[i].closed) { continue; } - pWindowResInfo->pResult[i].status.closed = true; + pWindowResInfo->pResult[i].closed = true; } } @@ -204,7 +207,7 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].window.skey < pWindowResInfo->pResult[1].window.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1; if (order != resultOrder) { return; @@ -212,11 +215,12 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.ekey < lastKey)) { + TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval; + while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) { ++i; } } @@ -227,11 +231,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->status.closed == true); + return (getWindowResult(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->status.closed = true; + getWindowResult(pWindowResInfo, slot)->closed = true; } void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { @@ -253,8 +257,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow pWindowRes->numOfRows = 0; pWindowRes->pos = (SPosInfo){-1, -1}; - pWindowRes->status.closed = false; - pWindowRes->window = TSWINDOW_INITIALIZER; + pWindowRes->closed = false; + pWindowRes->skey = TSKEY_INITIAL_VAL; } /** @@ -264,8 +268,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - dst->window = src->window; - dst->status = src->status; + dst->skey = src->skey; + dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; From 2238b29ffecc124d1fa4ccbd053e8d3ecb349f91 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Aug 2020 15:51:52 +0800 Subject: [PATCH 010/111] [td-1103] fix a typo --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 15277ee228..67464c36eb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -28,7 +28,7 @@ #include "queryLog.h" #include "tlosertree.h" -#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1); +#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) /** * check if the primary column is load by default, otherwise, the program will From 9031c50ca4cc9b77b04cc1dc59da1f0cd30e203f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 19 Aug 2020 01:22:36 +0800 Subject: [PATCH 011/111] [td-1103] fix bugs --- src/common/src/tvariant.c | 12 +++++++++++- src/query/inc/qExecutor.h | 2 +- src/query/inc/qUtil.h | 4 ++-- src/query/inc/tsqlfunction.h | 10 +++++----- src/query/src/qExecutor.c | 22 +++++++++++++++------- src/query/src/qTsbuf.c | 13 +++++++------ 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 6716a1827e..6e8111aa72 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -175,7 +175,17 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { - assert((p1->nType != TSDB_DATA_TYPE_NULL) || (p2->nType != TSDB_DATA_TYPE_NULL)); + if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) { + return 0; + } + + if (p1->nType == TSDB_DATA_TYPE_NULL) { + return -1; + } + + if (p2->nType == TSDB_DATA_TYPE_NULL) { + return 1; + } switch (p1->nType) { case TSDB_DATA_TYPE_BINARY: diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 320ae8d137..5126dd347e 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -118,7 +118,7 @@ typedef struct SQueryCostInfo { uint32_t loadBlockStatis; uint32_t discardBlocks; uint64_t elapsedTime; - uint64_t computTime; + uint64_t firstStageMergeTime; uint64_t internalSupSize; uint64_t numOfTimeWindows; } SQueryCostInfo; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 98b0ed0011..6de3c7c0e5 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -38,8 +38,8 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf return &pWindowResInfo->pResult[slot]; } -#define curTimeWindowIndex(_winres) ((_winres)->curIndex) -#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval)} +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval - 1)} #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 1dafe15989..a443a43a23 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -138,11 +138,11 @@ typedef struct SInterpInfoDetail { typedef struct SResultInfo { int8_t hasResult; // result generated, not NULL value - bool initialized:1; // output buffer has been initialized - bool complete:1; // query has completed - bool superTableQ:1; // is super table query - int16_t numOfRes; // num of output result in current buffer - uint32_t bufLen; // buffer size + bool initialized; // output buffer has been initialized + bool complete; // query has completed + bool superTableQ; // is super table query + uint32_t bufLen; // buffer size + uint64_t numOfRes; // num of output result in current buffer void* interResultBuf; // output result buffer } SResultInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 67464c36eb..c98bac96ba 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -970,7 +970,6 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; - assert(tsCols != NULL); while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; @@ -2668,7 +2667,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } int32_t mergeIntoGroupResult(SQInfo *pQInfo) { - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampUs(); int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); @@ -2695,9 +2694,11 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { SET_STABLE_QUERY_OVER(pQInfo); } - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo, - pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + int64_t elapsedTime = taosGetTimestampUs() - st; + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, elapsedTime); + pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; return TSDB_CODE_SUCCESS; } @@ -2830,6 +2831,13 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { while (1) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); + + taosTFree(pTableList); + taosTFree(posList); + taosTFree(pTree); + taosTFree(pResultInfo); + taosTFree(buf); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -3996,9 +4004,9 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; - qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, total blocks:%d, load block statis:%d," - " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, + pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); qDebug("QInfo:%p :cost summary: internal size:%"PRId64", numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index a6fc3da05f..518bb4083b 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -232,7 +232,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); - UNUSED(r); + assert(r == 0); /* * format for output data: @@ -241,13 +241,14 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * * both side has the compressed length is used to support load data forwards/backwords. */ - fwrite(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - fwrite(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t metaLen = 0; + metaLen += fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - fwrite(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + metaLen += fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - fwrite(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + metaLen += fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); } fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); @@ -255,7 +256,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; pTSBuf->tsData.len = 0; From 91275a73b6bd6b5a1b1093d83470f28a68f4bc7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 13:54:11 +0800 Subject: [PATCH 012/111] [td-225] fix jdbc memory leaks. --- .../jni/com_taosdata_jdbc_TSDBJNIConnector.h | 8 ++++++++ src/client/src/TSDBJNIConnector.c | 18 ++++++++++++++++++ src/connector/jdbc/.classpath | 17 ----------------- .../com/taosdata/jdbc/TSDBJNIConnector.java | 2 ++ 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index d81bbd9b6d..eaea91d1bf 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -89,6 +89,14 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp (JNIEnv *env, jobject jobj, jlong con, jlong tres); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: isUpdateQueryImp + * Signature: (J)J + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp + (JNIEnv *env, jobject jobj, jlong con, jlong tres); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: freeResultSetImp diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 34204f96bf..bc7d459a44 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -336,6 +336,24 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp( } } +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp(JNIEnv *env, jobject jobj, jlong con, + jlong tres) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection is closed", jobj); + return JNI_CONNECTION_NULL; + } + + if ((void *)tres == NULL) { + jniError("jobj:%p, conn:%p, resultset is null", jobj, tscon); + return JNI_RESULT_SET_NULL; + } + + SSqlObj *pSql = (TAOS_RES *)tres; + + return (tscIsUpdateQuery(pSql)? 1:0); +} + JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(JNIEnv *env, jobject jobj, jlong con, jlong res) { TAOS *tscon = (TAOS *)con; diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath index 39abf1c5e9..a5d95095cc 100644 --- a/src/connector/jdbc/.classpath +++ b/src/connector/jdbc/.classpath @@ -28,22 +28,5 @@ - - - - - - - - - - - - - - - - - diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index bab3c79089..20a329074e 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -170,6 +170,8 @@ public class TSDBJNIConnector { } private native long getResultSetImp(long connection, long pSql); + + private native long isUpdateQueryImp(long connection, long pSql); /** * Free resultset operation from C to release resultset pointer by JNI From 61622a73b1abb2bb558453873f3c27ce8280785a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 14:20:31 +0800 Subject: [PATCH 013/111] [td-225] fix jdbc memory leaks. --- src/client/src/TSDBJNIConnector.c | 7 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 34 ++-- .../java/com/taosdata/jdbc/TSDBStatement.java | 31 ++-- src/query/inc/qExecutor.h | 23 +-- src/query/src/qExecutor.c | 153 +++++++++++------- src/tsdb/src/tsdbRead.c | 2 +- tests/script/general/parser/fill_stb.sim | 8 +- 7 files changed, 162 insertions(+), 96 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index bc7d459a44..b25f620508 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -327,13 +327,12 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp( STscObj *pObj = pSql->pTscObj; if (tscIsUpdateQuery(pSql)) { - // taos_free_result(pSql); // free result here - jniDebug("jobj:%p, conn:%p, no resultset, %p", jobj, pObj, (void *)tres); - return 0; + jniDebug("jobj:%p, conn:%p, update query, no resultset, %p", jobj, pObj, (void *)tres); } else { jniDebug("jobj:%p, conn:%p, get resultset, %p", jobj, pObj, (void *)tres); - return tres; } + + return tres; } JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp(JNIEnv *env, jobject jobj, jlong con, diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 20a329074e..0cd185de50 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -111,6 +111,8 @@ public class TSDBJNIConnector { * @throws SQLException */ public long executeQuery(String sql) throws SQLException { + // close previous result set if the user forgets to invoke the + // free method to close previous result set. if (!this.isResultsetClosed) { freeResultSet(taosResultSetPointer); } @@ -123,21 +125,20 @@ public class TSDBJNIConnector { this.freeResultSet(pSql); throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding")); } + int code = this.getErrCode(pSql); if (code != 0) { affectedRows = -1; - String err_msg = this.getErrMsg(pSql); + String msg = this.getErrMsg(pSql); + this.freeResultSet(pSql); - throw new SQLException(TSDBConstants.WrapErrMsg(err_msg), "", code); + throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code); } - // Try retrieving result set for the executed SQL using the current connection pointer. If the executed - // SQL is a DML/DDL which doesn't return a result set, then taosResultSetPointer should be 0L. Otherwise, - // taosResultSetPointer should be a non-zero value. + // Try retrieving result set for the executed SQL using the current connection pointer. taosResultSetPointer = this.getResultSetImp(this.taos, pSql); - if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - isResultsetClosed = false; - } + isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER); + return pSql; } @@ -171,6 +172,10 @@ public class TSDBJNIConnector { private native long getResultSetImp(long connection, long pSql); + public boolean isUpdateQuery(long pSql) { + return isUpdateQueryImp(this.taos, pSql) == 1? true:false; + } + private native long isUpdateQueryImp(long connection, long pSql); /** @@ -180,13 +185,14 @@ public class TSDBJNIConnector { int res = TSDBConstants.JNI_SUCCESS; if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { throw new RuntimeException("Invalid result set pointer"); - } else if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, result); - isResultsetClosed = true; // reset resultSetPointer to 0 after freeResultSetImp() return - taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - } else { - isResultsetClosed = true; } + + if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { + res = this.freeResultSetImp(this.taos, result); + taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; + } + + isResultsetClosed = true; return res; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index e828864313..075a351661 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -51,6 +51,8 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } + + // TODO make sure it is not a update query pSql = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); @@ -58,33 +60,40 @@ public class TSDBStatement implements Statement { if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { this.connecter.freeResultSet(pSql); throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { -// create/insert/update/del/alter + } + + // create/insert/update/delete/alter + if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { this.connecter.freeResultSet(pSql); return null; - } else { - return new TSDBResultSet(this.connecter, resultSetPointer); } + + if (!this.connecter.isUpdateQuery(pSql)) { + return new TSDBResultSet(this.connecter, resultSetPointer); + } else { + this.connecter.freeResultSet(pSql); + return null; + } + } public int executeUpdate(String sql) throws SQLException { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } + + // TODO check if current query is update query pSql = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { this.connecter.freeResultSet(pSql); throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - this.connecter.freeResultSet(); - throw new SQLException("The executed SQL is not a DML or a DDL"); - } else { - int num = this.connecter.getAffectedRows(pSql); - this.connecter.freeResultSet(pSql); - return num; } + + int num = this.connecter.getAffectedRows(pSql); + this.connecter.freeResultSet(pSql); + return num; } public String getErrorMsg(long pSql) { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 5126dd347e..b023de0dc9 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,6 +33,17 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct SPosInfo { + int32_t pageId:20; + int32_t rowId:12; +} SPosInfo; + +typedef struct SGroupResInfo { + int32_t groupId; + int32_t numOfDataPages; + SPosInfo pos; +} SGroupResInfo; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -41,11 +52,6 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SPosInfo { - int32_t pageId:20; - int32_t rowId:12; -} SPosInfo; - typedef struct SWindowResult { SPosInfo pos; // Position of current result in disk-based output buffer uint16_t numOfRows; // number of rows of current time window @@ -190,18 +196,15 @@ typedef struct SQInfo { STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; - int32_t groupIndex; - int32_t offset; // offset in group result set of subgroup, todo refactor SArray* arrTableIdInfo; + int32_t groupIndex; /* * the query is executed position on which meter of the whole list. * when the index reaches the last one of the list, it means the query is completed. - * We later may refactor to remove this attribution by using another flag to denote - * whether a multimeter query is completed or not. */ int32_t tableIndex; - int32_t numOfGroupResultPages; + SGroupResInfo groupResInfo; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2cab8cef71..3754276f93 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -155,9 +155,9 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQInfo *pQInfo); +static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); -bool doFilterData(SQuery *pQuery, int32_t elemPos) { + bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; @@ -231,7 +231,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static int32_t getGroupResultId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) { int32_t base = 20000000; return base + (groupIndex * 10000); } @@ -2686,11 +2686,12 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { break; } - assert(pQInfo->numOfGroupResultPages == 0); + assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - if (pQInfo->groupIndex == numOfGroups && pQInfo->offset == pQInfo->numOfGroupResultPages) { + SGroupResInfo* info = &pQInfo->groupResInfo; + if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -2703,8 +2704,12 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { } void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - if (pQInfo->offset == pQInfo->numOfGroupResultPages) { - pQInfo->numOfGroupResultPages = 0; + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + // all results have been return to client, try next group + if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { + pGroupResInfo->numOfDataPages = 0; + pGroupResInfo->pos.rowId = 0; // current results of group has been sent to client, try next group if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { @@ -2713,7 +2718,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } @@ -2722,30 +2727,50 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t id = getGroupResultId(pQInfo->groupIndex - 1); - SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); - - int32_t size = (int32_t)(taosArrayGetSize(list)); + int32_t id = pQInfo->groupResInfo.groupId; + SIDList list = getDataBufPagesIdList(pResultBuf, id); int32_t offset = 0; - for (int32_t j = 0; j < size; ++j) { + int32_t numOfCopiedRows = 0; + + size_t size = taosArrayGetSize(list); + assert(size == pGroupResInfo->numOfDataPages); + + bool done = false; + for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage *pData = getResBufPage(pResultBuf, pi->pageId); + tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); + + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); + int32_t numOfRes = pData->num - pGroupResInfo->pos.rowId; + + if (numOfRes > pQuery->rec.capacity - offset) { + numOfCopiedRows = pQuery->rec.capacity - offset; + pGroupResInfo->pos.rowId += numOfCopiedRows; + done = true; + } else { + numOfCopiedRows = pData->num; + + pGroupResInfo->pos.pageId += 1; + pGroupResInfo->pos.rowId = 0; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, (size_t)(bytes * pData->num)); + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, + (size_t)bytes * numOfCopiedRows); } - offset += (int32_t)pData->num; + offset += numOfCopiedRows; + if (done) { + break; + } } assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; - pQInfo->offset += 1; } int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { @@ -2791,23 +2816,38 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { // todo opt for the case of one table per group int32_t numOfTables = 0; + SIDList pageList = NULL; + int32_t tid = -1; + for (int32_t i = 0; i < size; ++i) { STableQueryInfo *item = taosArrayGetP(pGroup, i); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); + pageList = list; + tid = TSDB_TABLEID(item->pTable)->tid; + if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables] = item; - numOfTables += 1; + pTableList[numOfTables++] = item; } } + // there is no data in current group if (numOfTables == 0) { taosTFree(posList); taosTFree(pTableList); - - assert(pQInfo->numOfGroupResultPages == 0); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group + taosTFree(posList); + taosTFree(pTableList); + + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + pGroupResInfo->numOfDataPages = taosArrayGetSize(pageList); + pGroupResInfo->groupId = tid; + pGroupResInfo->pos.pageId = 0; + pGroupResInfo->pos.rowId = 0; + + return pGroupResInfo->numOfDataPages; } SCompSupporter cs = {pTableList, posList, pQInfo}; @@ -2824,6 +2864,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); + // todo add windowRes iterator int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -2868,7 +2910,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { doMerge(pRuntimeEnv, ts, pWindowRes, true); } else { // copy data to disk buffer if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { return -1; } @@ -2905,7 +2947,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); taosTFree(pTree); @@ -2929,16 +2971,14 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(posList); taosTFree(pTree); - pQInfo->offset = 0; - taosTFree(pResultInfo); taosTFree(buf); - return pQInfo->numOfGroupResultPages; + + return pQInfo->groupResInfo.numOfDataPages; } -int32_t flushFromResultBuf(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -2946,32 +2986,32 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { int32_t pageId = -1; int32_t capacity = pResultBuf->numOfRowsPerPage; - int32_t remain = (int32_t)pQuery->sdata[0]->num; + int32_t remain = (int32_t) pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { - int32_t r = remain; - if (r > capacity) { - r = capacity; - } + int32_t rows = (remain > capacity)? capacity:remain; + assert(rows > 0); - int32_t id = getGroupResultId(pQInfo->groupIndex) + pQInfo->numOfGroupResultPages; - tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); + // get the output buffer page + tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); + buf->num = rows; - // pagewise copy to dest buffer + // pagewisely copy to dest buffer for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - buf->num = r; - memcpy(buf->data + pRuntimeEnv->offset[i] * buf->num, ((char *)pQuery->sdata[i]->data) + offset * bytes, - buf->num * bytes); + char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; + char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; + memcpy(output, src, buf->num * bytes); } - offset += r; - remain -= r; + offset += rows; + remain -= rows; + + pGroupResInfo->numOfDataPages += 1; } - pQInfo->numOfGroupResultPages += 1; return TSDB_CODE_SUCCESS; } @@ -3773,27 +3813,27 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ step = -1; } + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { if (result[i].numOfRows == 0) { - pQInfo->offset = 0; pQInfo->groupIndex += 1; + pGroupResInfo->pos.rowId = 0; continue; } - assert(pQInfo->offset <= 1); - - int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; - int32_t oldOffset = pQInfo->offset; + int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId; + int32_t oldOffset = pGroupResInfo->pos.rowId; /* - * current output space is not enough to keep all the result data of this group, only copy partial results - * to SQuery object's result buffer + * current output space is not enough to accommodate all data of this page, only partial results + * will be copied to SQuery object's result buffer */ if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { - numOfRowsToCopy = (int32_t)pQuery->rec.capacity - numOfResult; - pQInfo->offset += numOfRowsToCopy; + numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; + pGroupResInfo->pos.rowId += numOfRowsToCopy; } else { - pQInfo->offset = 0; + pGroupResInfo->pos.rowId = 0; pQInfo->groupIndex += 1; } @@ -4004,12 +4044,15 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; + // add the merge time + pSummary->elapsedTime += pSummary->firstStageMergeTime; + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64", numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, + qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, pSummary->numOfTimeWindows); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 7db9937a6d..924288ea55 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2564,7 +2564,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDestroyHelper(&pQueryHandle->rhelper); SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", + tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); taosTFree(pQueryHandle); diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index 402b79d6b5..f0cd058352 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -133,12 +133,18 @@ endi if $data74 != -4.000000000 then return -1 endi + ## fill(value) + group by sql select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) group by t1 $val = $rowNum * 2 +print $rowNum, $val + $val = $val - 1 $val = $val * $tbNum -if $rows != $val then +print ==================== $val + +if $rows != 190 then + print expect 190, actual:$rows return -1 endi if $data06 != 0 then From 49f23d689d5580cdacdcaf61a3fd8e4373a7716b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 18:08:31 +0800 Subject: [PATCH 014/111] [td-1067] opt last_row queries --- src/query/src/qExecutor.c | 8 +- src/tsdb/src/tsdbRead.c | 394 +++++++++++++++++++------------------- 2 files changed, 195 insertions(+), 207 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3754276f93..1103b9b531 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4623,7 +4623,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); + assert(0); // last_row query switch to other routine to handle +// pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); } else { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } @@ -4984,7 +4985,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { el = scanMultiTableDataBlocks(pQInfo); qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); -// doCloseAllTimeWindowAfterScan(pQInfo); doRestoreContext(pQInfo); } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); @@ -5262,15 +5262,13 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol && - !isFirstLastRowQuery(pQuery))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol)) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); sequentialTableProcess(pQInfo); - } // record the total elapsed time diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 924288ea55..ba46427b1a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -28,6 +28,14 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ + .tid = (_checkInfo)->tableId.tid, \ + .uid = (_checkInfo)->tableId.uid}) + + enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, @@ -119,7 +127,14 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +typedef struct STableGroupSupporter { + int32_t numOfCols; + SColIndex* pCols; + STSchema* pTagSchema; +} STableGroupSupporter; + +static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); + static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); @@ -283,12 +298,10 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { + pCond->order = TSDB_ORDER_DESC; + pCond->twindow = changeTableGroupByLastrow(groupList); + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); - if (pQueryHandle != NULL) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; - pQueryHandle->order = TSDB_ORDER_DESC; - changeQueryHandleForLastrowQuery(pQueryHandle); - } return pQueryHandle; } @@ -672,14 +685,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return code; } -#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ - ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ - .numOfCols = (_block)->numOfCols, \ - .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) - - static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { STsdbRepo *pRepo = pQueryHandle->pTsdb; int64_t st = taosGetTimestampUs(); @@ -1660,6 +1665,128 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { return false; } +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { + // filter the queried time stamp in the first place + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + pQueryHandle->order = TSDB_ORDER_DESC; + + assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); + + // starts from the buffer in case of descending timestamp order check data blocks + // todo consider the query time window, current last_row does not apply the query time window + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + int32_t i = 0; + while(i < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && + pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { + break; + } + + i++; + } + + // there are no data in all the tables + if (i == numOfTables) { + return; + } + + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); + taosArrayClear(pQueryHandle->pTableCheckInfo); + + info.lastKey = pQueryHandle->window.skey; + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + + // update the query time window according to the chosen last timestamp + pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; +} + +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle) { + int numOfRows = 0; + int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + win->skey = TSKEY_INITIAL_VAL; + + int64_t st = taosGetTimestampUs(); + STable* pTable = pCheckInfo->pTableObj; + + do { + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + if (row == NULL) { + break; + } + + TSKEY key = dataRowKey(row); + if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + pQueryHandle->window.ekey); + + break; + } + + if (win->skey == INT64_MIN) { + win->skey = key; + } + + win->ekey = key; + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + + if (++numOfRows >= maxRowsToRead) { + moveToNextRowInMem(pCheckInfo); + break; + } + + } while(moveToNextRowInMem(pCheckInfo)); + + assert(numOfRows <= maxRowsToRead); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + int32_t emptySize = maxRowsToRead - numOfRows; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } + + int64_t elapsedTime = taosGetTimestampUs() - st; + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); + + return numOfRows; +} + +static int32_t getAllTableList(STable* pSuperTable, SArray* list) { + SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + while (tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + + STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(list, &info); + } + + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} + +static void destroyHelper(void* param) { + if (param == NULL) { + return; + } + + + tQueryInfo* pInfo = (tQueryInfo*)param; + if (pInfo->optr != TSDB_RELATION_IN) { + taosTFree(pInfo->q); + } + + free(param); +} + // handle data in cache situation bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; @@ -1796,6 +1923,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { } if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block bool exists = true; int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); @@ -1824,150 +1952,48 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return ret; } -void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; - assert(!ASCENDING_TRAVERSE(pQueryHandle->order)); - - // starts from the buffer in case of descending timestamp order check data blocks +STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { + STimeWindow window = {INT64_MAX, INT64_MIN}; + // NOTE: starts from the buffer in case of descending timestamp order check data blocks // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); + for(int32_t j = 0; j < numOfGroups; ++j) { + SArray* pGroup = taosArrayGetP(groupList->pGroupList, j); + TSKEY key = TSKEY_INITIAL_VAL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t index = -1; + STableKeyInfo keyInfo = {0}; - for(int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pCheckInfo->pTableObj->lastKey > key) { - key = pCheckInfo->pTableObj->lastKey; - index = i; + size_t numOfTables = taosArrayGetSize(pGroup); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + + if (key < lastKey) { + key = lastKey; + + keyInfo.pTable = pKeyInfo->pTable; + keyInfo.lastKey = key; + pKeyInfo->lastKey = key; + + if (key < window.skey) { + window.skey = key; + } + + if (key > window.ekey) { + window.ekey = key; + } + } + } + + // more than one table in each group, only one table left for each group + if (numOfTables > 1) { + taosArrayClear(pGroup); + taosArrayPush(pGroup, &keyInfo); } } - if (index == -1) { - // todo add failure test cases - return; - } - - // erase all other elements in array list - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - if (i == index) { - continue; - } - - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - tSkipListDestroyIter(pTableCheckInfo->iter); - - if (pTableCheckInfo->pDataCols != NULL) { - taosTFree(pTableCheckInfo->pDataCols->buf); - } - - taosTFree(pTableCheckInfo->pDataCols); - taosTFree(pTableCheckInfo->pCompInfo); - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = key; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {key, key}; -} - -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { - // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - pQueryHandle->order = TSDB_ORDER_DESC; - - assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); - - // starts from the buffer in case of descending timestamp order check data blocks - // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - - int32_t i = 0; - while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && - pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { - break; - } - - i++; - } - - // there are no data in all the tables - if (i == numOfTables) { - return; - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; -} - -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { - int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - win->skey = TSKEY_INITIAL_VAL; - - int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; - - do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); - if (row == NULL) { - break; - } - - TSKEY key = dataRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); - - break; - } - - if (win->skey == INT64_MIN) { - win->skey = key; - } - - win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); - - if (++numOfRows >= maxRowsToRead) { - moveToNextRowInMem(pCheckInfo); - break; - } - - } while(moveToNextRowInMem(pCheckInfo)); - - assert(numOfRows <= maxRowsToRead); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { - int32_t emptySize = maxRowsToRead - numOfRows; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); - } - } - - int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); - - return numOfRows; + return window; } void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { @@ -2104,36 +2130,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } -static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); - while (tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); - - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(list, &info); - } - - tSkipListDestroyIter(iter); - return TSDB_CODE_SUCCESS; -} - -static void destroyHelper(void* param) { - if (param == NULL) { - return; - } - - - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - taosTFree(pInfo->q); - } - -// tVariantDestroy(&(pInfo->q)); - free(param); -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -2160,13 +2156,7 @@ void filterPrepare(void* expr, void* param) { } } -typedef struct STableGroupSupporter { - int32_t numOfCols; - SColIndex* pCols; - STSchema* pTagSchema; -} STableGroupSupporter; - -int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { +static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; STable* pTable1 = ((STableKeyInfo*) p1)->pTable; STable* pTable2 = ((STableKeyInfo*) p2)->pTable; @@ -2219,8 +2209,19 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, - __ext_compar_fn_t compareFn) { +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} + +void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, + STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); @@ -2293,7 +2294,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); @@ -2591,14 +2592,3 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } - -static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { - return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { - return 1; - } else { - ASSERT(false); - return 0; - } -} From c5cfeee49373c48dee545161cbe6eaff2211efbc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 23:40:39 +0800 Subject: [PATCH 015/111] [td-1067] fix bugs in lastrow query. --- src/query/src/qExecutor.c | 2 +- src/tsdb/src/tsdbRead.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1103b9b531..bf72f209d6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5262,7 +5262,7 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol)) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ba46427b1a..b189c7a406 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -301,6 +301,9 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab pCond->order = TSDB_ORDER_DESC; pCond->twindow = changeTableGroupByLastrow(groupList); + //descending order query, skey >= ekey + SWAP(pCond->twindow.skey, pCond->twindow.ekey, TSKEY); + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); return pQueryHandle; } From 2c4aff8f520c2dd351f3ef2ee8c838273a9335c8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 20 Aug 2020 23:58:35 +0800 Subject: [PATCH 016/111] [td-1067] fix bugs in lastrow query. --- src/client/src/tscFunctionImpl.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index f1f235cd36..495136b393 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1811,23 +1811,23 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { * NOTE: last_row does not use the interResultBuf to keep the result */ static void last_row_function(SQLFunctionCtx *pCtx) { - assert(pCtx->size == 1); - + assert(pCtx->size >= 1); char *pData = GET_INPUT_CHAR(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + + // assign the last element in current data block + assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; - pInfo->ts = pCtx->ptsList[0]; - + pInfo->ts = pCtx->ptsList[pCtx->size - 1]; pInfo->hasResult = DATA_SET_FLAG; - // set the result to final result buffer + // set the result to final result buffer in case of super table query if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[0]; + pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); From 084cbca717607beadd77660997d759e4aa02c850 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Aug 2020 00:15:08 +0800 Subject: [PATCH 017/111] [td-1067] fix bugs in lastrow query. --- src/client/src/tscFunctionImpl.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 495136b393..e494446ce5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1820,10 +1820,6 @@ static void last_row_function(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; - pInfo->ts = pCtx->ptsList[pCtx->size - 1]; - pInfo->hasResult = DATA_SET_FLAG; - // set the result to final result buffer in case of super table query if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); From b28dde67ee79a85aba7f1fb03116e3d6f608d424 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Aug 2020 00:57:18 +0800 Subject: [PATCH 018/111] [td-1067] update the tablemeta life time --- src/client/src/tscServer.c | 4 ++-- src/util/inc/tcache.h | 12 +----------- src/util/src/tcache.c | 4 ++-- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 172f18af94..64562ef912 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1702,7 +1702,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(pTableMetaInfo->pTableMeta == NULL); pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer); + strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); // todo handle out of memory case if (pTableMetaInfo->pTableMeta == NULL) { @@ -1906,7 +1906,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer); + tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 11121fcf3b..3d6f9705ee 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -103,7 +103,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS); /** * get data from cache @@ -113,16 +113,6 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v */ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen); -/** - * update the expire time of data in cache - * @param pCacheObj cache object - * @param key key - * @param keyLen keyLen - * @param expireTime new expire time of data - * @return - */ -//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); - /** * Add one reference count for the exist data, and assign this data for a new owner. * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore. diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index bd903c8c23..dfa982b848 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -189,12 +189,12 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return pCacheObj; } -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { return NULL; } - SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); + SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); if (pNode1 == NULL) { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); return NULL; From 3050af21b297616b9485eea124e6a437c97bc3af Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 21 Aug 2020 14:16:42 +0800 Subject: [PATCH 019/111] [td-225] add some logs. --- src/query/src/qExecutor.c | 6 ++++-- src/tsdb/src/tsdbRead.c | 20 +++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bf72f209d6..8cd6a12322 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4501,8 +4501,10 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - qDebug("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64, - pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); + qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " + "lastKey:%" PRId64, + pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, + pQuery->current->lastKey); } if (terrno != TSDB_CODE_SUCCESS) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b189c7a406..6d0966e52a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -409,7 +409,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, pCheckInfo->lastKey, pHandle->qinfo); @@ -744,6 +744,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; + tsdbDebug("%p key in mem:%"PRId64", %p", pQueryHandle, key, pQueryHandle->qinfo); + cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || @@ -755,7 +757,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); + TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; // update the last key value @@ -1170,6 +1173,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->mixBlock = true; } + tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + "end:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pQueryHandle->qinfo); + // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; int32_t pos = cur->pos; @@ -1199,6 +1207,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveDataToFront(pQueryHandle, numOfRows, numOfCols); updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); + + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, + cur->win.ekey, cur->rows, pQueryHandle->qinfo); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; @@ -1279,6 +1291,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; + cur->mixBlock = true; } } } @@ -1295,7 +1308,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey, + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pQueryHandle->qinfo); } From 3ff538c6cc71faef1dafe14990bd5d95c6f2e8c5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 10:52:00 +0800 Subject: [PATCH 020/111] [td-225] merge develop --- src/client/inc/tscLocalMerge.h | 1 - src/client/inc/tsclient.h | 1 + src/client/src/tscLocalMerge.c | 9 +++--- src/client/src/tscServer.c | 28 +++++++++++------ src/client/src/tscSql.c | 14 ++++----- src/client/src/tscSubquery.c | 56 ++++++--------------------------- src/client/src/tscUtil.c | 2 ++ src/kit/shell/src/shellEngine.c | 10 ++---- 8 files changed, 45 insertions(+), 76 deletions(-) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 397a60d140..4e579b0729 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -87,7 +87,6 @@ typedef struct SRetrieveSupport { SSqlObj * pParentSql; tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to uint32_t numOfRetry; // record the number of retry times - pthread_mutex_t queryMutex; } SRetrieveSupport; int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pDesc, diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d02bc7fbd..65a868cf40 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -323,6 +323,7 @@ typedef struct SSqlObj { SSqlRes res; uint16_t numOfSubs; struct SSqlObj **pSubs; + tsem_t subReadySem; struct SSqlObj * prev, *next; } SSqlObj; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 156bc1de9e..cabf2a6a11 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -1377,13 +1377,12 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tscResetForNextRetrieve(pRes); if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed - tscDebug("%p %s call the drop local reducer", pSql, __FUNCTION__); - tscDestroyLocalReducer(pSql); - return 0; + tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); + return pRes->code; } SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // set the data merge in progress int32_t prevStatus = @@ -1478,8 +1477,8 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * so the processing of previous group is completed. */ int32_t numOfRes = finalizeRes(pQueryInfo, pLocalReducer); + bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); tFilePage *pResBuf = pLocalReducer->pResultBuf; /* diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 05264a2665..5a7a858ba1 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -314,10 +314,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { pRes->rspLen = 0; - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { tscDebug("%p query is cancelled, code:%s", pSql, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; } if (pRes->code == TSDB_CODE_SUCCESS) { @@ -460,11 +460,10 @@ void tscKillSTableQuery(SSqlObj *pSql) { continue; } - /* - * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause - * sub-queries not correctly released and master sql object of super table query reaches an abnormal state. - */ - rpcCancelRequest(pSub->pRpcCtx); + if (pSub->pRpcCtx != NULL) { + rpcCancelRequest(pSub->pRpcCtx); + } + pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; tscQueueAsyncRes(pSub); } @@ -1443,6 +1442,17 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return code; + } + + // all subquery have completed already + if (pRes->pLocalReducer == NULL) { + sem_wait(&pSql->subReadySem); + } + pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1453,7 +1463,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { pRes->row = 0; pRes->completed = (pRes->numOfRows == 0); - int32_t code = pRes->code; + code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1bd885466c..cfcc9f0fa6 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -295,6 +295,8 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { } tsem_init(&pSql->rspSem, 0, 0); + tsem_init(&pSql->subReadySem, 0, 0); + doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); tsem_wait(&pSql->rspSem); @@ -655,16 +657,13 @@ int* taos_fetch_lengths(TAOS_RES *res) { char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { - if (res == NULL) { + SSqlObj *pSql = (SSqlObj *)res; + if (pSql == NULL || pSql->signature != pSql) { return; } - SSqlObj *pSql = (SSqlObj *)res; - SSqlCmd *pCmd = &pSql->cmd; - - if (pSql->signature != pSql) return; tscDebug("%p start to cancel query", res); - + SSqlCmd *pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { @@ -674,9 +673,8 @@ void taos_stop_query(TAOS_RES *res) { if (pSql->cmd.command < TSDB_SQL_LOCAL) { rpcCancelRequest(pSql->pRpcCtx); } - pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSql); + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; tscDebug("%p query is cancelled", res); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index dc3445680c..1aeaba650d 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1338,10 +1338,6 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState SRetrieveSupport* pSupport = pSub->param; taosTFree(pSupport->localBuffer); - - pthread_mutex_unlock(&pSupport->queryMutex); - pthread_mutex_destroy(&pSupport->queryMutex); - taosTFree(pSupport); tscFreeSqlObj(pSub); @@ -1414,13 +1410,6 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSql = pSql; trs->pFinalColModel = pModel; - pthread_mutexattr_t mutexattr; - memset(&mutexattr, 0, sizeof(pthread_mutexattr_t)); - - pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); - pthread_mutex_init(&trs->queryMutex, &mutexattr); - pthread_mutexattr_destroy(&mutexattr); - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); @@ -1461,6 +1450,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscDebug("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); tscProcessSql(pSub); } + + // set the command flag must be after the semaphore been correctly set. + pSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, 0); + } return TSDB_CODE_SUCCESS; } @@ -1469,12 +1464,8 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tscDebug("%p start to free subquery result", pSql); taos_free_result(pSql); - taosTFree(trsupport->localBuffer); - pthread_mutex_unlock(&trsupport->queryMutex); - pthread_mutex_destroy(&trsupport->queryMutex); - taosTFree(trsupport); } @@ -1498,8 +1489,6 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES pParentSql->res.code = code; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - pthread_mutex_unlock(&trsupport->queryMutex); tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); } @@ -1518,8 +1507,6 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - pthread_mutex_unlock(&trsupport->queryMutex); - tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); @@ -1602,15 +1589,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO tscFreeSubSqlObj(trsupport, pSql); // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); - - if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { - (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); - } else { // regular super table query - if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pParentSql); - } - } + tsem_post(&pParentSql->subReadySem); } static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { @@ -1682,14 +1661,9 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // only free once taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - - // set the command flag must be after the semaphore been correctly set. - pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; - if (pParentSql->res.code == TSDB_CODE_SUCCESS) { - (*pParentSql->fp)(pParentSql->param, pParentSql, 0); - } else { - tscQueueAsyncRes(pParentSql); - } + + // all subqueries are completed, retrieve from local can be proceeded. + tsem_post(&pParentSql->subReadySem); } static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { @@ -1707,9 +1681,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); - // query process and cancel query process may execute at the same time - pthread_mutex_lock(&trsupport->queryMutex); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; @@ -1783,7 +1754,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR return; } else { // continue fetch data from dnode - pthread_mutex_unlock(&trsupport->queryMutex); taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } @@ -2083,12 +2053,6 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { tsem_post(&pSql->rspSem); return; } - -// if (pSql->res.code == TSDB_CODE_SUCCESS) { -// (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); -// } else { -// tscQueueAsyncRes(pSql); -// } } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 716d48fa03..feae58090c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -388,6 +388,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); + tsem_destroy(&pSql->subReadySem); + free(pSql); } diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d82ddf8871..45a81c7b56 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -25,6 +25,8 @@ #include "taosdef.h" #include "taoserror.h" #include "tglobal.h" +#include "tsclient.h" + #include /**************** Global variables ****************/ @@ -64,11 +66,6 @@ TAOS *shellInit(SShellArguments *args) { } taos_init(); - /* - * set tsTableMetaKeepTimer = 3000ms - * means not save cache in shell - */ - tsTableMetaKeepTimer = 3000; // Connect to the database. TAOS *con = NULL; @@ -303,8 +300,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { return; } - int num_fields = taos_field_count(pSql); - if (num_fields != 0) { // select and show kinds of commands + if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { From 2ad3133465b4fac0e230d70bfb99d45f39c4b05d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 11:57:13 +0800 Subject: [PATCH 021/111] [td-225] refactor shell code. --- src/kit/shell/src/shellEngine.c | 11 +++++------ src/kit/shell/src/shellMain.c | 9 ++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 45a81c7b56..8f7f4d9324 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -284,7 +284,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); TAOS_RES* pSql = taos_query(con, command); - result = pSql; // set it into the global variable + atomic_store_ptr(&result, pSql); // set the global TAOS_RES pointer if (taos_errno(pSql)) { taos_error(pSql); @@ -295,7 +295,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } @@ -304,7 +304,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } @@ -327,7 +327,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { wordfree(&full_path); } - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); } @@ -493,7 +493,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { } while( row != NULL); result = NULL; - //taos_free_result(tres); fclose(fp); return numOfRows; @@ -798,8 +797,8 @@ void write_history() { } void taos_error(TAOS_RES *tres) { + atomic_store_ptr(&result, 0); fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); - result = NULL; taos_free_result(tres); } diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 8481f498dd..44de6641f6 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -18,11 +18,10 @@ pthread_t pid; -// TODO: IMPLEMENT INTERRUPT HANDLER. -void interruptHandler(int signum) { +void shellQueryInterruptHandler(int signum) { #ifdef LINUX - taos_stop_query(result); - result = NULL; + void* pResHandle = atomic_val_compare_exchange_64(&result, result, 0); + taos_stop_query(pResHandle); #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); @@ -86,7 +85,7 @@ int main(int argc, char* argv[]) { struct sigaction act; memset(&act, 0, sizeof(struct sigaction)); - act.sa_handler = interruptHandler; + act.sa_handler = shellQueryInterruptHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); From b16fa964d795051816337cbef8b69a3bfc407687 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 11:59:48 +0800 Subject: [PATCH 022/111] [td-225] refactor shell code. --- src/client/src/tscSql.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index cfcc9f0fa6..4c9439f0a7 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -665,6 +665,8 @@ void taos_stop_query(TAOS_RES *res) { tscDebug("%p start to cancel query", res); SSqlCmd *pCmd = &pSql->cmd; + // TODO there are multi-thread problem. + // It may have been released by the other thread already. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { tscKillSTableQuery(pSql); From f22098b6ddefd0e0bc4413f5f5adce2252a9f045 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 12:03:57 +0800 Subject: [PATCH 023/111] [td-225] refactor shell code. --- src/client/src/tscSql.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 4c9439f0a7..e243002ff6 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -667,6 +667,7 @@ void taos_stop_query(TAOS_RES *res) { // TODO there are multi-thread problem. // It may have been released by the other thread already. + // The ref count may fix this problem. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { tscKillSTableQuery(pSql); From 74fdb1c2aa9b390c81d305d1c8e4ede0bf8a03d5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 13:24:33 +0800 Subject: [PATCH 024/111] [td-225] refactor shell code to support ctrl+c. --- src/client/src/tscServer.c | 19 ++----------------- src/client/src/tscSql.c | 18 +++++++++++------- src/client/src/tscSubquery.c | 27 +++++++++++---------------- 3 files changed, 24 insertions(+), 40 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5a7a858ba1..3b380af76c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -455,35 +455,20 @@ void tscKillSTableQuery(SSqlObj *pSql) { } for (int i = 0; i < pSql->numOfSubs; ++i) { + // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; if (pSub == NULL) { continue; } + pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; if (pSub->pRpcCtx != NULL) { rpcCancelRequest(pSub->pRpcCtx); } - pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; tscQueueAsyncRes(pSub); } - /* - * 1. if the subqueries are not launched or partially launched, we need to waiting the launched - * query return to successfully free allocated resources. - * 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage, - * set the res.code, and return. - */ - const int64_t MAX_WAITING_TIME = 10000; // 10 Sec. - int64_t stime = taosGetTimestampMs(); - - while (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - taosMsleep(100); - if (taosGetTimestampMs() - stime > MAX_WAITING_TIME) { - break; - } - } - tscDebug("%p super table query cancelled", pSql); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index e243002ff6..2906e5f4e2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -669,15 +669,19 @@ void taos_stop_query(TAOS_RES *res) { // It may have been released by the other thread already. // The ref count may fix this problem. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillSTableQuery(pSql); - } - - if (pSql->cmd.command < TSDB_SQL_LOCAL) { - rpcCancelRequest(pSql->pRpcCtx); - } + // set the error code for master pSqlObj firstly pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + assert(pSql->pRpcCtx == NULL); + tscKillSTableQuery(pSql); + } else { + if (pSql->cmd.command < TSDB_SQL_LOCAL) { + rpcCancelRequest(pSql->pRpcCtx); + } + } + tscDebug("%p query is cancelled", res); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1aeaba650d..e99f5ab153 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1462,10 +1462,15 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { tscDebug("%p start to free subquery result", pSql); - + + int32_t index = trsupport->subqueryIndex; + SSqlObj *pParentSql = trsupport->pParentSql; + + assert(pSql == pParentSql->pSubs[index]); + pParentSql->pSubs[index] = NULL; + taos_free_result(pSql); taosTFree(trsupport->localBuffer); - taosTFree(trsupport); } @@ -1474,17 +1479,7 @@ static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, i static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { // set no disk space error info -#ifdef WINDOWS - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); - LocalFree(lpMsgBuf); -#else tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code)); -#endif - SSqlObj* pParentSql = trsupport->pParentSql; pParentSql->res.code = code; @@ -1548,14 +1543,14 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO */ pSql->res.numOfRows = 0; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscDebug("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pParentSql, pSql, numOfRows, subqueryIndex); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { From 8ba7387e5394152edb7bbd586920eae369ece177 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 16:00:22 +0800 Subject: [PATCH 025/111] [td-225] refactor codes. --- cmake/define.inc | 2 +- src/tsdb/src/tsdbRead.c | 68 +++++++++++++++++++---------------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 0a6e70f375..005ec29fdd 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -32,7 +32,7 @@ ENDIF () IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -pg -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6d0966e52a..690bbc2cc7 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -257,10 +257,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab for (int32_t j = 0; j < gsize; ++j) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); - STableCheckInfo info = { - .lastKey = pKeyInfo->lastKey, - .pTableObj = pKeyInfo->pTable, - }; + STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; info.tableId = ((STable*)(pKeyInfo->pTable))->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || @@ -432,7 +429,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -448,47 +445,46 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { } } - if (rmem != NULL && rimem != NULL) { - TSKEY r1 = dataRowKey(rmem); - TSKEY r2 = dataRowKey(rimem); - - if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; - } else { - if (ASCENDING_TRAVERSE(order)) { - if (r1 < r2) { - pCheckInfo->chosen = 0; - return rmem; - } else { - pCheckInfo->chosen = 1; - return rimem; - } - } else { - if (r1 < r2) { - pCheckInfo->chosen = 1; - return rimem; - } else { - pCheckInfo->chosen = 0; - return rmem; - } - } - } + if (rmem == NULL && rimem == NULL) { + return NULL; } - // at least one (rmem or rimem) is absent here - if (rmem != NULL) { + if (rmem != NULL && rimem == NULL) { pCheckInfo->chosen = 0; return rmem; } - if (rimem != NULL) { + if (rmem == NULL && rimem != NULL) { pCheckInfo->chosen = 1; return rimem; } - return NULL; + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } + } } static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { From 0fc1c8cdb502f84ce3bc091e3ea396b2499cf711 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 17:50:12 +0800 Subject: [PATCH 026/111] [td-837] check mem after malloc --- cmake/define.inc | 2 +- src/client/src/tscSubquery.c | 21 ++--- src/query/src/qExecutor.c | 160 +++++++++++++++++++++++++---------- src/tsdb/src/tsdbRead.c | 34 ++++++-- src/vnode/src/vnodeMain.c | 5 ++ 5 files changed, 161 insertions(+), 61 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 005ec29fdd..0a6e70f375 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -32,7 +32,7 @@ ENDIF () IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -pg -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e99f5ab153..ef42398001 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1502,7 +1502,7 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscDebug("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); @@ -1521,9 +1521,10 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // if failed to process sql, let following code handle the pSql if (ret == TSDB_CODE_SUCCESS) { taos_free_result(pSql); + return ret; + } else { + return ret; } - - return code; } void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1681,7 +1682,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscDebug("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, numOfRows); @@ -1692,13 +1693,13 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(numOfRows == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscDebug("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); + tscDebug("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort } @@ -1792,7 +1793,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscError("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, code); @@ -1810,12 +1811,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscWarn("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); + tscError("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort } @@ -1823,7 +1824,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } - tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + tscDebug("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 10963ea26c..c612837f5f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -113,6 +113,15 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { } } +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5 <= 1) { + return NULL; + } else { + return realloc(p, __size); + } +} + #define calloc u_calloc #define malloc u_malloc #endif @@ -430,7 +439,10 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { - createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } } pWindowResInfo->capacity = (int32_t)newCap; @@ -1465,7 +1477,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY } // set the output buffer for the selectivity + tag query -static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { SQuery* pQuery = pRuntimeEnv->pQuery; if (isSelectivityWithTagsQuery(pQuery)) { @@ -1474,6 +1486,9 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p SQLFunctionCtx *p = NULL; SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; @@ -1499,6 +1514,8 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p taosTFree(pTagCtx); } } + + return TSDB_CODE_SUCCESS; } static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { @@ -1600,7 +1617,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order resetCtxOutputBuf(pRuntimeEnv); } - setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { + goto _clean; + } qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; @@ -2232,7 +2251,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { pQuery->sdata[i] = (tFilePage *)tmp; } @@ -2263,7 +2282,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; @@ -2803,7 +2822,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { size_t size = taosArrayGetSize(pGroup); tFilePage **buffer = pQuery->sdata; - int32_t* posList = calloc(size, sizeof(int32_t)); + int32_t *posList = calloc(size, sizeof(int32_t)); STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { @@ -2861,6 +2880,10 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } char* buf = calloc(1, pRuntimeEnv->interBufSize); + if (buf == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); @@ -4307,6 +4330,10 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { int32_t offset = 0; SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); + if (pFillCol == NULL) { + return NULL; + } + for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; @@ -5433,6 +5460,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + if (pColInfo->filters == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } } for (int32_t f = 0; f < numOfFilters; ++f) { @@ -5447,6 +5478,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pColFilter->len = htobe64(pFilterMsg->len); pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + if (pColFilter->pz == 0) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + memcpy((void *)pColFilter->pz, pMsg, (size_t)pColFilter->len); pMsg += (pColFilter->len + 1); } else { @@ -5460,6 +5496,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (*pExpr == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { @@ -5546,6 +5587,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (*tagCols == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { SColumnInfo* pTagCol = (SColumnInfo*) pMsg; @@ -5562,6 +5608,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); + + if (*tagCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + + } memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } @@ -5752,6 +5804,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { } pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); + if (pQuery->pFilterInfo == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { if (pQuery->colList[i].numOfFilters > 0) { @@ -5762,6 +5817,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); + if (pFilterInfo->pFilters == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; @@ -5911,6 +5969,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQuery == NULL) { goto _cleanup_query; } + pQInfo->runtimeEnv.pQuery = pQuery; pQuery->numOfCols = numOfCols; @@ -5996,6 +6055,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pQInfo->pBuf == NULL) { + goto _cleanup; + } + int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { @@ -6010,8 +6073,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - STableId* id = TSDB_TABLEID(info->pTable); + STableId* id = TSDB_TABLEID(info->pTable); STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); if (pTableId != NULL ) { window.skey = pTableId->key; @@ -6140,35 +6203,59 @@ static void freeQInfo(SQInfo *pQInfo) { return; } - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; qDebug("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - taosTFree(pQuery->sdata[col]); - } teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { - SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; - if (pColFilter->numOfFilters > 0) { - taosTFree(pColFilter->pFilters); + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (pQuery != NULL) { + if (pQuery->sdata != NULL) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + taosTFree(pQuery->sdata[col]); + } + taosTFree(pQuery->sdata); } - } - if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; + if (pQuery->fillVal != NULL) { + taosTFree(pQuery->fillVal); + } - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(&pExprInfo->pExpr, NULL); + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; + if (pColFilter->numOfFilters > 0) { + taosTFree(pColFilter->pFilters); } } - taosTFree(pQuery->pSelectExpr); - } + if (pQuery->pSelectExpr != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SExprInfo *pExprInfo = &pQuery->pSelectExpr[i]; - if (pQuery->fillVal != NULL) { - taosTFree(pQuery->fillVal); + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(&pExprInfo->pExpr, NULL); + } + } + + taosTFree(pQuery->pSelectExpr); + } + + if (pQuery->pGroupbyExpr != NULL) { + taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); + taosTFree(pQuery->pGroupbyExpr); + } + + taosTFree(pQuery->tagColList); + taosTFree(pQuery->pFilterInfo); + + if (pQuery->colList != NULL) { + for (int32_t i = 0; i < pQuery->numOfCols; i++) { + SColumnInfo *column = pQuery->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + taosTFree(pQuery->colList); + } + + taosTFree(pQuery); } // todo refactor, extract method to destroytableDataInfo @@ -6193,24 +6280,7 @@ static void freeQInfo(SQInfo *pQInfo) { tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - if (pQuery->pGroupbyExpr != NULL) { - taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - taosTFree(pQuery->pGroupbyExpr); - } - taosTFree(pQuery->tagColList); - taosTFree(pQuery->pFilterInfo); - - if (pQuery->colList != NULL) { - for (int32_t i = 0; i < pQuery->numOfCols; i++) { - SColumnInfo* column = pQuery->colList + i; - freeColumnFilterInfo(column->filters, column->numOfFilters); - } - taosTFree(pQuery->colList); - } - - taosTFree(pQuery->sdata); - taosTFree(pQuery); pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -6786,12 +6856,16 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 60; // every 30 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + if (pQueryMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); pQueryMgmt->closed = false; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 690bbc2cc7..c89241a76b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1840,6 +1840,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return true; } else { STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + if (pSecQueryHandle == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return false; + } + pSecQueryHandle->order = TSDB_ORDER_ASC; pSecQueryHandle->window = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX}; pSecQueryHandle->pTsdb = pQueryHandle->pTsdb; @@ -1851,6 +1856,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock; if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; free(pSecQueryHandle); return false; } @@ -1862,6 +1868,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pSecQueryHandle->statis == NULL || pSecQueryHandle->pColumns == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; @@ -1869,6 +1880,12 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { colInfo.info = pCol->info; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes); + if (colInfo.pData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } + taosArrayPush(pSecQueryHandle->pColumns, &colInfo); } @@ -2280,6 +2297,10 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); + if (sa == NULL) { + taosArrayDestroy(pTableGroup); + return NULL; + } for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); @@ -2294,14 +2315,13 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosArrayPush(pTableGroup, &sa); tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { - STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); - pSupp->numOfCols = numOfOrderCols; - pSupp->pTagSchema = pTagSchema; - pSupp->pCols = pCols; + STableGroupSupporter sup = {0}; + sup.numOfCols = numOfOrderCols; + sup.pTagSchema = pTagSchema; + sup.pCols = pCols; - taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, skey, pSupp, tableGroupComparFn); - taosTFree(pSupp); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e5536bfaaf..456f2be83b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -295,6 +295,11 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); + if (pVnode->qMgmt == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } + pVnode->events = NULL; pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); From 51290fd1aa7f5eb08a146546045740b07e643653 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 19:02:58 +0800 Subject: [PATCH 027/111] [td-1067] --- src/query/inc/qExecutor.h | 2 +- src/query/src/qExecutor.c | 62 ++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b023de0dc9..7093495763 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -193,7 +193,7 @@ typedef struct SQInfo { int64_t owner; // if it is in execution void* tsdb; int32_t vgId; - STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> + STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; SArray* arrTableIdInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c612837f5f..79bbeb65a2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -124,6 +124,7 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define calloc u_calloc #define malloc u_malloc +#define realloc u_realloc #endif #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) @@ -3146,7 +3147,8 @@ static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - // update the last key in tableKeyInfo list + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); pTableKeyInfo->lastKey = pCheckInfo->lastKey; @@ -3607,12 +3609,6 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo) { cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } -#define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \ - do { \ - assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_q)) || \ - (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_q))); \ - } while (0) - /** * set output buffer for different group * @param pRuntimeEnv @@ -4316,6 +4312,34 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + + // update the query time window + pQuery->window = cond.twindow; + + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); + + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); + SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); + + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle + STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); + pCheckInfo->win.skey = pTableKeyInfo->lastKey; + pCheckInfo->win.ekey = pTableKeyInfo->lastKey; + + pCheckInfo->lastKey = pTableKeyInfo->lastKey; + + assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + } + } + + } else if (isPointInterpoQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } else { @@ -4361,6 +4385,9 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); setScanLimitationByResultBuffer(pQuery); + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + // TODO fixme changeExecuteScanOrder(pQInfo, false); code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); @@ -4512,7 +4539,17 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - CHECK_QUERY_TIME_RANGE(pQuery, *pTableQueryInfo); + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert( + ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); + } else { + assert( + ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); + } if (!pRuntimeEnv->groupbyNormalCol) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); @@ -4653,7 +4690,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (isFirstLastRowQuery(pQuery)) { assert(0); // last_row query switch to other routine to handle -// pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); } else { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } @@ -6069,6 +6105,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (p1 == NULL) { goto _cleanup; } + taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { @@ -6076,13 +6113,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, STableId* id = TSDB_TABLEID(info->pTable); STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); - if (pTableId != NULL ) { - window.skey = pTableId->key; - } else { - window.skey = pQueryMsg->window.skey; - } + window.skey = (pTableId != NULL)? pTableId->key:pQueryMsg->window.skey; void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; From e4e4c7fcd3b76c7b68cdaebf6ff468ce6a8b89cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Aug 2020 23:02:12 +0800 Subject: [PATCH 028/111] [td-1067] --- src/query/src/qExecutor.c | 37 +++++++++++-------------------------- src/tsdb/src/tsdbRead.c | 6 +----- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 79bbeb65a2..511658c3ce 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -97,7 +97,8 @@ typedef struct { #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + + if (v % 1000 <= 0) { return NULL; } else { return malloc(__size); @@ -106,7 +107,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + if (v % 1000 <= 0) { return NULL; } else { return calloc(num, __size); @@ -1887,16 +1888,12 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), - pQuery->order.order, TSDB_ORDER_DESC); - - pQuery->order.order = TSDB_ORDER_DESC; - - int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); - int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); - - pQuery->window.skey = ekey; - pQuery->window.ekey = skey; + pQuery->order.order, TSDB_ORDER_ASC); + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } return; } @@ -4316,30 +4313,18 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) // update the query time window pQuery->window = cond.twindow; - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); for(int32_t i = 0; i < numOfGroups; ++i) { SArray *group = GET_TABLEGROUP(pQInfo, i); - SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide - // the start check timestamp of tsdbQueryHandle - STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); - pCheckInfo->win.skey = pTableKeyInfo->lastKey; - pCheckInfo->win.ekey = pTableKeyInfo->lastKey; - - pCheckInfo->lastKey = pTableKeyInfo->lastKey; - - assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; } } - - } else if (isPointInterpoQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } else { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c89241a76b..f8ff25ddab 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -295,12 +295,8 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { - pCond->order = TSDB_ORDER_DESC; + pCond->order = TSDB_ORDER_ASC; pCond->twindow = changeTableGroupByLastrow(groupList); - - //descending order query, skey >= ekey - SWAP(pCond->twindow.skey, pCond->twindow.ekey, TSKEY); - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); return pQueryHandle; } From ba9ef1377df724578eb0aa87d06a9256f53ac78c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Aug 2020 15:36:06 +0800 Subject: [PATCH 029/111] [td-1151] add test cases and fix the found bugs. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 5 +- src/client/src/tscFunctionImpl.c | 21 ++- src/client/src/tscSQLParser.c | 96 +++++----- src/client/src/tscServer.c | 11 +- src/client/src/tscSubquery.c | 3 +- src/client/src/tscUtil.c | 9 +- src/common/inc/tname.h | 3 +- src/common/src/tname.c | 17 +- src/inc/taosdef.h | 2 +- src/inc/taosmsg.h | 2 +- src/query/inc/qSqlparser.h | 1 + src/query/src/qExecutor.c | 17 +- src/query/src/qParserImpl.c | 2 +- tests/script/general/parser/bug.sim | 274 ++++++++++++++++++++++++++++ 15 files changed, 376 insertions(+), 89 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 7c406d1676..afbbd1d24b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -198,7 +198,7 @@ int32_t tscValidateName(SSQLToken* pToken); void tscIncStreamExecutionCount(void* pStream); -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 65a868cf40..47f8732de9 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -195,9 +195,9 @@ typedef struct STableDataBlocks { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert/import type + uint32_t type; // query/insert type char slidingTimeUnit; - STimeWindow window; + STimeWindow window; // query time window int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info @@ -216,6 +216,7 @@ typedef struct SQueryInfo { char * msg; // pointer to the pCmd->payload to keep error message temporarily int64_t clauseLimit; // limit for current sub clause int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX } SQueryInfo; typedef struct { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index e494446ce5..b5c17e57b9 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2902,12 +2902,12 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { - if (pCtx->numOfParams == 1) { // the number of output rows should not affect the final number of rows, so set it to be 1 - INC_INIT_VAL(pCtx, 1); + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + SET_VAL(pCtx, pCtx->size, 1); char* output = pCtx->aOutputBuf; for(int32_t i = 0; i < pCtx->size; ++i) { - tVariantDump(&pCtx->param[0], output, pCtx->outputType, true); + tVariantDump(&pCtx->param[1], output, pCtx->outputType, true); output += pCtx->outputBytes; } @@ -2936,11 +2936,16 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - - INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - + + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + SET_VAL(pCtx, pCtx->size, 1); + tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->outputType, true); + } else { + INC_INIT_VAL(pCtx, 1); + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + } + pCtx->aOutputBuf += pCtx->inputBytes; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 204c91bc52..e43b0d16d0 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1094,18 +1094,6 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } -static void extractColumnNameFromString(tSQLExprItem* pItem) { - if (pItem->pNode->nSQLOptr == TK_STRING) { - pItem->pNode->val.nLen = strdequote(pItem->pNode->val.pz); - pItem->pNode->nSQLOptr = TK_ID; - - SSQLToken* pIdToken = &pItem->pNode->colInfo; - pIdToken->type = TK_ID; - pIdToken->z = pItem->pNode->val.pz; - pIdToken->n = pItem->pNode->val.nLen; - } -} - static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { const char* msg1 = "invalid column name, or illegal column type"; const char* msg2 = "invalid arithmetic expression in select clause"; @@ -1234,6 +1222,11 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t return TSDB_CODE_SUCCESS; } +static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnListInsert(pQueryInfo->colList, &tsCol); +} + int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { assert(pSelection != NULL && pCmd != NULL); @@ -1254,15 +1247,12 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // project on all fields int32_t optr = pItem->pNode->nSQLOptr; - if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || (optr == TK_INTEGER || optr == TK_FLOAT)) { + if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - // if the name of column is quoted, remove it and set the right information for later process - extractColumnNameFromString(pItem); - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -1292,6 +1282,13 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } + // there is only one user-defined column in the final result field, add the timestamp column. + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + SColumnIndex index = {0}; + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + } + if (!functionCompatibleCheck(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1431,17 +1428,11 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnListInsert(pQueryInfo->colList, &tsCol); -} - int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; - - int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); int32_t optr = pItem->pNode->nSQLOptr; if (optr == TK_ALL) { // project on all fields @@ -1465,40 +1456,43 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t // add the primary timestamp column even though it is not required by user tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); - } else if (optr == TK_ID || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query + } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; // user-specified constant value as a new result column - if ((optr == TK_INTEGER || optr == TK_FLOAT) || (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { - index.columnIndex = TSDB_UD_COLUMN_INDEX; - index.tableIndex = 0; + index.columnIndex = (pQueryInfo->udColumnId--); + index.tableIndex = 0; - SSchema colSchema = tGetUserSpecifiedColumnSchema(pItem->pNode->val.pz, pItem->pNode->val.nType, pItem->aliasName); - SSqlExpr* pExpr = tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); - pExpr->numOfParams = 1; - tVariantAssign(&pExpr->param[0], &pItem->pNode->val); + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, pItem->aliasName); + SSqlExpr* pExpr = + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); - } else { // columns from the queried table - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + // NOTE: the first parameter is reserved for the tag column id during join query process. + pExpr->numOfParams = 2; + tVariantAssign(&pExpr->param[1], &pItem->pNode->val); + } else if (optr == TK_ID) { + SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = tGetTableNameColumnSchema(); - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); - } else { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } - - addProjectQueryCol(pQueryInfo, startPos, &index, pItem); - } - - // add the primary timestamp column even though it is not required by user - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + SSchema colSchema = tGetTableNameColumnSchema(); + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); + } else { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + } + + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2069,7 +2063,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo refactor static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && columnIndex >= -2 && tableIndex >= 0); + assert(num == 1 && tableIndex >= 0); SColumnList columnList = {0}; columnList.num = num; @@ -4706,7 +4700,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu } } - if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF || + if ((pExpr->functionId == TSDB_FUNC_PRJ && pExpr->numOfParams == 0) || pExpr->functionId == TSDB_FUNC_DIFF || pExpr->functionId == TSDB_FUNC_ARITHM) { isProjectionFunction = true; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3b380af76c..b0c66a22cd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -628,9 +628,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - - if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); + + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + tscError("%p illegal value of numOfCols in query msg: %"PRIu64", table cols:%d", pSql, numOfSrcCols, + tscGetNumOfColumns(pTableMeta)); + return TSDB_CODE_TSC_INVALID_SQL; } @@ -728,7 +731,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return TSDB_CODE_TSC_INVALID_SQL; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index ef42398001..c50b663e02 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -353,11 +353,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr = tscSqlExprGet(pQueryInfo, 0); } - // set the join condition tag column info, to do extract method + // set the join condition tag column info, todo extract method if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pQueryInfo->tagCond.joinInfo.hasJoin); int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + // set the tag column id for executor to extract correct tag value pExpr->param[0].i64Key = colId; pExpr->numOfParams = 1; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index feae58090c..d8f030938e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -937,8 +937,8 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; - } else if (pColIndex->columnIndex == TSDB_UD_COLUMN_INDEX) { - pExpr->colInfo.colId = TSDB_UD_COLUMN_INDEX; + } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { + pExpr->colInfo.colId = pColIndex->columnIndex; } else { if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); @@ -1290,12 +1290,12 @@ void tscIncStreamExecutionCount(void* pStream) { ps->num += 1; } -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) { if (pTableMetaInfo->pTableMeta == NULL) { return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_UD_COLUMN_INDEX) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -1511,6 +1511,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 41d6fe7280..65f3d8c3f9 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -4,6 +4,7 @@ #include "os.h" #include "taosmsg.h" #include "tstoken.h" +#include "tvariant.h" typedef struct SDataStatis { int16_t colId; @@ -28,7 +29,7 @@ void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); SSchema tGetTableNameColumnSchema(); -SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name); +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name); bool tscValidateTableNameLength(size_t len); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 6aa308cb45..e59d20b9ce 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -4,6 +4,7 @@ #include "tname.h" #include "tstoken.h" #include "ttokendef.h" +#include "tvariant.h" // todo refactor UNUSED_FUNC static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { @@ -47,19 +48,23 @@ SSchema tGetTableNameColumnSchema() { return s; } -SSchema tGetUserSpecifiedColumnSchema(const char* v, int16_t type, const char* name) { +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name) { SSchema s = {0}; - s.type = type; + s.type = pVal->nType; if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { - size_t len = strlen(v); - s.bytes = len + VARSTR_HEADER_SIZE; + s.bytes = pVal->nLen + VARSTR_HEADER_SIZE; } else { - s.bytes = tDataTypeDesc[type].nSize; + s.bytes = tDataTypeDesc[pVal->nType].nSize; } s.colId = TSDB_UD_COLUMN_INDEX; - tstrncpy(s.name, name, sizeof(s.name)); + if (name != NULL) { + tstrncpy(s.name, name, sizeof(s.name)); + } else { + tVariantToString(pVal, s.name); + strdequote(s.name); + } return s; } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 12c4699665..d3c744b684 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -287,7 +287,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_REPLICA 5 #define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-2) +#define TSDB_UD_COLUMN_INDEX (-100) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 7a3fc6b050..54b302760f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -169,7 +169,7 @@ enum _mgmt_table { #define TSDB_COL_NORMAL 0x0u // the normal column of the table #define TSDB_COL_TAG 0x1u // the tag column type -#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column extern char *taosMsg[]; diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index cd9618b223..2cfee344aa 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -210,6 +210,7 @@ typedef struct tSQLExprItem { char * aliasName; // alias name, null-terminated string } tSQLExprItem; +// todo refactor by using SArray typedef struct tSQLExprList { int32_t nExpr; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 511658c3ce..205d7aab7d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -168,7 +168,7 @@ static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); - bool doFilterData(SQuery *pQuery, int32_t elemPos) { +bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; @@ -899,7 +899,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } assert(dataBlock != NULL); - sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/; // start from the offset + sas->data[i] = dataBlock; // start from the offset } } else { // other type of query function @@ -1583,7 +1583,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int16_t type = pSqlFuncMsg->arg[j].argType; int16_t bytes = pSqlFuncMsg->arg[j].argBytes; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg->argValue.pz, bytes, type); + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); } @@ -5728,11 +5728,12 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * SSchema s = tGetTableNameColumnSchema(); type = s.type; bytes = s.bytes; - } else if (pExprs[i].base.colInfo.colId == TSDB_UD_COLUMN_INDEX) { + } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX) { + // it is a user-defined constant value column assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); - type = pExprs[i].base.arg[0].argType; - bytes = pExprs[i].base.arg[0].argBytes; + type = pExprs[i].base.arg[1].argType; + bytes = pExprs[i].base.arg[1].argBytes; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { bytes += VARSTR_HEADER_SIZE; @@ -5927,8 +5928,8 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } assert(f < pQuery->numOfCols); - } else if (pColIndex->colId == TSDB_UD_COLUMN_INDEX) { - // do nothing + } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { + // do nothing for user-defined constant value result columns } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index d8f12da642..0537726ad5 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -78,7 +78,7 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken pList = calloc(1, sizeof(tSQLExprList)); } - if (pList->nAlloc <= pList->nExpr) { // + if (pList->nAlloc <= pList->nExpr) { pList->nAlloc = (pList->nAlloc << 1) + 4; pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); if (pList->a == 0) { diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/bug.sim index 2a46ad1fd6..889fe83042 100644 --- a/tests/script/general/parser/bug.sim +++ b/tests/script/general/parser/bug.sim @@ -40,4 +40,278 @@ sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql +print ==============select with user-defined columns +sql select 'abc' as f, ts,f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select 'abc', ts, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select 'abc' from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 'abc' as f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select 1 as f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select 1 as f, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1.123 as f, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.123000000 then + print expect 1.123000000 , actual:$data00 + return -1 +endi + +sql select 1, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1.2391, f1 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +print ===================== user-defined columns with agg functions +sql select 1 as t, count(*) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1, sum(f1) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1,2,3, sum(f1)*99, 4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +if $data02 != 3 then + return -1 +endi + +if $data03 != 99.000000000 then + print expect 99.000000000, actual:$data03 + return -1 +endi + +sql select sum(f1)*avg(f1)+12, 1,2,3,4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 13.000000000 then + print expect 13.000000000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +sql select 1.2987, f1, 'k' from t1 where f1=1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.298700000 then + print expect 1.298700000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != @k@ then + return -1 +endi + +print ====================user-defined columns with union +sql select f1, 'f1' from t1 union all select f1, 'f1' from t1; +if $rows != 2 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != @f1@ then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data11 != @f1@ then + return -1 +endi + +print =====================udc with join +sql select st1.ts, st1.f1, st2.f2, 'abc', 1.9827 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts +if $rows != 1 then + return -1 +endi + +if $data00 != @19-12-09 16:27:35.000@ then + print expect @19-12-09 16:27:35.000@ actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +if $data03 != @abc@ then + return -1 +endi + +if $data04 != 1.982700000 then + print expect 1.982700000 actual:$data04 + return -1 +endi + +print ======================udc with interval +sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; +if $rows != 1 then + return -1 +endi + +print ======================udc with tags +sql select t1,'abc',tbname from st1 +if $rows != 1 then + return -1 +endi + +print ======================udc with arithmetic +sql select 1+1 from t1 +if $rows != 1 then + return -1 +endi + +sql_error select from t1 +sql_error select abc from t1 +sql_error select abc as tu from t1 + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5b5edf00264bf313ea6bec535614fad7634058c9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Aug 2020 16:13:31 +0800 Subject: [PATCH 030/111] [td-1010] add check for max interval time range. --- src/client/src/tscSQLParser.c | 2 +- src/inc/taoserror.h | 1 + src/query/inc/tsqlfunction.h | 8 ++++---- src/query/src/qExecutor.c | 5 +++++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e43b0d16d0..995a64ebe8 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6144,7 +6144,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (pQueryInfo->intervalTime > 0) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { + if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_INTERVAL_TIME_WINDOW) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 57c2b322fa..e58c24b531 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -220,6 +220,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag condit TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index a443a43a23..65ab82883b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -91,16 +91,16 @@ extern "C" { #define QUERY_COND_REL_PREFIX_IN "IN|" #define QUERY_COND_REL_PREFIX_LIKE "LIKE|" -#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_IN_LEN 3 #define QUERY_COND_REL_PREFIX_LIKE_LEN 5 -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -#define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 -#define TOP_BOTTOM_QUERY_LIMIT 100 +#define MAX_INTERVAL_TIME_WINDOW 10000000 +#define TOP_BOTTOM_QUERY_LIMIT 100 enum { MASTER_SCAN = 0x0u, diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 205d7aab7d..5baa3f2bf8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -455,6 +455,11 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } + // too many time window in query + if (pWindowResInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); } From dac0a985c2bd9aaf723ec70a4520fd9e9fac7b3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Aug 2020 22:50:16 +0800 Subject: [PATCH 031/111] [td-1213] --- src/client/inc/tscUtil.h | 4 +- src/client/src/tscParseInsert.c | 28 ++-- src/client/src/tscPrepare.c | 2 +- src/client/src/tscSQLParser.c | 139 +++++++++--------- src/client/src/tscServer.c | 8 +- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 6 +- src/common/inc/tname.h | 4 +- src/common/inc/tvariant.h | 2 +- src/common/src/tname.c | 8 +- src/common/src/tvariant.c | 14 +- src/query/inc/qAst.h | 2 +- src/query/inc/qSqlparser.h | 84 +++++------ src/query/inc/sql.y | 30 ++-- src/query/src/qAst.c | 10 +- src/query/src/qParserImpl.c | 94 ++++++------ src/query/src/qTokenizer.c | 4 +- src/query/src/sql.c | 10 +- src/query/tests/unitTest.cpp | 8 +- src/util/inc/tstoken.h | 8 +- .../general/parser/{bug.sim => constCol.sim} | 0 tests/script/general/parser/testSuite.sim | 2 +- 22 files changed, 239 insertions(+), 230 deletions(-) rename tests/script/general/parser/{bug.sim => constCol.sim} (100%) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index afbbd1d24b..42959ade3c 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -141,7 +141,7 @@ bool tscQueryTags(SQueryInfo* pQueryInfo); SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); @@ -194,7 +194,7 @@ SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); -int32_t tscValidateName(SSQLToken* pToken); +int32_t tscValidateName(SStrToken* pToken); void tscIncStreamExecutionCount(void* pStream); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cb49bd80b7..939ebb6abe 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -40,7 +40,7 @@ enum { static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); -static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { +static int32_t tscToInteger(SStrToken *pToken, int64_t *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -73,7 +73,7 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { return pToken->type; } -static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { +static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -89,9 +89,9 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { return pToken->type; } -int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { +int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int64_t interval; int64_t useconds = 0; char * pTokenEnd = *next; @@ -128,7 +128,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 * time expression: * e.g., now+12a, now-5h */ - SSQLToken valueToken; + SStrToken valueToken; index = 0; sToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); pTokenEnd += index; @@ -163,7 +163,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } -int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, char *msg, char **str, bool primaryKey, +int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec) { int64_t iv; int32_t numType; @@ -409,7 +409,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken = {0}; + SStrToken sToken = {0}; char * payload = pDataBlocks->pData + pDataBlocks->size; // 1. set the parsed value from sql string @@ -524,7 +524,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, SParsedDataColInfo *spd, char *error, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int16_t numOfRows = 0; @@ -734,8 +734,8 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { int32_t index = 0; - SSQLToken sToken = {0}; - SSQLToken tableToken = {0}; + SStrToken sToken = {0}; + SStrToken tableToken = {0}; int32_t code = TSDB_CODE_SUCCESS; const int32_t TABLE_INDEX = 0; @@ -993,7 +993,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } -int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { +int validateTableName(char *tblName, int len, SStrToken* psTblToken) { tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); psTblToken->n = len; @@ -1057,7 +1057,7 @@ int tsParseInsertSql(SSqlObj *pSql) { while (1) { int32_t index = 0; - SSQLToken sToken = tStrGetToken(str, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(str, &index, false, 0, NULL); // no data in the sql string anymore. if (sToken.n == 0) { @@ -1083,7 +1083,7 @@ int tsParseInsertSql(SSqlObj *pSql) { pCmd->curSql = sToken.z; char buf[TSDB_TABLE_FNAME_LEN]; - SSQLToken sTblToken; + SStrToken sTblToken; sTblToken.z = buf; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { @@ -1285,7 +1285,7 @@ int tsInsertInitialCheck(SSqlObj *pSql) { int32_t index = 0; SSqlCmd *pCmd = &pSql->cmd; - SSQLToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); pCmd->count = 0; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index b996dd958a..620e8ea57a 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -155,7 +155,7 @@ static int normalStmtPrepare(STscStmt* stmt) { uint32_t i = 0, start = 0; while (sql[i] != 0) { - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(sql + i, &token.type); if (token.type == TK_QUESTION) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 995a64ebe8..bc2b2672d3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -51,12 +51,12 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name); -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken); -static bool hasSpecifyDB(SSQLToken* pTableName); +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken); +static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd); -static int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* len); +static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); @@ -100,11 +100,11 @@ static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); -static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); -static int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); @@ -151,7 +151,7 @@ static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo return TSDB_CODE_SUCCESS; } -static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) { +static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { const char* msg1 = "password can not be empty"; const char* msg2 = "name or password too long"; const char* msg3 = "password needs single quote marks enclosed"; @@ -206,7 +206,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; - SSQLToken* pzName = &pInfo->pDCLInfo->a[0]; + SStrToken* pzName = &pInfo->pDCLInfo->a[0]; if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -241,7 +241,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_USE_DB: { const char* msg = "invalid db name"; - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); @@ -296,7 +296,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0]; + SStrToken* pIpAddr = &pInfo->pDCLInfo->a[0]; pIpAddr->n = strdequote(pIpAddr->z); break; } @@ -307,8 +307,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; - SSQLToken* pName = &pInfo->pDCLInfo->user.user; - SSQLToken* pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken* pName = &pInfo->pDCLInfo->user.user; + SStrToken* pPwd = &pInfo->pDCLInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -337,7 +337,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_DESCRIBE_TABLE: { - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg2 = "table name is too long"; const char* msg1 = "invalid table name"; @@ -400,8 +400,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { // tDCLSQL* pDCL = pInfo->pDCLInfo; SUserInfo* pUser = &pInfo->pDCLInfo->user; - SSQLToken* pName = &pUser->user; - SSQLToken* pPwd = &pUser->passwd; + SStrToken* pName = &pUser->user; + SStrToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -423,7 +423,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); - SSQLToken* pPrivilege = &pUser->privilege; + SStrToken* pPrivilege = &pUser->privilege; if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { pCmd->count = 1; @@ -581,7 +581,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // interval is not null - SSQLToken* t = &pQuerySql->interval; + SStrToken* t = &pQuerySql->interval; if (getTimestampInUsFromStr(t->z, t->n, &pQueryInfo->intervalTime) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -667,7 +667,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SSQLToken* pSliding = &pQuerySql->sliding; + SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n != 0) { getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { @@ -692,7 +692,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -709,7 +709,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableNa // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); } else { // get current DB name first, then set it into path - SSQLToken t = {0}; + SStrToken t = {0}; getCurrentDBName(pSql, &t); code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); @@ -1027,13 +1027,13 @@ static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken) { +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken) { pDBToken->z = pSql->pTscObj->db; pDBToken->n = (uint32_t)strlen(pSql->pTscObj->db); } /* length limitation, strstr cannot be applied */ -static bool hasSpecifyDB(SSQLToken* pTableName) { +static bool hasSpecifyDB(SStrToken* pTableName) { for (uint32_t i = 0; i < pTableName->n; ++i) { if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { return true; @@ -1043,7 +1043,7 @@ static bool hasSpecifyDB(SSQLToken* pTableName) { return false; } -int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* xlen) { +int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { int32_t totalLen = 0; if (account != NULL) { @@ -1463,7 +1463,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t index.columnIndex = (pQueryInfo->udColumnId--); index.tableIndex = 0; - SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, pItem->aliasName); + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); SSqlExpr* pExpr = tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); @@ -1489,6 +1489,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } // add the primary timestamp column even though it is not required by user @@ -1579,7 +1580,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - SSQLToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; + SStrToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; if (pToken->z == NULL || pToken->n == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1588,7 +1589,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* // check if the table name is valid or not - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1786,7 +1787,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -2088,16 +2089,16 @@ void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, c snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); } -static bool isTablenameToken(SSQLToken* token) { - SSQLToken tmpToken = *token; - SSQLToken tableToken = {0}; +static bool isTablenameToken(SStrToken* token) { + SStrToken tmpToken = *token; + SStrToken tableToken = {0}; extractTableNameFromToken(&tmpToken, &tableToken); return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken* pToken) { +static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); @@ -2119,7 +2120,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken return columnIndex; } -int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { const char* msg0 = "ambiguous column name"; const char* msg1 = "invalid column name"; @@ -2161,7 +2162,7 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQu } } -int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pTableToken->n == 0) { // only one table and no table name prefix in column name if (pQueryInfo->numOfTables == 1) { pIndex->tableIndex = 0; @@ -2187,8 +2188,8 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum return TSDB_CODE_SUCCESS; } -int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SSQLToken tableToken = {0}; +int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { @@ -2198,12 +2199,12 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn return TSDB_CODE_SUCCESS; } -int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken tmpToken = *pToken; + SStrToken tmpToken = *pToken; if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2315,7 +2316,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { // db prefix in tagCond, show table conds in payload - SSQLToken* pDbPrefixToken = &pShowInfo->prefix; + SStrToken* pDbPrefixToken = &pShowInfo->prefix; if (pDbPrefixToken->type != 0) { if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long @@ -2337,7 +2338,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show table/stable like 'xxxx', set the like pattern for show tables - SSQLToken* pPattern = &pShowInfo->pattern; + SStrToken* pPattern = &pShowInfo->pattern; if (pPattern->type != 0) { pPattern->n = strdequote(pPattern->z); @@ -2355,7 +2356,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show vnodes may be ip addr of dnode in payload - SSQLToken* pDnodeIp = &pShowInfo->prefix; + SStrToken* pDnodeIp = &pShowInfo->prefix; if (pDnodeIp->n >= TSDB_IPv4ADDR_LEN) { // ip addr is too long return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2376,7 +2377,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SSqlCmd* pCmd = &pSql->cmd; pCmd->command = pInfo->type; - SSQLToken* idStr = &(pInfo->pDCLInfo->ip); + SStrToken* idStr = &(pInfo->pDCLInfo->ip); if (idStr->n > TSDB_KILL_MSG_LEN) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2597,7 +2598,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; - SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &token, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -3623,7 +3624,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* return; } - SSQLToken t = {0}; + SStrToken t = {0}; extractTableNameFromToken(&pLeft->colInfo, &t); *pOut = *pExpr; @@ -3703,7 +3704,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, num = j; char* name = extractDBName(pTableMetaInfo->name, db); - SSQLToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; + SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; for (int32_t i = 0; i < num; ++i) { if (i >= 1) { @@ -3712,7 +3713,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; int32_t xlen = (int32_t)strlen(segments[i]); - SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; + SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { @@ -4033,7 +4034,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t return TSDB_CODE_TSC_INVALID_SQL; } } else { - SSQLToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; + SStrToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; int32_t len = tSQLGetToken(pRight->val.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { @@ -4279,7 +4280,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } - SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query @@ -4348,7 +4349,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu } tVariant* pVar2 = &pSortorder->a[1].pVar; - SSQLToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; + SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4477,7 +4478,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; + SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4515,12 +4516,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } - SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; + SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); } @@ -4544,7 +4545,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t numOfTags = tscGetNumOfTags(pTableMeta); SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; + SStrToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -4637,7 +4638,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pItem = &pAlterSQL->varList->a[0]; SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; + SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } @@ -4757,7 +4758,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, }; - SSQLToken* pOptionToken = &pOptions->a[1]; + SStrToken* pOptionToken = &pOptions->a[1]; if (pOptions->nTokens == 2) { // reset log and reset query cache does not need value @@ -4769,7 +4770,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t vnodeId = 0; int32_t dnodeId = 0; strdequote(pValToken->z); @@ -4780,14 +4781,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } return TSDB_CODE_SUCCESS; } else { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { @@ -4816,7 +4817,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - SSQLToken* pOptionToken = &pOptions->a[0]; + SStrToken* pOptionToken = &pOptions->a[0]; if (pOptions->nTokens == 1) { // reset log does not need value @@ -4827,7 +4828,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { } } } else { - SSQLToken* pValToken = &pOptions->a[1]; + SStrToken* pValToken = &pOptions->a[1]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 131 || val > 199) { @@ -4852,7 +4853,7 @@ int32_t validateColumnName(char* name) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken token = {.z = name}; + SStrToken token = {.z = name}; token.n = tSQLGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { @@ -5031,7 +5032,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SSQLToken* pToken = &pCreateDbInfo->precision; + SStrToken* pToken = &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -5702,7 +5703,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p assert(pFieldList != NULL); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5757,7 +5758,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst - SSQLToken* pToken = &(pCreateTable->usingInfo.stableName); + SStrToken* pToken = &(pCreateTable->usingInfo.stableName); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5850,7 +5851,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); SQuerySQL* pQuerySql = pCreateTable->pSelect; if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { @@ -5863,7 +5864,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } tVariant* pVar = &pSrcMeterName->a[0].pVar; - SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -6004,7 +6005,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pTableItem->nLen = strdequote(pTableItem->pz); - SSQLToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; + SStrToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -6015,13 +6016,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); - SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; + SStrToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar; - SSQLToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; + SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); } @@ -6243,7 +6244,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SSQLToken t = {.type = pSqlExpr->nSQLOptr}; + SStrToken t = {.type = pSqlExpr->nSQLOptr}; (*pExpr)->_node.optr = getBinaryExprOptr(&t); assert((*pExpr)->_node.optr != 0); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b0c66a22cd..e33bfe44d1 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -931,8 +931,8 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload; - SSQLToken *pName = &pInfo->pDCLInfo->user.user; - SSQLToken *pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken *pName = &pInfo->pDCLInfo->user.user; + SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; strncpy(pAlterMsg->user, pName->z, pName->n); strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); @@ -1134,13 +1134,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SSQLToken *pPattern = &pShowInfo->pattern; + SStrToken *pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query strncpy(pShowMsg->payload, pPattern->z, pPattern->n); pShowMsg->payloadLen = htons(pPattern->n); } } else { - SSQLToken *pEpAddr = &pShowInfo->prefix; + SStrToken *pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 2906e5f4e2..1a774e8784 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -855,7 +855,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t str = nextStr + 1; len = (int32_t)strtrim(tblName); - SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName}; + SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; tSQLGetToken(tblName, &sToken.type); // Check if the table name available or not diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d8f030938e..788283a1ad 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -744,7 +744,7 @@ bool tscIsInsertData(char* sqlstr) { int32_t index = 0; do { - SSQLToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); + SStrToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); if (t0.type != TK_LP) { return t0.type == TK_INSERT || t0.type == TK_IMPORT; } @@ -1187,7 +1187,7 @@ void tscColumnListDestroy(SArray* pColumnList) { * 'first_part.second_part' * */ -static int32_t validateQuoteToken(SSQLToken* pToken) { +static int32_t validateQuoteToken(SStrToken* pToken) { strdequote(pToken->z); pToken->n = (uint32_t)strtrim(pToken->z); @@ -1203,7 +1203,7 @@ static int32_t validateQuoteToken(SSQLToken* pToken) { return TSDB_CODE_SUCCESS; } -int32_t tscValidateName(SSQLToken* pToken) { +int32_t tscValidateName(SStrToken* pToken) { if (pToken->type != TK_STRING && pToken->type != TK_ID) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 65f3d8c3f9..2a4ac3fc40 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -25,11 +25,11 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); -void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); +void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); SSchema tGetTableNameColumnSchema(); -SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name); +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); bool tscValidateTableNameLength(size_t len); diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 211635b4af..e9973bcb95 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -36,7 +36,7 @@ typedef struct tVariant { }; } tVariant; -void tVariantCreate(tVariant *pVar, SSQLToken *token); +void tVariantCreate(tVariant *pVar, SStrToken *token); void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index e59d20b9ce..2eda5953a3 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -48,7 +48,7 @@ SSchema tGetTableNameColumnSchema() { return s; } -SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name) { +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) { SSchema s = {0}; s.type = pVal->nType; @@ -62,8 +62,8 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, const char* name) { if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - tVariantToString(pVal, s.name); - strdequote(s.name); + size_t len = MIN(sizeof(s.name), exprStr->n + 1); + tstrncpy(s.name, exprStr->z, len); } return s; @@ -132,7 +132,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in * tablePrefix.columnName * extract table name and save it in pTable, with only column name in pToken */ -void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) { +void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) { const char sep = TS_PATH_DELIMITER[0]; if (pToken == pTable || pToken == NULL || pTable == NULL) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 6e8111aa72..4036e578b4 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -23,7 +23,7 @@ #include "tutil.h" // todo support scientific expression number and oct number -void tVariantCreate(tVariant *pVar, SSQLToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } +void tVariantCreate(tVariant *pVar, SStrToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { memset(pVar, 0, sizeof(tVariant)); @@ -270,7 +270,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -319,7 +319,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -478,7 +478,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { - SSQLToken stoken = {.z = pStr, .n = len}; + SStrToken stoken = {.z = pStr, .n = len}; if (TK_ILLEGAL == isValidNumber(&stoken)) { return -1; @@ -504,7 +504,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -521,7 +521,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result return 0; } - SSQLToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; + SStrToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; if (TK_ILLEGAL == isValidNumber(&sToken)) { return -1; } @@ -557,7 +557,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index ec568a6cdb..547616dee6 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -80,7 +80,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); -uint8_t getBinaryExprOptr(SSQLToken *pToken); +uint8_t getBinaryExprOptr(SStrToken *pToken); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 2cfee344aa..69fc0bc7ef 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -26,7 +26,7 @@ extern "C" { #include "tstoken.h" #include "tvariant.h" -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken extern char tTokenTypeSwitcher[13]; #define toTSDBType(x) \ @@ -71,16 +71,16 @@ typedef struct SQuerySQL { struct tSQLExpr * pWhere; // where clause [optional] tVariantList * pGroupby; // groupby clause, only for tags[optional] tVariantList * pSortOrder; // orderby [optional] - SSQLToken interval; // interval [optional] - SSQLToken sliding; // sliding window [optional] + SStrToken interval; // interval [optional] + SStrToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] SLimitVal slimit; // group limit offset [optional] tVariantList * fillType; // fill type[optional] - SSQLToken selectToken; // sql string + SStrToken selectToken; // sql string } SQuerySQL; typedef struct SCreateTableSQL { - struct SSQLToken name; // meter name, create table [meterName] xxx + struct SStrToken name; // meter name, create table [meterName] xxx bool existCheck; int8_t type; // create normal table/from super table/ stream @@ -90,7 +90,7 @@ typedef struct SCreateTableSQL { } colInfo; struct { - SSQLToken stableName; // super table name, for using clause + SStrToken stableName; // super table name, for using clause tVariantList *pTagVals; // create by using metric, tag value STagData tagdata; } usingInfo; @@ -99,7 +99,7 @@ typedef struct SCreateTableSQL { } SCreateTableSQL; typedef struct SAlterTableSQL { - SSQLToken name; + SStrToken name; int16_t type; STagData tagData; @@ -108,7 +108,7 @@ typedef struct SAlterTableSQL { } SAlterTableSQL; typedef struct SCreateDBInfo { - SSQLToken dbname; + SStrToken dbname; int32_t replica; int32_t cacheBlockSize; int32_t maxTablesPerVnode; @@ -121,7 +121,7 @@ typedef struct SCreateDBInfo { int32_t walLevel; int32_t quorum; int32_t compressionLevel; - SSQLToken precision; + SStrToken precision; bool ignoreExists; tVariantList *keep; @@ -136,33 +136,33 @@ typedef struct SCreateAcctSQL { int64_t maxStorage; int64_t maxQueryTime; int32_t maxConnections; - SSQLToken stat; + SStrToken stat; } SCreateAcctSQL; typedef struct SShowInfo { uint8_t showType; - SSQLToken prefix; - SSQLToken pattern; + SStrToken prefix; + SStrToken pattern; } SShowInfo; typedef struct SUserInfo { - SSQLToken user; - SSQLToken passwd; - SSQLToken privilege; + SStrToken user; + SStrToken passwd; + SStrToken privilege; int16_t type; } SUserInfo; typedef struct tDCLSQL { int32_t nTokens; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ - SSQLToken *a; /* one entry for element */ + SStrToken *a; /* one entry for element */ bool existsCheck; union { SCreateDBInfo dbOpt; SCreateAcctSQL acctOpt; SShowInfo showOpt; - SSQLToken ip; + SStrToken ip; }; SUserInfo user; @@ -194,14 +194,14 @@ typedef struct tSQLExpr { // the full sql string of function(col, param), which is actually the raw // field name, since the function name is kept in nSQLOptr already - SSQLToken operand; - SSQLToken colInfo; // field id + SStrToken operand; + SStrToken colInfo; // field id tVariant val; // value only for string, float, int struct tSQLExpr *pLeft; // left child struct tSQLExpr *pRight; // right child - struct tSQLExprList *pParam; // function parameters + SStrToken token; // original sql expr string } tSQLExpr; // used in select clause. select from xxx @@ -243,7 +243,7 @@ tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t so tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder); +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder); void tVariantListDestroy(tVariantList *pList); tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField); @@ -254,61 +254,61 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType); void tSQLExprDestroy(tSQLExpr *); -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken); +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); void tSQLExprListDestroy(tSQLExprList *pList); -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pMetricName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type); void tSQLExprNodeDestroy(tSQLExpr *pExpr); tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr); -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); tSQLExprListList *tSQLListListAppend(tSQLExprListList *pList, tSQLExprList *pExprList); void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); -SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type); +SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists); void SQLInfoDestroy(SSqlInfo *pInfo); void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck); -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns); +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken); +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists); +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo); -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd); -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip); -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege); +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip); +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); // prefix show db.tables; -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB); +void setDBName(SStrToken *pCpxName, SStrToken *pDB); -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pToken, int32_t optType); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optType); -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType); +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType); +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *pToken); +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken); void *ParseAlloc(void *(*mallocProc)(size_t)); diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index edf59327ab..7500dcba56 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -2,8 +2,8 @@ //usage: lemon sql.y %token_prefix TK_ -%token_type {SSQLToken} -%default_type {SSQLToken} +%token_type {SStrToken} +%default_type {SStrToken} %extra_argument {SSqlInfo* pInfo} %fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. @@ -80,11 +80,11 @@ cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } cmd ::= SHOW VNODES IPTOKEN(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } -%type dbPrefix {SSQLToken} +%type dbPrefix {SStrToken} dbPrefix(A) ::=. {A.n = 0; A.type = 0;} dbPrefix(A) ::= ids(X) DOT. {A = X; } -%type cpxName {SSQLToken} +%type cpxName {SStrToken} cpxName(A) ::= . {A.n = 0; } cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } @@ -101,19 +101,19 @@ cmd ::= SHOW dbPrefix(X) STABLES. { } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } @@ -145,7 +145,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} @@ -153,15 +153,15 @@ cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(p // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. // And "ids" is an identifer-or-string. -%type ids {SSQLToken} +%type ids {SStrToken} ids(A) ::= ID(X). {A = X; } ids(A) ::= STRING(X). {A = X; } -%type ifexists {SSQLToken} +%type ifexists {SStrToken} ifexists(X) ::= IF EXISTS. {X.n = 1;} ifexists(X) ::= . {X.n = 0;} -%type ifnotexists {SSQLToken} +%type ifnotexists {SStrToken} ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} ifnotexists(X) ::= . {X.n = 0;} @@ -416,7 +416,7 @@ selcollist(A) ::= sclp(P) STAR. { // An option "AS " phrase that can follow one of the expressions that // define the result set, or one of the tables in the FROM clause. // -%type as {SSQLToken} +%type as {SStrToken} as(X) ::= AS ids(Y). { X = Y; } as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } @@ -458,10 +458,10 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SSQLToken} +%type tmvar {SStrToken} tmvar(A) ::= VARIABLE(X). {A = X;} -%type interval_opt {SSQLToken} +%type interval_opt {SStrToken} interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N = E; } interval_opt(N) ::= . {N.n = 0; N.z = NULL; N.type = 0; } @@ -482,7 +482,7 @@ fill_opt(N) ::= FILL LP ID(Y) RP. { N = tVariantListAppendToken(NULL, &Y, -1); } -%type sliding_opt {SSQLToken} +%type sliding_opt {SStrToken} sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index afa3618a96..634f014d97 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -44,7 +44,7 @@ * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14 * */ -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken); static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); static void destroySyntaxTree(tExprNode *); @@ -103,7 +103,7 @@ static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, u } } -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken) { +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) { /* if the token is not a value, return false */ if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID && pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) { @@ -117,7 +117,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken int32_t i = 0; if (pToken->type == TK_ID) { do { - SSQLToken tableToken = {0}; + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); size_t len = strlen(pSchema[i].name); @@ -157,7 +157,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken return pNode; } -uint8_t getBinaryExprOptr(SSQLToken *pToken) { +uint8_t getBinaryExprOptr(SStrToken *pToken) { switch (pToken->type) { case TK_LT: return TSDB_RELATION_LESS; @@ -234,7 +234,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef } static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); + SStrToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 0537726ad5..0b5408f793 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -33,7 +33,7 @@ SSqlInfo qSQLParse(const char *pStr) { int32_t i = 0; while (1) { - SSQLToken t0 = {0}; + SStrToken t0 = {0}; if (pStr[i] == 0) { Parse(pParser, 0, t0, &sqlInfo); @@ -73,7 +73,7 @@ abort_parse: return sqlInfo; } -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken) { +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { if (pList == NULL) { pList = calloc(1, sizeof(tSQLExprList)); } @@ -117,41 +117,45 @@ void tSQLExprListDestroy(tSQLExprList *pList) { free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pAliasToken, int32_t optrType) { - tSQLExpr *nodePtr = calloc(1, sizeof(tSQLExpr)); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { + tSQLExpr *pSQLExpr = calloc(1, sizeof(tSQLExpr)); + + if (pToken != NULL) { + pSQLExpr->token = *pToken; + } if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pAliasToken->type); + toTSDBType(pToken->type); - tVariantCreate(&nodePtr->val, pAliasToken); - nodePtr->nSQLOptr = optrType; + tVariantCreate(&pSQLExpr->val, pToken); + pSQLExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // default use microsecond - nodePtr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = getTimestampInUsFromStr(pAliasToken->z, pAliasToken->n, &nodePtr->val.i64Key); + int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &pSQLExpr->val.i64Key); UNUSED(ret); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; } else { // it must be the column name (tk_id) if it is not the number assert(optrType == TK_ID || optrType == TK_ALL); - if (pAliasToken != NULL) { - nodePtr->colInfo = *pAliasToken; + if (pToken != NULL) { + pSQLExpr->colInfo = *pToken; } - nodePtr->nSQLOptr = optrType; + pSQLExpr->nSQLOptr = optrType; } - return nodePtr; + return pSQLExpr; } /* * pList is the parameters for function with id(optType) * function name is denoted by pFunctionToken */ -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType) { +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { if (pFuncToken == NULL) return NULL; tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); @@ -173,8 +177,12 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || - optrType == TK_REM) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = endPos - pExpr->token.z; + pExpr->token.type = pLeft->token.type; + + if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { /* * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. @@ -373,7 +381,7 @@ void tVariantListDestroy(tVariantList *pList) { free(pList); } -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder) { +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) { if (pList == NULL) { pList = calloc(1, sizeof(tVariantList)); } @@ -382,9 +390,9 @@ tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToke return pList; } - if (pAliasToken) { + if (pToken) { tVariant t = {0}; - tVariantCreate(&t, pAliasToken); + tVariantCreate(&t, pToken); tVariantListItem *pItem = &pList->a[pList->nExpr++]; memcpy(pItem, &t, sizeof(tVariant)); @@ -420,7 +428,7 @@ void tFieldListDestroy(tFieldList *pList) { free(pList); } -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB) { +void setDBName(SStrToken *pCpxName, SStrToken *pDB) { pCpxName->type = pDB->type; pCpxName->z = pDB->z; pCpxName->n = pDB->n; @@ -464,7 +472,7 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { return 0; } -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); // truncate the column name @@ -479,7 +487,7 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) pField->bytes = pType->bytes; } -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = -1; int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar. @@ -517,9 +525,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL)); @@ -611,7 +619,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { taosTFree(pClause->pClause); } -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pStableName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) { SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); @@ -644,7 +652,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLTo return pCreate; } -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); pAlterTable->name = *pMeterName; @@ -716,7 +724,7 @@ SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { return pSubclause; } -SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type) { +SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type) { pInfo->type = type; if (type == TSDB_SQL_SELECT) { @@ -745,7 +753,7 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists) { +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists) { pInfo->pCreateTableInfo->name = *pMeterName; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } @@ -760,7 +768,7 @@ void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { } } -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken) { +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken) { if (pToken == NULL) return NULL; if (pTokenList == NULL) pTokenList = calloc(1, sizeof(tDCLSQL)); @@ -781,19 +789,19 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { va_start(va, nParam); while (nParam-- > 0) { - SSQLToken *pToken = va_arg(va, SSQLToken *); + SStrToken *pToken = va_arg(va, SStrToken *); pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); } va_end(va); } -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck) { +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { pInfo->type = type; pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); } -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns) { +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); } @@ -816,7 +824,7 @@ void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* } } -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists) { +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -827,7 +835,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -843,7 +851,7 @@ void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken } } -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->type = TSDB_SQL_CREATE_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -855,7 +863,7 @@ void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { pInfo->pDCLInfo->user.passwd = *pPasswd; } -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege) { +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { pInfo->type = TSDB_SQL_ALTER_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -880,7 +888,7 @@ void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* } } -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip) { +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -909,5 +917,5 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; - memset(&pDBInfo->precision, 0, sizeof(SSQLToken)); + memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 6fb63c4a20..5fdabca166 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -580,8 +580,8 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return 0; } -SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { - SSQLToken t0 = {0}; +SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { + SStrToken t0 = {0}; // here we reach the end of sql string, null-terminated string if (str[*i] == 0) { diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 0b977977af..f3f55791b0 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -99,7 +99,7 @@ #define YYCODETYPE unsigned short int #define YYNOCODE 274 #define YYACTIONTYPE unsigned short int -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; @@ -2074,21 +2074,21 @@ static void yy_reduce( break; case 22: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; case 23: /* cmd ::= SHOW dbPrefix VGROUPS */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; case 24: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } @@ -2139,7 +2139,7 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index b59e0783a2..b6e1170d0e 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -16,7 +16,7 @@ namespace { int32_t testValidateName(char* name) { - SSQLToken token = {0}; + SStrToken token = {0}; token.z = name; token.n = strlen(name); token.type = 0; @@ -720,8 +720,8 @@ TEST(testCase, tGetToken_Test) { EXPECT_FALSE(type == TK_HEX); } -static SSQLToken createStrToken(char* s) { - SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; +static SStrToken createStrToken(char* s) { + SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; t.type = TK_STRING; t.z = s; t.n = strlen(s); @@ -730,7 +730,7 @@ static SSQLToken createStrToken(char* s) { } TEST(testCase, isValidNumber_test) { - SSQLToken t1 = createStrToken("123abc"); + SStrToken t1 = createStrToken("123abc"); EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index c1c6f2de7a..7aeb2da6b6 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -28,11 +28,11 @@ extern "C" { #define TSQL_TBNAME_L "tbname" // used to denote the minimum unite in sql parsing -typedef struct SSQLToken { +typedef struct SStrToken { uint32_t n; uint32_t type; char * z; -} SSQLToken; +} SStrToken; /** * tokenizer for sql string @@ -52,7 +52,7 @@ uint32_t tSQLGetToken(char *z, uint32_t *tokenType); * @param ignoreTokenTypes * @return */ -SSQLToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); +SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); /** * check if it is a keyword or not @@ -76,7 +76,7 @@ bool isKeyWord(const char *z, int32_t len); * @param pToken * @return token type, if it is not a number, TK_ILLEGAL will return */ -static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) { +static FORCE_INLINE int32_t isValidNumber(const SStrToken* pToken) { const char* z = pToken->z; int32_t type = TK_ILLEGAL; diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/constCol.sim similarity index 100% rename from tests/script/general/parser/bug.sim rename to tests/script/general/parser/constCol.sim diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index aafba2d328..4e26d14cfd 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -97,7 +97,7 @@ run general/parser/topbot.sim sleep 2000 run general/parser/union.sim sleep 2000 -run general/parser/bug.sim +run general/parser/constCol.sim sleep 2000 run general/parser/sliding.sim From 1b3e5d0f0c9c7c857a4666fcbbdc51ab05fb7f7f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 15:03:19 +0000 Subject: [PATCH 032/111] TD-1216 update normal table uid --- src/mnode/src/mnodeTable.c | 5 +- tests/test/c/CMakeLists.txt | 11 +- tests/test/c/createNormalTable.c | 218 +++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 tests/test/c/createNormalTable.c diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index dbc1bffa5c..8506501194 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1714,7 +1714,8 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->superTable = pMsg->pSTable; } else { - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t)pTable->vgId) << 40) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 16) + + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); pTable->sqlLen = htons(pCreate->sqlLen); @@ -1744,7 +1745,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); pTable->sql[pTable->sqlLen - 1] = 0; - mDebug("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mInfo("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->sqlLen, pTable->sql); } } diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index cc8bc78e33..9d11895104 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -10,8 +10,8 @@ IF (TD_LINUX) #add_executable(insertPerTable insertPerTable.c) #target_link_libraries(insertPerTable taos_static pthread) - add_executable(insertPerRow insertPerRow.c) - target_link_libraries(insertPerRow taos_static pthread) + #add_executable(insertPerRow insertPerRow.c) + #target_link_libraries(insertPerRow taos_static pthread) #add_executable(importOneRow importOneRow.c) #target_link_libraries(importOneRow taos_static pthread) @@ -22,6 +22,9 @@ IF (TD_LINUX) #add_executable(hashPerformance hashPerformance.c) #target_link_libraries(hashPerformance taos_static tutil common pthread) - add_executable(createTablePerformance createTablePerformance.c) - target_link_libraries(createTablePerformance taos_static tutil common pthread) + #add_executable(createTablePerformance createTablePerformance.c) + #target_link_libraries(createTablePerformance taos_static tutil common pthread) + + add_executable(createNormalTable createNormalTable.c) + target_link_libraries(createNormalTable taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/createNormalTable.c b/tests/test/c/createNormalTable.c new file mode 100644 index 0000000000..18a648b9e1 --- /dev/null +++ b/tests/test/c/createNormalTable.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "tglobal.h" +#include "hash.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +char dbName[32] = "db"; +char stableName[64] = "st"; +int32_t numOfThreads = 30; +int32_t numOfTables = 100000; +int32_t replica = 1; +int32_t numOfColumns = 2; + +typedef struct { + int32_t tableBeginIndex; + int32_t tableEndIndex; + int32_t threadIndex; + char dbName[32]; + char stableName[64]; + float createTableSpeed; + pthread_t thread; +} SThreadInfo; + +void shellParseArgument(int argc, char *argv[]); +void *threadFunc(void *param); +void createDbAndSTable(); + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + taos_init(); + createDbAndSTable(); + + pPrint("%d threads are spawned to create table", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + + int32_t numOfTablesPerThread = numOfTables / numOfThreads; + numOfTables = numOfTablesPerThread * numOfThreads; + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = i * numOfTablesPerThread; + pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].threadIndex = i; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stableName, stableName); + pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + pPrint("%s total speed:%.1f tables/second, threads:%d %s", GREEN, createTableSpeed, numOfThreads, NC); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void createDbAndSTable() { + pPrint("start to create db and stable"); + char qstr[64000]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s replica %d", dbName, replica); + TAOS_RES *pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + sprintf(qstr, "use %s", dbName); + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + taos_close(con); +} + +void *threadFunc(void *param) { + SThreadInfo *pInfo = (SThreadInfo *)param; + char qstr[65000]; + int code; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "use %s", pInfo->dbName); + TAOS_RES *pSql = taos_query(con, qstr); + taos_free_result(pSql); + + int64_t startMs = taosGetTimestampMs(); + + for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t); + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code)); + } + taos_free_result(pSql); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + int64_t endMs = taosGetTimestampMs(); + int32_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + float seconds = (endMs - startMs) / 1000.0; + float speed = totalTables / seconds; + pInfo->createTableSpeed = speed; + + pPrint("thread:%d, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, seconds, speed); + taos_close(con); + + return 0; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "replica, default is ", replica); + printf("%s%s\n", indent, "-columns"); + printf("%s%s%s%d\n", indent, indent, "numOfColumns, default is ", numOfColumns); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stableName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + replica = atoi(argv[++i]); + } else if (strcmp(argv[i], "-columns") == 0) { + numOfColumns = atoi(argv[++i]); + } else { + } + } + + pPrint("%s dbName:%s %s", GREEN, dbName, NC); + pPrint("%s stableName:%s %s", GREEN, stableName, NC); + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s numOfTables:%d %s", GREEN, numOfTables, NC); + pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); + pPrint("%s numOfColumns:%d %s", GREEN, numOfColumns, NC); + pPrint("%s replica:%d %s", GREEN, replica, NC); + + pPrint("%s start create table performace test %s", GREEN, NC); +} From 99715c7b52274021d498db9ed1b7b41368364c0d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 15:06:47 +0000 Subject: [PATCH 033/111] TD-1217 let monitor database can be dropped --- src/mnode/src/mnodeDb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 26b4430777..0367ce21fd 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1059,10 +1059,12 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { } } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("db:%s, can't drop monitor database", pDrop->db); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif int32_t code = mnodeSetDbDropping(pMsg->pDb); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { From 3fb64ca0e21547b79751bf6cfe475d479a61a5d0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 24 Aug 2020 15:11:29 +0000 Subject: [PATCH 034/111] minor changes --- src/mnode/src/mnodeTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 8506501194..794958f7f0 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1745,7 +1745,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen); pTable->sql[pTable->sqlLen - 1] = 0; - mInfo("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + mDebug("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pTable->sqlLen, pTable->sql); } } From a7f58f94378ffdaa34c9bdd026e70eb73dca26df Mon Sep 17 00:00:00 2001 From: Amos Kong Date: Tue, 25 Aug 2020 07:17:09 +0800 Subject: [PATCH 035/111] install.sh: create empty history file after installation There is a error message in first use of taos, because the history file doesn't exist. This patch tried to create an empty history file after installation, then users won't see the error message. | [root@045a9307c53a ~]# taos | | Welcome to the TDengine shell from Linux, Client Version:2.0.1.1 | Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. | | Failed to open file /root/.taos_history | taos> Signed-off-by: Amos Kong --- packaging/tools/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 64de09df6d..ee500a727b 100644 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -713,6 +713,7 @@ function install_TDengine() { echo echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" fi + touch ~/.taos_history rm -rf $(tar -tf taos.tar.gz) } From a3e77f23fa943a56a57ed52c00725831b54c32ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 10:56:53 +0800 Subject: [PATCH 036/111] [td-1213] --- src/client/src/tscSQLParser.c | 36 ++++++++++------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index bc2b2672d3..3b91b381ef 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1111,42 +1111,31 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } int32_t tableIndex = columnList.ids[0].tableIndex; - - // todo potential data overflow - char* arithmeticExprStr = malloc(1024*1024); - char* p = arithmeticExprStr; - if (arithmeticType == NORMAL_ARITHMETIC) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; // all columns in arithmetic expression must belong to the same table for (int32_t f = 1; f < columnList.num; ++f) { if (columnList.ids[f].tableIndex != tableIndex) { - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } } - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double), false); - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); + char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; + size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1); + tstrncpy(pExpr->aliasName, name, len); tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); if (ret != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); @@ -1157,7 +1146,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t SColIndex* pIndex = taosArrayGet(colList, k); if (TSDB_COL_IS_TAG(pIndex->flag)) { tExprTreeDestroy(&pNode, NULL); - taosTFree(arithmeticExprStr); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1174,7 +1162,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t // TODO: other error handling } END_TRY - size_t len = tbufTell(&bw); + len = tbufTell(&bw); char* c = tbufGetData(&bw, true); // set the serialized binary string as the parameter of arithmetic expression @@ -1185,16 +1173,14 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); } else { - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, name, NULL); + char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; + size_t len = MIN(sizeof(pItem->aliasName), pItem->pNode->token.n + 1); + tstrncpy(pItem->aliasName, name, len); + + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pItem->aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); @@ -1210,7 +1196,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } @@ -1218,7 +1203,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } } - taosTFree(arithmeticExprStr); return TSDB_CODE_SUCCESS; } @@ -3170,7 +3154,7 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { +static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { char* start = *str; int32_t code = doArithmeticExprToString(pExpr, str); From 77816db719215dae11b7d6d1f75bd071c2abf7bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 10:57:35 +0800 Subject: [PATCH 037/111] [td-225] --- src/inc/taoserror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index e58c24b531..eda5fd0f3c 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -220,7 +220,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag condit TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") From c43cca3cb4ee785a9814899479bb0c34ae2df5f5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 25 Aug 2020 11:35:56 +0800 Subject: [PATCH 038/111] minor changes --- tests/script/general/user/monitor.sim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim index 208b85f16c..eb543612f5 100644 --- a/tests/script/general/user/monitor.sim +++ b/tests/script/general/user/monitor.sim @@ -11,9 +11,9 @@ sleep 3000 sql connect print ========== step2 -sql drop database log -x step21 - return -1 -step21: +#sql drop database log -x step21 +# return -1 +#step21: sql drop table log.dn -x step22 return -1 step22: From a8417e7b6e162fd3ab84f3bf0740a6c046229e2c Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 13:48:32 +0800 Subject: [PATCH 039/111] [TD-1203] --- packaging/release.sh | 7 +- packaging/tools/install.sh | 74 ++++---- packaging/tools/install_arbi.sh | 297 ++++++++++++++++++++++++++++++++ packaging/tools/makearbi.sh | 75 ++++++++ packaging/tools/remove.sh | 12 +- packaging/tools/remove_arbi.sh | 129 ++++++++++++++ 6 files changed, 545 insertions(+), 49 deletions(-) mode change 100644 => 100755 packaging/tools/install.sh create mode 100755 packaging/tools/install_arbi.sh create mode 100755 packaging/tools/makearbi.sh create mode 100755 packaging/tools/remove_arbi.sh diff --git a/packaging/release.sh b/packaging/release.sh index 3ba1ba72b2..fe46c97928 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -80,9 +80,9 @@ fi versioninfo="${top_dir}/src/util/src/version.c" csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" -fi +#if command -v sudo > /dev/null; then +# csudo="sudo" +#fi function is_valid_version() { [ -z $1 ] && return 1 || : @@ -267,6 +267,7 @@ if [ "$osType" != "Darwin" ]; then ${csudo} ./makepkg.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makearbi.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else cd ${script_dir}/tools ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh old mode 100644 new mode 100755 index 64de09df6d..f38677f7ac --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -76,7 +76,7 @@ fi # get the operating system type for using the corresponding init file # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -if [[ -d /etc/os-release ]]; then +if [[ -e /etc/os-release ]]; then osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: else osinfo="" @@ -381,34 +381,29 @@ function install_service_on_sysvinit() { function clean_service_on_systemd() { taosd_service_config="${service_config_dir}/taosd.service" - if systemctl is-active --quiet taosd; then echo "TDengine is running, stopping it..." ${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${taosd_service_config} - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - if systemctl is-active --quiet tarbitratord; then - echo "tarbitrator is running, stopping it..." - ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null - - ${csudo} rm -f ${tarbitratord_service_config} fi } @@ -418,7 +413,6 @@ function install_service_on_systemd() { clean_service_on_systemd taosd_service_config="${service_config_dir}/taosd.service" - ${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" ${csudo} bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" @@ -439,32 +433,30 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" ${csudo} systemctl enable taosd - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + if [ "$verMode" == "cluster" ]; then - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - - ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - #${csudo} systemctl enable tarbitratord - nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}" diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh new file mode 100755 index 0000000000..662d2dfae1 --- /dev/null +++ b/packaging/tools/install_arbi.sh @@ -0,0 +1,297 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/taos" + +# old bin dir +bin_dir="/usr/local/taos/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install taosd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +# taos:2345:respawn:/etc/init.d/tarbitratord start + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop taosd + kill_tarbitrator + fi +} + +function update_TDengine() { + # Start to update + echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" +} + +function install_TDengine() { + # Start to install + echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_TDengine +else + install_TDengine +fi + diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh new file mode 100755 index 0000000000..bc6179eff2 --- /dev/null +++ b/packaging/tools/makearbi.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TDengine-enterprise-arbitrator" +else + install_dir="${release_dir}/TDengine-arbitrator" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" +install_files="${script_dir}/install_arbi.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index c775f98b83..dd6fe65eb7 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -69,10 +69,12 @@ function kill_tarbitrator() { } function clean_bin() { # Remove link - ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosd || : - ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/taos || : + ${csudo} rm -f ${bin_link_dir}/taosd || : + ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : } function clean_lib() { @@ -207,7 +209,7 @@ ${csudo} rm -rf ${data_link_dir} || : ${csudo} rm -rf ${install_main_dir} ${csudo} rm -rf ${install_nginxd_dir} -if [[ -d /etc/os-release ]]; then +if [[ -e /etc/os-release ]]; then osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) else osinfo="" diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh new file mode 100755 index 0000000000..127da68fcd --- /dev/null +++ b/packaging/tools/remove_arbi.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/taos" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TDengine's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}" From 37da0e660f55312a4c5190216af4c60b616fd8b0 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 14:13:02 +0800 Subject: [PATCH 040/111] [TD-1203] --- src/inc/taosdef.h | 1 + src/sync/src/tarbitrator.c | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 3dea8da18a..ff5b9a958f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -395,6 +395,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_PORT_DNODEDNODE 5 #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 #define TAOS_QTYPE_RPC 0 #define TAOS_QTYPE_FWD 1 diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 3538391a94..625c0d6838 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -40,12 +40,16 @@ typedef struct { void *pConn; } SNodeConn; +uint16_t tsArbitratorPort = 0; + int main(int argc, char *argv[]) { char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; + tsArbitratorPort = tsServerPort + TSDB_PORT_ARBITRATOR; + for (int i=1; i Date: Tue, 25 Aug 2020 06:30:30 +0000 Subject: [PATCH 041/111] TD-1201 minor changes --- src/vnode/src/vnodeMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3c9da7fd23..c723c766e1 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -549,7 +549,7 @@ static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { static void vnodeNotifyRole(void *ahandle, int8_t role) { SVnodeObj *pVnode = ahandle; - vInfo("vgId:%d, sync role changed from %d to %d", pVnode->vgId, pVnode->role, role); + vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; dnodeSendStatusMsgToMnode(); From 50b51f15002491ebd9df9f51ccd90f4c1984ebc5 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 14:31:15 +0800 Subject: [PATCH 042/111] [TD-1203] --- packaging/tools/install_arbi.sh | 4 ++-- packaging/tools/remove_arbi.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index 662d2dfae1..a89d2257dc 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -13,10 +13,10 @@ bin_link_dir="/usr/bin" #inc_link_dir="/usr/include" #install main path -install_main_dir="/usr/local/taos" +install_main_dir="/usr/local/tarbitrator" # old bin dir -bin_dir="/usr/local/taos/bin" +bin_dir="/usr/local/tarbitrator/bin" service_config_dir="/etc/systemd/system" diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh index 127da68fcd..68fd9275fb 100755 --- a/packaging/tools/remove_arbi.sh +++ b/packaging/tools/remove_arbi.sh @@ -12,7 +12,7 @@ GREEN='\033[1;32m' NC='\033[0m' #install main path -install_main_dir="/usr/local/taos" +install_main_dir="/usr/local/tarbitrator" bin_link_dir="/usr/bin" #inc_link_dir="/usr/include" From b3b3108e5bebc7c5e1d295bf8270746df7d3bead Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 14:53:10 +0800 Subject: [PATCH 043/111] [TD-1203] --- packaging/cfg/taos.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 93a6936da3..a91dde8796 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -24,7 +24,7 @@ # dataDir /var/lib/taos # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only -# arbitrator arbitrator_hostname:6030 +# arbitrator arbitrator_hostname:6042 # number of threads per CPU core # numOfThreadsPerCore 1.0 From 24d33d02436f72de2ba2a13f083465a189ae2a0f Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 14:56:43 +0800 Subject: [PATCH 044/111] [TD-1203] --- documentation20/webdocs/markdowndocs/administrator-ch.md | 2 +- documentation20/webdocs/markdowndocs/cluster-ch.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 97f9f9e839..c5ecb862e3 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -94,7 +94,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 -**注意：**对于端口，TDengine会使用从serverPort起12个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6041共12个端口，而且必须TCP和UDP都打开。 +**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 31d70e73e1..097433a18a 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -10,7 +10,7 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预 **第一步**：如果搭建集群的节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) -**第二步**：建议关闭防火墙，至少保证端口：6030 - 6041的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； +**第二步**：建议关闭防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； **第三步**：在所有节点安装TDengine，且版本必须是一致的，**但不要启动taosd**； @@ -33,7 +33,7 @@ fqdn h1.taosdata.com serverPort 6030 // 副本数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分 -arbitrator ha.taosdata.com:6030 +arbitrator ha.taosdata.com:6042 ``` 一定要修改的参数是firstEp, 其他参数可不做任何修改，除非你很清楚为什么要修改。 @@ -169,5 +169,5 @@ SHOW MNODES; 如果副本数为偶数，当一个vnode group里一半或超过一半的vnode不工作时，是无法从中选出master的。同理，一半或超过一半的mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 -TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6030。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 +TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 From 6cca3cdcb929e1c27385250bf5b1c65d086b411f Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 15:00:40 +0800 Subject: [PATCH 045/111] [TD-1121] --- src/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index e9d7a71477..eb6fcde385 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -54,7 +54,7 @@ int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string -int32_t tsEnableCoreFile = 1; +int32_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; /* From 2271ac926014683f6eef1c95817827521fc0d622 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 15:53:39 +0800 Subject: [PATCH 046/111] [td-1227] --- src/mnode/src/mnodeProfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 01a824baa7..85457d7a26 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -35,8 +35,8 @@ #include "mnodeVgroup.h" #include "mnodeWrite.h" -#define CONN_KEEP_TIME (tsShellActivityTimer * 3000) -#define CONN_CHECK_TIME (tsShellActivityTimer * 2000) +#define CONN_KEEP_TIME (tsShellActivityTimer * 3) +#define CONN_CHECK_TIME (tsShellActivityTimer * 2) #define QUERY_ID_SIZE 20 #define QUERY_STREAM_SAVE_SIZE 20 From 73fcf1e1ce2c5e4d2eebe8bc7681c41e374d652f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 15:56:51 +0800 Subject: [PATCH 047/111] [td-1151] --- src/client/inc/tsclient.h | 39 +++++++++++++++------------ src/client/src/tscFunctionImpl.c | 45 +++++++++++++------------------- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 47f8732de9..aecbdd8207 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -433,31 +433,36 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField int32_t bytes = pInfo->pSqlExpr->resBytes; char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t realLen = varDataLen(pData); - assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + // user defined constant value output columns + if (pInfo->pSqlExpr->colInfo.flag == TSDB_COL_UDC) { + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + pData = pInfo->pSqlExpr->param[1].pz; + pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; + pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : pData; } else { - pRes->tsrow[columnIndex] = ((tstr*)pData)->data; - } + assert(bytes == tDataTypeDesc[type].nSize); - if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor - *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : &pInfo->pSqlExpr->param[1].i64Key; + pRes->length[columnIndex] = bytes; } - - pRes->length[columnIndex] = realLen; } else { - assert(bytes == tDataTypeDesc[type].nSize); + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : ((tstr *)pData)->data; + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor + *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + } + + pRes->length[columnIndex] = realLen; } else { - pRes->tsrow[columnIndex] = pData; - } + assert(bytes == tDataTypeDesc[type].nSize); - pRes->length[columnIndex] = bytes; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : pData; + pRes->length[columnIndex] = bytes; + } } } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index b5c17e57b9..a0ef40d367 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2902,27 +2902,20 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { - if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 - SET_VAL(pCtx, pCtx->size, 1); + // the number of output rows should not affect the final number of rows, so set it to be 0 + if (pCtx->numOfParams == 2) { + return; + } - char* output = pCtx->aOutputBuf; - for(int32_t i = 0; i < pCtx->size; ++i) { - tVariantDump(&pCtx->param[1], output, pCtx->outputType, true); - output += pCtx->outputBytes; - } + INC_INIT_VAL(pCtx, pCtx->size); + char *pData = GET_INPUT_CHAR(pCtx); + if (pCtx->order == TSDB_ORDER_ASC) { + memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { - - INC_INIT_VAL(pCtx, pCtx->size); - - char *pData = GET_INPUT_CHAR(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); - } else { - for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); - } + for(int32_t i = 0; i < pCtx->size; ++i) { + memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, + pCtx->inputBytes); } } @@ -2931,20 +2924,18 @@ static void col_project_function(SQLFunctionCtx *pCtx) { static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + return; + } + // only one output if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 - SET_VAL(pCtx, pCtx->size, 1); - tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->outputType, true); - } else { - INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - } + INC_INIT_VAL(pCtx, 1); + char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pCtx->aOutputBuf += pCtx->inputBytes; } From dffb6511e5c83194151fbef644d95877e5089a24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Aug 2020 16:04:00 +0800 Subject: [PATCH 048/111] [td-1227] --- src/mnode/src/mnodeProfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 01a824baa7..85457d7a26 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -35,8 +35,8 @@ #include "mnodeVgroup.h" #include "mnodeWrite.h" -#define CONN_KEEP_TIME (tsShellActivityTimer * 3000) -#define CONN_CHECK_TIME (tsShellActivityTimer * 2000) +#define CONN_KEEP_TIME (tsShellActivityTimer * 3) +#define CONN_CHECK_TIME (tsShellActivityTimer * 2) #define QUERY_ID_SIZE 20 #define QUERY_STREAM_SAVE_SIZE 20 From 37c11dd8dbac1cb8f408cae0b76b52874026c20e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 25 Aug 2020 16:10:23 +0800 Subject: [PATCH 049/111] TD-1090 first version of windows 32 client --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 2 +- src/client/src/tscUtil.c | 2 +- src/inc/taosdef.h | 2 +- src/inc/taosmsg.h | 5 ++++- src/sync/inc/taosTcpPool.h | 2 +- tests/tsim/inc/sim.h | 14 +++++++------- tests/tsim/inc/simParse.h | 21 ++++++++++----------- 8 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2ca6ba6691..cb285c8df1 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -108,7 +108,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff void tscDestroyDataBlock(STableDataBlocks* pDataBlock); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset); void* tscDestroyBlockArrayList(SArray* pDataBlockList); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d02bc7fbd..e117f544bf 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -234,7 +234,7 @@ typedef struct { char * curSql; // current sql, resume position of sql after parsing paused int8_t parseFinished; - short numOfCols; + int16_t numOfCols; uint32_t allocSize; char * payload; int32_t payloadLen; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 64a871ff74..fa148ebb2a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -404,7 +404,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { taosTFree(pDataBlock); } -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset) { uint32_t needed = pDataBlock->numOfParams + 1; if (needed > pDataBlock->numOfAllocedParams) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 3dea8da18a..3e598c49ac 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -63,7 +63,7 @@ typedef struct tstr { extern const int32_t TYPE_BYTES[11]; // TODO: replace and remove code below #define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(short) +#define SHORT_BYTES sizeof(int16_t) #define INT_BYTES sizeof(int) #define LONG_BYTES sizeof(int64_t) #define FLOAT_BYTES sizeof(float) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 1daa1da038..5e7da9bbdc 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -424,7 +424,10 @@ typedef struct SColumnInfo { int16_t type; int16_t bytes; int16_t numOfFilters; - SColumnFilterInfo *filters; + union{ + int64_t placeholder; + SColumnFilterInfo *filters; + }; } SColumnInfo; typedef struct STableIdInfo { diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h index 1e410acc26..5f7ca9ede5 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/taosTcpPool.h @@ -26,7 +26,7 @@ typedef void* tthread_h; typedef struct { int numOfThreads; uint32_t serverIp; - short port; + int16_t port; int bufferSize; void (*processBrokenLink)(void *ahandle); int (*processIncomingMsg)(void *ahandle, void *buffer); diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 18af21a506..dbb1a7b1f1 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -96,8 +96,8 @@ enum { struct _script_t; typedef struct _cmd_t { - short cmdno; - short nlen; + int16_t cmdno; + int16_t nlen; char name[MAX_SIM_CMD_NAME_LEN]; bool (*parseCmd)(char *, struct _cmd_t *, int); bool (*executeCmd)(struct _script_t *script, char *option); @@ -105,11 +105,11 @@ typedef struct _cmd_t { } SCommand; typedef struct { - short cmdno; - short jump; // jump position - short errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag - // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ - short lineNum; // correspodning line number in original file + int16_t cmdno; + int16_t jump; // jump position + int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag + // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ + int16_t lineNum; // correspodning line number in original file int optionOffset; // relative option offset } SCmdLine; diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index cff66e6c4f..15d912c02a 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -33,21 +33,20 @@ enum { /* label stack */ typedef struct { - char top; /* number of labels */ - short pos[MAX_NUM_LABLES]; /* the position of the label */ - char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ + int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ + char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ } SLabel; /* block definition */ typedef struct { - char top; /* the number of blocks stacked */ - char type[MAX_NUM_BLOCK]; /* the block type */ - short *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ - short back[MAX_NUM_BLOCK]; /* go back, endw and continue */ - char numJump[MAX_NUM_BLOCK]; - short *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ - char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ - char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ + char top; /* the number of blocks stacked */ + char type[MAX_NUM_BLOCK]; /* the block type */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + char numJump[MAX_NUM_BLOCK]; + int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ + char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ + char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ } SBlock; bool simParseExpression(char *token, int lineNum); From d50a9d153b80195941c03e8933aa53d2e2710023 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 25 Aug 2020 16:24:15 +0800 Subject: [PATCH 050/111] TD-1090 --- tests/tsim/inc/sim.h | 16 ++++++++-------- tests/tsim/inc/simParse.h | 11 ++++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index dbb1a7b1f1..6f3bc7099d 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -98,9 +98,9 @@ struct _script_t; typedef struct _cmd_t { int16_t cmdno; int16_t nlen; - char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int); - bool (*executeCmd)(struct _script_t *script, char *option); + char name[MAX_SIM_CMD_NAME_LEN]; + bool (*parseCmd)(char *, struct _cmd_t *, int); + bool (*executeCmd)(struct _script_t *script, char *option); struct _cmd_t *next; } SCommand; @@ -120,7 +120,7 @@ typedef struct _var_t { } SVariable; typedef struct _script_t { - int type; + int type; bool killed; void *taos; @@ -130,10 +130,10 @@ typedef struct _script_t { char system_exit_code[12]; char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - int varLen; - int linePos; // current cmd position - int numOfLines; // number of lines in the script - int bgScriptLen; + int varLen; + int linePos; // current cmd position + int numOfLines; // number of lines in the script + int bgScriptLen; char fileName[MAX_FILE_NAME_LEN]; // script file name char error[MAX_ERROR_LEN]; char *optionBuffer; diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index 15d912c02a..d3f92add71 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -33,7 +33,8 @@ enum { /* label stack */ typedef struct { - int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ + char top; /* number of labels */ + int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ } SLabel; @@ -41,12 +42,12 @@ typedef struct { typedef struct { char top; /* the number of blocks stacked */ char type[MAX_NUM_BLOCK]; /* the block type */ - int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ - int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ char numJump[MAX_NUM_BLOCK]; int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ - char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ - char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ + char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ + char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ } SBlock; bool simParseExpression(char *token, int lineNum); From 308998cd223cfd87076c0b4351e476bfe9596e6d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 25 Aug 2020 16:50:45 +0800 Subject: [PATCH 051/111] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index de47f376ac..5e247a796d 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -77,4 +77,10 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` +## 14. 怎么报告问题？ +如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: +1. /var/log/taos +2. /etc/taos +附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 +为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。 From 92bff4ed55f32f4c3872cecb2de4e85a761981e9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 25 Aug 2020 16:59:16 +0800 Subject: [PATCH 052/111] TD-1090 rename some files --- deps/MsvcLibX/CMakeLists.txt | 2 +- src/os/src/windows/CMakeLists.txt | 2 +- src/os/src/windows/{w64Atomic.c => wAtomic.c} | 0 src/os/src/windows/{w64Dir.c => wDir.c} | 0 src/os/src/windows/{w64Env.c => wEnv.c} | 0 src/os/src/windows/{w64File.c => wFile.c} | 0 src/os/src/windows/{w64Getline.c => wGetline.c} | 0 src/os/src/windows/{w64Godll.c => wGodll.c} | 0 src/os/src/windows/{w64Lz4.c => wLz4.c} | 0 src/os/src/windows/{w64Semphone.c => wSemphone.c} | 0 src/os/src/windows/{w64Socket.c => wSocket.c} | 0 src/os/src/windows/{w64String.c => wString.c} | 0 src/os/src/windows/{w64Strptime.c => wStrptime.c} | 0 src/os/src/windows/{w64Sysinfo.c => wSysinfo.c} | 0 src/os/src/windows/{w64Time.c => wTime.c} | 0 src/os/src/windows/{w64Timer.c => wTimer.c} | 0 src/os/src/windows/{w64Wordexp.c => wWordexp.c} | 0 17 files changed, 2 insertions(+), 2 deletions(-) rename src/os/src/windows/{w64Atomic.c => wAtomic.c} (100%) rename src/os/src/windows/{w64Dir.c => wDir.c} (100%) rename src/os/src/windows/{w64Env.c => wEnv.c} (100%) rename src/os/src/windows/{w64File.c => wFile.c} (100%) rename src/os/src/windows/{w64Getline.c => wGetline.c} (100%) rename src/os/src/windows/{w64Godll.c => wGodll.c} (100%) rename src/os/src/windows/{w64Lz4.c => wLz4.c} (100%) rename src/os/src/windows/{w64Semphone.c => wSemphone.c} (100%) rename src/os/src/windows/{w64Socket.c => wSocket.c} (100%) rename src/os/src/windows/{w64String.c => wString.c} (100%) rename src/os/src/windows/{w64Strptime.c => wStrptime.c} (100%) rename src/os/src/windows/{w64Sysinfo.c => wSysinfo.c} (100%) rename src/os/src/windows/{w64Time.c => wTime.c} (100%) rename src/os/src/windows/{w64Timer.c => wTimer.c} (100%) rename src/os/src/windows/{w64Wordexp.c => wWordexp.c} (100%) diff --git a/deps/MsvcLibX/CMakeLists.txt b/deps/MsvcLibX/CMakeLists.txt index fc77a3b447..4428579e1c 100644 --- a/deps/MsvcLibX/CMakeLists.txt +++ b/deps/MsvcLibX/CMakeLists.txt @@ -4,5 +4,5 @@ PROJECT(TDengine) IF (TD_WINDOWS) INCLUDE_DIRECTORIES(include) AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(MsvcLibXw64 ${SRC}) + ADD_LIBRARY(MsvcLibXw ${SRC}) ENDIF () diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt index 588d3b7f68..9dcc9e7e6d 100644 --- a/src/os/src/windows/CMakeLists.txt +++ b/src/os/src/windows/CMakeLists.txt @@ -4,4 +4,4 @@ PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) ADD_LIBRARY(os ${SRC}) -TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32 MsvcLibXw64) +TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32 MsvcLibXw) diff --git a/src/os/src/windows/w64Atomic.c b/src/os/src/windows/wAtomic.c similarity index 100% rename from src/os/src/windows/w64Atomic.c rename to src/os/src/windows/wAtomic.c diff --git a/src/os/src/windows/w64Dir.c b/src/os/src/windows/wDir.c similarity index 100% rename from src/os/src/windows/w64Dir.c rename to src/os/src/windows/wDir.c diff --git a/src/os/src/windows/w64Env.c b/src/os/src/windows/wEnv.c similarity index 100% rename from src/os/src/windows/w64Env.c rename to src/os/src/windows/wEnv.c diff --git a/src/os/src/windows/w64File.c b/src/os/src/windows/wFile.c similarity index 100% rename from src/os/src/windows/w64File.c rename to src/os/src/windows/wFile.c diff --git a/src/os/src/windows/w64Getline.c b/src/os/src/windows/wGetline.c similarity index 100% rename from src/os/src/windows/w64Getline.c rename to src/os/src/windows/wGetline.c diff --git a/src/os/src/windows/w64Godll.c b/src/os/src/windows/wGodll.c similarity index 100% rename from src/os/src/windows/w64Godll.c rename to src/os/src/windows/wGodll.c diff --git a/src/os/src/windows/w64Lz4.c b/src/os/src/windows/wLz4.c similarity index 100% rename from src/os/src/windows/w64Lz4.c rename to src/os/src/windows/wLz4.c diff --git a/src/os/src/windows/w64Semphone.c b/src/os/src/windows/wSemphone.c similarity index 100% rename from src/os/src/windows/w64Semphone.c rename to src/os/src/windows/wSemphone.c diff --git a/src/os/src/windows/w64Socket.c b/src/os/src/windows/wSocket.c similarity index 100% rename from src/os/src/windows/w64Socket.c rename to src/os/src/windows/wSocket.c diff --git a/src/os/src/windows/w64String.c b/src/os/src/windows/wString.c similarity index 100% rename from src/os/src/windows/w64String.c rename to src/os/src/windows/wString.c diff --git a/src/os/src/windows/w64Strptime.c b/src/os/src/windows/wStrptime.c similarity index 100% rename from src/os/src/windows/w64Strptime.c rename to src/os/src/windows/wStrptime.c diff --git a/src/os/src/windows/w64Sysinfo.c b/src/os/src/windows/wSysinfo.c similarity index 100% rename from src/os/src/windows/w64Sysinfo.c rename to src/os/src/windows/wSysinfo.c diff --git a/src/os/src/windows/w64Time.c b/src/os/src/windows/wTime.c similarity index 100% rename from src/os/src/windows/w64Time.c rename to src/os/src/windows/wTime.c diff --git a/src/os/src/windows/w64Timer.c b/src/os/src/windows/wTimer.c similarity index 100% rename from src/os/src/windows/w64Timer.c rename to src/os/src/windows/wTimer.c diff --git a/src/os/src/windows/w64Wordexp.c b/src/os/src/windows/wWordexp.c similarity index 100% rename from src/os/src/windows/w64Wordexp.c rename to src/os/src/windows/wWordexp.c From 6aefd4963031ab2587387f70a137a521ddf82408 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 25 Aug 2020 17:17:32 +0800 Subject: [PATCH 053/111] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 5e247a796d..928096dac3 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -83,4 +83,4 @@ Connection = DriverManager.getConnection(url, properties); 2. /etc/taos 附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 -为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。 +为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 From 2bb07eb03f7e176c802ca09f04991b2d3e061444 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 17:40:27 +0800 Subject: [PATCH 054/111] [TD-1132] --- src/kit/taosnetwork/client.c | 139 +++++++++++++++++++++++++---------- src/kit/taosnetwork/server.c | 4 +- 2 files changed, 103 insertions(+), 40 deletions(-) diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c index 102f9b9d89..a157a3a9d2 100644 --- a/src/kit/taosnetwork/client.c +++ b/src/kit/taosnetwork/client.c @@ -27,35 +27,46 @@ #include #include #include +#include -#define MAX_PKG_LEN (64*1000) -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) +#define MAX_PKG_LEN (64*1000) +#define BUFFER_SIZE (MAX_PKG_LEN + 1024) +#define TSDB_FQDN_LEN 128 +#define TSDB_IPv4ADDR_LEN 16 typedef struct { - int port; - char *host; + uint16_t port; + uint32_t hostIp; + char fqdn[TSDB_FQDN_LEN]; uint16_t pktLen; } info_s; typedef struct Arguments { - char * host; + char host[TSDB_IPv4ADDR_LEN]; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; uint16_t max_port; uint16_t pktLen; } SArguments; static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0}, + {0, 'h', "host ip", 0, "The host ip to connect to TDEngine. Default is localhost.", 0}, {0, 'p', "port", 0, "The TCP or UDP port number to use for the connection. Default is 6030.", 1}, - {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6060.", 2}, + {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6042.", 2}, + {0, 'f', "host fqdn", 0, "The host fqdn to connect to TDEngine.", 3}, {0, 'l', "test pkg len", 0, "The len of pkg for test. Default is 1000 Bytes, max not greater than 64k Bytes.\nNotes: This parameter must be consistent between the client and the server.", 3}}; static error_t parse_opt(int key, char *arg, struct argp_state *state) { - + wordexp_t full_path; SArguments *arguments = state->input; switch (key) { case 'h': - arguments->host = arg; + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid host ip %s\n", arg); + return -1; + } + strcpy(arguments->host, full_path.we_wordv[0]); + wordfree(&full_path); break; case 'p': arguments->port = atoi(arg); @@ -66,6 +77,14 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'l': arguments->pktLen = atoi(arg); break; + case 'f': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid host fqdn %s\n", arg); + return -1; + } + strcpy(arguments->fqdn, full_path.we_wordv[0]); + wordfree(&full_path); + break; default: return ARGP_ERR_UNKNOWN; @@ -76,8 +95,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { static struct argp argp = {options, parse_opt, 0, 0}; int checkTcpPort(info_s *info) { - int port = info->port; - char *host = info->host; int clientSocket; struct sockaddr_in serverAddr; @@ -89,9 +106,9 @@ int checkTcpPort(info_s *info) { return -1; } serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(port); + serverAddr.sin_port = htons(info->port); - serverAddr.sin_addr.s_addr = inet_addr(host); + serverAddr.sin_addr.s_addr = info->hostIp; //printf("=================================\n"); if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { @@ -102,7 +119,9 @@ int checkTcpPort(info_s *info) { memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); - sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", host, port); + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); send(clientSocket, sendbuf, info->pktLen, 0); @@ -120,7 +139,7 @@ int checkTcpPort(info_s *info) { if (errno == EINTR) { continue; } else { - printf("recv ack pkg from TCP port: %d fail:%s.\n", port, strerror(errno)); + printf("recv ack pkg from TCP port: %d fail:%s.\n", info->port, strerror(errno)); close(clientSocket); return -1; } @@ -132,7 +151,7 @@ int checkTcpPort(info_s *info) { } if (iDataNum < info->pktLen) { - printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, port); + printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, info->port); return -1; } //printf("Read ack pkg len:%d from tcp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); @@ -142,8 +161,6 @@ int checkTcpPort(info_s *info) { } int checkUdpPort(info_s *info) { - int port = info->port; - char *host = info->host; int clientSocket; struct sockaddr_in serverAddr; @@ -156,13 +173,15 @@ int checkUdpPort(info_s *info) { } serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = inet_addr(host); + serverAddr.sin_port = htons(info->port); + serverAddr.sin_addr.s_addr = info->hostIp; memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); - sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", host, port); + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); @@ -176,7 +195,7 @@ int checkUdpPort(info_s *info) { iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); if (iDataNum < info->pktLen) { - printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\n", iDataNum, info->pktLen, port); + printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\n", iDataNum, info->pktLen, info->port); return -1; } @@ -185,25 +204,35 @@ int checkUdpPort(info_s *info) { return 0; } -int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", 6030, 6060, 1000}; - info_s info; - int ret; - - argp_parse(&argp, argc, argv, 0, 0, &arguments); - if (arguments.pktLen > MAX_PKG_LEN) { - printf("test pkg len overflow: %d, max len not greater than %d bytes\n", arguments.pktLen, MAX_PKG_LEN); - exit(0); +int32_t getIpFromFqdn(const char *fqdn, uint32_t* ip) { + struct addrinfo hints = {0}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + struct addrinfo *result = NULL; + + int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); + if (result) { + struct sockaddr *sa = result->ai_addr; + struct sockaddr_in *si = (struct sockaddr_in*)sa; + struct in_addr ia = si->sin_addr; + *ip = ia.s_addr; + freeaddrinfo(result); + return 0; + } else { + printf("Failed get the ip address from fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); + return -1; } +} - printf("host: %s\tport: %d\tmax_port: %d\tpkgLen: %d\n", arguments.host, arguments.port, arguments.max_port, arguments.pktLen); +void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t pktLen) { + int ret; + info_s info; + memset(&info, 0, sizeof(info_s)); + info.hostIp = hostIp; + info.pktLen = pktLen; - int port = arguments.port; - - info.host = arguments.host; - info.pktLen = arguments.pktLen; - - for (; port <= arguments.max_port; port++) { + for (uint16_t port = startPort; port <= maxPort; port++) { //printf("test: %s:%d\n", info.host, port); printf("\n"); @@ -222,6 +251,40 @@ int main(int argc, char *argv[]) { printf("udp port:%d test ok.\t\t", port); } } + printf("\n"); + return ; +} + +int main(int argc, char *argv[]) { + SArguments arguments = {"127.0.0.1", "", 6030, 6042, 1000}; + int ret; + + argp_parse(&argp, argc, argv, 0, 0, &arguments); + if (arguments.pktLen > MAX_PKG_LEN) { + printf("test pkg len overflow: %d, max len not greater than %d bytes\n", arguments.pktLen, MAX_PKG_LEN); + exit(0); + } + + printf("host ip: %s\thost fqdn: %s\tport: %d\tmax_port: %d\tpkgLen: %d\n", arguments.host, arguments.fqdn, arguments.port, arguments.max_port, arguments.pktLen); + + if (arguments.host[0] != 0) { + printf("\nstart connect to %s test:\n", arguments.host); + checkPort(inet_addr(arguments.host), arguments.port, arguments.max_port, arguments.pktLen); + printf("\n"); + } + + if (arguments.fqdn[0] != 0) { + uint32_t hostIp = 0; + ret = getIpFromFqdn(arguments.fqdn, &hostIp); + if (ret) { + printf("\n"); + return 0; + } + printf("\nstart connetc to %s test:\n", arguments.fqdn); + checkPort(hostIp, arguments.port, arguments.max_port, arguments.pktLen); + printf("\n"); + } + return 0; } diff --git a/src/kit/taosnetwork/server.c b/src/kit/taosnetwork/server.c index 1c3bc6fa09..97be1d3b63 100644 --- a/src/kit/taosnetwork/server.c +++ b/src/kit/taosnetwork/server.c @@ -142,9 +142,9 @@ static void *bindTcpPort(void *sarg) { printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); if (iDataNum > 0) { send(client, buffer, iDataNum, 0); - break; } } + close(serverSocket); return NULL; } @@ -201,7 +201,7 @@ static void *bindUdpPort(void *sarg) { int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", 6030, 6060, 1000}; + SArguments arguments = {"127.0.0.1", 6030, 6042, 1000}; argp_parse(&argp, argc, argv, 0, 0, &arguments); if (arguments.pktLen > MAX_PKG_LEN) { printf("test pkg len overflow: %d, max len not greater than %d bytes\n", arguments.pktLen, MAX_PKG_LEN); From 66a977df9cb69eafc7ee2a944cbee54ab4de1195 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 17:43:47 +0800 Subject: [PATCH 055/111] [TD-1132] --- src/kit/taosnetwork/client.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c index a157a3a9d2..8554e2acee 100644 --- a/src/kit/taosnetwork/client.c +++ b/src/kit/taosnetwork/client.c @@ -31,19 +31,19 @@ #define MAX_PKG_LEN (64*1000) #define BUFFER_SIZE (MAX_PKG_LEN + 1024) -#define TSDB_FQDN_LEN 128 -#define TSDB_IPv4ADDR_LEN 16 +#define TEST_FQDN_LEN 128 +#define TEST_IPv4ADDR_LEN 16 typedef struct { uint16_t port; uint32_t hostIp; - char fqdn[TSDB_FQDN_LEN]; + char fqdn[TEST_FQDN_LEN]; uint16_t pktLen; } info_s; typedef struct Arguments { - char host[TSDB_IPv4ADDR_LEN]; - char fqdn[TSDB_FQDN_LEN]; + char host[TEST_IPv4ADDR_LEN]; + char fqdn[TEST_FQDN_LEN]; uint16_t port; uint16_t max_port; uint16_t pktLen; From fffbef43e34047c2c5e0fcf444abed0fc39adaac Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 25 Aug 2020 20:09:06 +0800 Subject: [PATCH 056/111] [TD-1132] --- src/kit/taosnetwork/client.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c index 8554e2acee..b7db2ba0a2 100644 --- a/src/kit/taosnetwork/client.c +++ b/src/kit/taosnetwork/client.c @@ -105,6 +105,18 @@ int checkTcpPort(info_s *info) { printf("socket() fail: %s\n", strerror(errno)); return -1; } + + // set send and recv overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } + serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(info->port); @@ -112,7 +124,7 @@ int checkTcpPort(info_s *info) { //printf("=================================\n"); if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - printf("connect() fail: %s\n", strerror(errno)); + printf("connect() fail: %s\t", strerror(errno)); return -1; } //printf("Connect to: %s:%d...success\n", host, port); @@ -171,6 +183,17 @@ int checkUdpPort(info_s *info) { perror("socket"); return -1; } + + // set overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(info->port); @@ -195,7 +218,7 @@ int checkUdpPort(info_s *info) { iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); if (iDataNum < info->pktLen) { - printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\n", iDataNum, info->pktLen, info->port); + printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\t\t", iDataNum, info->pktLen, info->port); return -1; } @@ -239,14 +262,14 @@ void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t p info.port = port; ret = checkTcpPort(&info); if (ret != 0) { - printf("tcp port:%d test fail.\t\t", port); + printf("tcp port:%d test fail.\t\n", port); } else { printf("tcp port:%d test ok.\t\t", port); } ret = checkUdpPort(&info); if (ret != 0) { - printf("udp port:%d test fail.\t\t", port); + printf("udp port:%d test fail.\t\n", port); } else { printf("udp port:%d test ok.\t\t", port); } From a373658308be3af32574aeb1b71f54630fa6b6b8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 25 Aug 2020 21:35:15 +0800 Subject: [PATCH 057/111] change to v16/v20 instead of master/develop --- .../perftest-scripts/tdengineTestWriteLoop.sh | 86 +++++++++++++------ 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index e1ee4418dc..cb9d87d0c1 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -24,22 +24,46 @@ function runTest { for r in ${!rowsPerRequest[@]}; do for c in `seq 1 $clients`; do totalRPR=0 - OUTPUT_FILE=tdengineTestWrite-RPR${rowsPerRequest[$r]}-clients$c.out + if $v16 ; then + OUTPUT_FILE=tdengineTestWrite-v16-RPR${rowsPerRequest[$r]}-clients$c.out + else + OUTPUT_FILE=tdengineTestWrite-v20-RPR${rowsPerRequest[$r]}-clients$c.out + fi for i in `seq 1 $NUM_LOOP`; do - restartTaosd - $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 - printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + if ! $printresultonly ; then + restartTaosd + $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 + + if $v16 ; then + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]}" - $TDTEST_DIR/tdengineTest \ - -dataDir $DATA_DIR \ - -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]} \ - | tee $OUTPUT_FILE + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + else + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + fi + fi + RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` printTo "rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR" @@ -86,25 +110,30 @@ function restartTaosd { ################ Main ################ -master=false -develop=true +v16=false +v20=true verbose=false clients=1 +printresultonly=false while : ; do case $1 in + printresultonly) + printresultonly=true + shift ;; + -v) verbose=true shift ;; - master) - master=true - develop=false + v16) + v16=true + v20=false shift ;; - develop) - master=false - develop=true + v20) + v16=false + v20=true shift ;; -c) @@ -120,19 +149,24 @@ while : ; do esac done -if $master ; then - echo "Test master branch.." - cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg - WORK_DIR=/mnt/root/TDengine.master +if $v16 ; then + echo "Test v16 branch.." + WORK_DIR=/mnt/root/TDengine.v16 + cp /mnt/root/cfg/v16/taos.cfg /etc/taos/taos.cfg else - echo "Test develop branch.." - cp /mnt/root/cfg/develop/taos.cfg /etc/taos/taos.cfg + echo "Test v20 branch.." + cp /mnt/root/cfg/v20/taos.cfg /etc/taos/taos.cfg WORK_DIR=/mnt/root/TDengine fi TAOSD_DIR=$WORK_DIR/debug/build/bin TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine +if [ ! -f $TDTEST_DIR/tdengineTest ]; then + echo "Please build tdengineTest first!" + exit 1 +fi + runTest echo "Test done!" From 60a1f46bfbbf8075b986939c8277cfaea034e381 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 25 Aug 2020 22:25:46 +0800 Subject: [PATCH 058/111] Update faq-ch.md --- documentation20/webdocs/markdowndocs/faq-ch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 928096dac3..4a3b88a256 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -81,6 +81,7 @@ Connection = DriverManager.getConnection(url, properties); 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos + 附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 From 5518b07395a737ff55240b2b31e93c2b30530fd4 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 26 Aug 2020 06:45:21 +0800 Subject: [PATCH 059/111] Update faq-ch.md --- .../webdocs/markdowndocs/faq-ch.md | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 928096dac3..2e2a2180a2 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -36,25 +36,34 @@ 检查客户端侧TCP端口链接是否工作：`nc {hostIP} {port}` -## 6. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 +## 6. 遇到错误“Unexpected generic error in RPC”， 我怎么办？ +产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： + +1. 请检查连接的服务器的FQDN是否正确 +2. 如果网络配置有DNS server, 请检查是否正常工作 +3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件，查看该FQDN是否配置，并是否有正确的IP地址。 +4. 如果网络配置OK，从客户端所在机器，你需要能Ping该连接的FQDN，否则客户端是无法链接服务器的 + + +## 7. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。 -## 7. 是否支持validation queries? +## 8. 是否支持validation queries? TDengine还没有一组专用的validation queries。然而建议你使用系统监测的数据库”log"来做。 -## 8. 我可以删除或更新一条记录吗？ +## 9. 我可以删除或更新一条记录吗？ 不能。因为TDengine是为联网设备采集的数据设计的，不容许修改。但TDengine提供数据保留策略，只要数据记录超过保留时长，就会被自动删除。 -## 10. 我怎么创建超过250列的表？ +## 10. 我怎么创建超过1024列的表？ 使用2.0及其以上版本，默认支持1024列；2.0之前的版本，TDengine最大允许创建250列的表。但是如果确实超过限值，建议按照数据特性，逻辑地将这个宽表分解成几个小表。 ## 10. 最有效的写入数据的方法是什么？ -批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的记录。 +批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的多条记录。 ## 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ @@ -77,7 +86,7 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 14. 怎么报告问题？ +## 15. 怎么报告问题？ 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos From 9efe5fe06ad0077c18aae2ee13bbe25d390e646a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 26 Aug 2020 00:00:58 +0000 Subject: [PATCH 060/111] minor changes : --- documentation20/webdocs/markdowndocs/faq-ch.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 2e2a2180a2..b760fe161a 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -86,10 +86,11 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 15. 怎么报告问题？ +## 14. 怎么报告问题？ 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos + 附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 From 18637381a43e30779a028cabc7d96e0c75edbaf0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 11:03:42 +0800 Subject: [PATCH 061/111] [td-1151] --- src/client/src/tscSQLParser.c | 104 ++++++++++++++--------- src/common/src/tname.c | 6 +- src/query/src/qExecutor.c | 15 +++- src/query/src/qParserImpl.c | 2 + tests/script/general/parser/constCol.sim | 77 ++++++++++++----- 5 files changed, 138 insertions(+), 66 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3b91b381ef..e9fc27bb4e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -65,11 +65,11 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int3 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t changeFunctionID(int32_t optr, int16_t* functionId); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); @@ -1176,11 +1176,15 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; - char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; - size_t len = MIN(sizeof(pItem->aliasName), pItem->pNode->token.n + 1); - tstrncpy(pItem->aliasName, name, len); + char aliasName[TSDB_COL_NAME_LEN] = {0}; + if (pItem->aliasName != NULL) { + tstrncpy(aliasName, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + int32_t nameLen = MIN(TSDB_COL_NAME_LEN, pItem->pNode->token.n + 1); + tstrncpy(aliasName, pItem->pNode->token.z, nameLen); + } - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pItem->aliasName, NULL); + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); @@ -1211,7 +1215,30 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* tscColumnListInsert(pQueryInfo->colList, &tsCol); } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { + SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); + + char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; + tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); + + SColumnList ids = {0}; + ids.num = 1; + ids.ids[0] = *pIndex; + + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || + pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { + ids.num = 0; + } + + insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); +} + +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { assert(pSelection != NULL && pCmd != NULL); const char* msg2 = "functions can not be mixed up"; @@ -1252,12 +1279,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (code != TSDB_CODE_SUCCESS) { return code; } - } else { - /* - * not support such expression - * e.g., select 12+5 from table_name - */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1270,10 +1292,22 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { SColumnIndex index = {0}; - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + + // set the constant column value always attached to first table. + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, clauseIndex, 0); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); + + // add the timestamp column into the output columns + int32_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); + tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + + SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); + pSupInfo->visible = false; + + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } - if (!functionCompatibleCheck(pQueryInfo)) { + if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1336,29 +1370,6 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { - SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - - char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); - - SColumnList ids = {0}; - ids.num = 1; - ids.ids[0] = *pIndex; - - if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || - pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { - ids.num = 0; - } - - insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); -} - SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, @@ -1837,10 +1848,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i + j, &index) != - 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } + + colIndex++; } numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -2513,7 +2525,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t startIdx = 0; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); @@ -2545,6 +2557,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { if (functionCompatList[functionId] != factor) { return false; } + + if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { + return false; + } } return true; @@ -5863,7 +5879,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6046,7 +6062,8 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6080,6 +6097,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } } else { // set the time rang pQueryInfo->window = TSWINDOW_INITIALIZER; + if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } } // user does not specified the query time window, twa is not allowed in such case. diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 2eda5953a3..960cc7d725 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -62,8 +62,10 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t len = MIN(sizeof(s.name), exprStr->n + 1); - tstrncpy(s.name, exprStr->z, len); + size_t len = strdequote(exprStr->z); + size_t tlen = MIN(sizeof(s.name), len + 1); + + tstrncpy(s.name, exprStr->z, tlen); } return s; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5baa3f2bf8..19324bc2cb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1770,13 +1770,20 @@ static bool needReverseScan(SQuery *pQuery) { return false; } +/** + * The following 4 kinds of query are treated as the tags query + * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query + */ static bool onlyQueryTags(SQuery* pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; int32_t functionId = pExprInfo->base.functionId; - if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG && - (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX))) { + + if (functionId != TSDB_FUNC_TAGPRJ && + functionId != TSDB_FUNC_TID_TAG && + (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == TSDB_FUNC_PRJ && pExprInfo->base.colInfo.flag == TSDB_COL_UDC))) { return false; } } @@ -6835,6 +6842,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) { char *data = NULL, *dst = NULL; int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (pExprInfo[j].base.colInfo.flag == TSDB_COL_UDC) { + continue; + } if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { bytes = tbnameSchema.bytes; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 0b5408f793..733459c6fe 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -167,6 +167,8 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt pExpr->operand.n = len; // raw field name pExpr->operand.type = pFuncToken->type; + + pExpr->token = pExpr->operand; return pExpr; } diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim index 889fe83042..a196ba2b50 100644 --- a/tests/script/general/parser/constCol.sim +++ b/tests/script/general/parser/constCol.sim @@ -34,6 +34,9 @@ sql create table t1 using st1 tags(1); sql create table t2 using st2 tags(1); sql insert into t1 values(1575880055000, 1); +sql insert into t1 values(1575880059000, 1); +sql insert into t1 values(1575880069000, 1); + sql insert into t2 values(1575880055000, 2); sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts @@ -42,7 +45,7 @@ system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl print ==============select with user-defined columns sql select 'abc' as f, ts,f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -59,7 +62,7 @@ if $data02 != 1 then endi sql select 'abc', ts, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -71,8 +74,12 @@ if $data02 != 1 then return -1 endi +if $data10 != @abc@ then + return -1 +endi + sql select 'abc' from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -81,7 +88,7 @@ if $data00 != @abc@ then endi sql select 'abc' as f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -90,7 +97,7 @@ if $data00 != @abc@ then endi sql select 1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -98,8 +105,16 @@ if $data00 != 1 then return -1 endi +if $data10 != 1 then + return -1 +endi + +if $data20 != 1 then + return -1 +endi + sql select 1 as f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -108,7 +123,7 @@ if $data00 != 1 then endi sql select 1 as f, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -121,7 +136,7 @@ if $data01 != 1 then endi sql select 1.123 as f, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -130,8 +145,13 @@ if $data00 != 1.123000000 then return -1 endi +if $data10 != 1.123000000 then + print expect 1.123000000 , actual:$data10 + return -1 +endi + sql select 1, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -143,8 +163,12 @@ if $data01 != 1 then return -1 endi +if $data10 != 1 then + return -1 +endi + sql select 1.2391, f1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -153,6 +177,11 @@ if $data00 != 1.239100000 then return -1 endi +if $data10 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + if $data01 != 1 then return -1 endi @@ -167,7 +196,7 @@ if $data00 != 1 then return -1 endi -if $data01 != 1 then +if $data01 != 3 then return -1 endi @@ -180,7 +209,7 @@ if $data00 != 1 then return -1 endi -if $data01 != 1 then +if $data01 != 3 then return -1 endi @@ -201,8 +230,8 @@ if $data02 != 3 then return -1 endi -if $data03 != 99.000000000 then - print expect 99.000000000, actual:$data03 +if $data03 != 297.000000000 then + print expect 297.000000000, actual:$data03 return -1 endi @@ -211,8 +240,8 @@ if $rows != 1 then return -1 endi -if $data00 != 13.000000000 then - print expect 13.000000000 actual:$data00 +if $data00 != 15.000000000 then + print expect 15.000000000 actual:$data00 return -1 endi @@ -225,7 +254,7 @@ if $data02 != 2 then endi sql select 1.2987, f1, 'k' from t1 where f1=1 -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -244,7 +273,7 @@ endi print ====================user-defined columns with union sql select f1, 'f1' from t1 union all select f1, 'f1' from t1; -if $rows != 2 then +if $rows != 6 then return -1 endi @@ -294,7 +323,7 @@ endi print ======================udc with interval sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; -if $rows != 1 then +if $rows != 3 then return -1 endi @@ -304,9 +333,17 @@ if $rows != 1 then return -1 endi +if $data01 != @abc@ then + return -1 +endi + +if $data02 != @t1@ then + return -1 +endi + print ======================udc with arithmetic sql select 1+1 from t1 -if $rows != 1 then +if $rows != 3 then return -1 endi From 8db31201473377b6c3a3e917d7e7a81e51eb091c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 26 Aug 2020 11:15:24 +0800 Subject: [PATCH 062/111] TD-1236 native c# examples for 2.0 --- src/inc/taos.h | 2 +- tests/examples/C#/TDengineDriver.cs | 56 ++++++++++----------- tests/examples/C#/TDengineTest.cs | 78 +++++++++++++++-------------- 3 files changed, 70 insertions(+), 66 deletions(-) diff --git a/src/inc/taos.h b/src/inc/taos.h index f3cc9bb4d7..7e8f174b7c 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -93,7 +93,7 @@ DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision DLL_EXPORT void taos_free_result(TAOS_RES *res); DLL_EXPORT int taos_field_count(TAOS_RES *tres); DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *taos); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); diff --git a/tests/examples/C#/TDengineDriver.cs b/tests/examples/C#/TDengineDriver.cs index 2797c362c1..fcd5723405 100644 --- a/tests/examples/C#/TDengineDriver.cs +++ b/tests/examples/C#/TDengineDriver.cs @@ -20,16 +20,17 @@ using System.Runtime.InteropServices; namespace TDengineDriver { enum TDengineDataType { - TSDB_DATA_TYPE_BOOL = 1, - TSDB_DATA_TYPE_TINYINT = 2, - TSDB_DATA_TYPE_SMALLINT = 3, - TSDB_DATA_TYPE_INT = 4, - TSDB_DATA_TYPE_BIGINT = 5, - TSDB_DATA_TYPE_FLOAT = 6, - TSDB_DATA_TYPE_DOUBLE = 7, - TSDB_DATA_TYPE_BINARY = 8, - TSDB_DATA_TYPE_TIMESTAMP = 9, - TSDB_DATA_TYPE_NCHAR = 10 + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10 // unicode string } enum TDengineInitOption @@ -83,50 +84,49 @@ namespace TDengineDriver [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.StdCall)] static extern public void Init(); + [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.StdCall)] + static extern public void Cleanup(); + [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.StdCall)] static extern public void Options(int option, string value); [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.StdCall)] - static extern public long Connect(string ip, string user, string password, string db, int port); + static extern public long Connect(string ip, string user, string password, string db, short port); [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.StdCall)] - static extern private IntPtr taos_errstr(long taos); - static public string Error(long conn) + static extern private IntPtr taos_errstr(long res); + static public string Error(long res) { - IntPtr errPtr = taos_errstr(conn); + IntPtr errPtr = taos_errstr(res); return Marshal.PtrToStringAnsi(errPtr); } [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.StdCall)] - static extern public int ErrorNo(long taos); + static extern public int ErrorNo(long res); [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.StdCall)] - static extern public int Query(long taos, string sqlstr); + static extern public long Query(long conn, string sqlstr); [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.StdCall)] - static extern public int AffectRows(long taos); - - [DllImport("taos.dll", EntryPoint = "taos_use_result", CallingConvention = CallingConvention.StdCall)] - static extern public long UseResult(long taos); + static extern public int AffectRows(long res); [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.StdCall)] - static extern public int FieldCount(long taos); + static extern public int FieldCount(long res); [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.StdCall)] static extern private IntPtr taos_fetch_fields(long res); - static public List FetchFields(long taos) + static public List FetchFields(long res) { const int fieldSize = 68; List metas = new List(); - long result = TDengine.UseResult(taos); - if (result == 0) + if (res == 0) { return metas; } - int fieldCount = FieldCount(taos); - IntPtr fieldsPtr = taos_fetch_fields(result); + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); for (int i = 0; i < fieldCount; ++i) { @@ -134,8 +134,8 @@ namespace TDengineDriver TDengineMeta meta = new TDengineMeta(); meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); - meta.size = Marshal.ReadInt16(fieldsPtr + offset + 64); - meta.type = Marshal.ReadByte(fieldsPtr + offset + 66); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); metas.Add(meta); } diff --git a/tests/examples/C#/TDengineTest.cs b/tests/examples/C#/TDengineTest.cs index 235775f68f..3f8161cadf 100644 --- a/tests/examples/C#/TDengineTest.cs +++ b/tests/examples/C#/TDengineTest.cs @@ -28,7 +28,7 @@ namespace TDengineDriver private string configDir; private string user; private string password; - private int port = 0; + private short port = 0; //sql parameters private string dbName; @@ -211,58 +211,62 @@ namespace TDengineDriver StringBuilder sql = new StringBuilder(); sql.Append("create database if not exists ").Append(this.dbName); - int code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + long res = TDengine.Query(this.conn, sql.ToString()); + if (res != 0) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); sql.Clear(); sql.Append("use ").Append(this.dbName); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + res = TDengine.Query(this.conn, sql.ToString()); + if (res != 0) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); sql.Clear(); - sql.Append("create table if not exists ").Append(this.stableName).Append("(ts timestamp, v1 int) tags(t1 int)"); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + sql.Append("create table if not exists ").Append(this.stableName).Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); + res = TDengine.Query(this.conn, sql.ToString()); + if (res != 0) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); for (int i = 0; i < this.tableCount; i++) { sql.Clear(); sql = sql.Append("create table if not exists ").Append(this.tablePrefix).Append(i) .Append(" using ").Append(this.stableName).Append(" tags(").Append(i).Append(")"); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + res = TDengine.Query(this.conn, sql.ToString()); + if (res != 0) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); } Console.WriteLine("create db and table success"); @@ -287,16 +291,22 @@ namespace TDengineDriver for (int batch = 0; batch < this.batchRows; ++batch) { long rows = loop * this.batchRows + batch; - sql.Append("(").Append(this.beginTimestamp + rows).Append(",").Append(rows).Append(")"); + sql.Append("(") + .Append(this.beginTimestamp + rows) + .Append(", 1, 2, 3,") + .Append(rows) + .Append(", 5, 6, 7, 'abc', 'def')"); } - int code = TDengine.Query(conn, sql.ToString()); - if (code != TDengine.TSDB_CODE_SUCCESS) + long res = TDengine.Query(conn, sql.ToString()); + if (res == 0) { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); } - int affectRows = TDengine.AffectRows(conn); + int affectRows = TDengine.AffectRows(res); this.rowsInserted += affectRows; + + TDengine.FreeResult(res); } } @@ -322,32 +332,25 @@ namespace TDengineDriver String sql = "select * from " + this.dbName + "." + tablePrefix + i; Console.WriteLine(sql); - int code = TDengine.Query(conn, sql); - if (code != TDengine.TSDB_CODE_SUCCESS) + long res = TDengine.Query(conn, sql); + if (res == 0) { - Console.WriteLine(sql + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } - int fieldCount = TDengine.FieldCount(conn); - //Console.WriteLine("field count: " + fieldCount); + int fieldCount = TDengine.FieldCount(res); + Console.WriteLine("field count: " + fieldCount); - List metas = TDengine.FetchFields(conn); + List metas = TDengine.FetchFields(res); for (int j = 0; j < metas.Count; j++) { TDengineMeta meta = (TDengineMeta)metas[j]; - //Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); - } - - long result = TDengine.UseResult(conn); - if (result == 0) - { - Console.WriteLine(sql + " result set is null"); - return; + Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); } IntPtr rowdata; - while ((rowdata = TDengine.FetchRows(result)) != IntPtr.Zero) + while ((rowdata = TDengine.FetchRows(res)) != IntPtr.Zero) { queryRows++; for (int fields = 0; fields < fieldCount; ++fields) @@ -411,12 +414,12 @@ namespace TDengineDriver //Console.WriteLine("---"); } - if (TDengine.ErrorNo(conn) != 0) + if (TDengine.ErrorNo(res) != 0) { - Console.Write("Query is not complete， Error {0:G}", TDengine.ErrorNo(conn), TDengine.Error(conn)); + Console.Write("Query is not complete, Error {0:G}", TDengine.ErrorNo(res), TDengine.Error(res)); } - TDengine.FreeResult(result); + TDengine.FreeResult(res); } System.DateTime end = new System.DateTime(); @@ -436,6 +439,7 @@ namespace TDengineDriver static void ExitProgram() { + TDengine.Cleanup(); System.Environment.Exit(0); } } From 728e44f50aac3ba73c242d43d054847548c02f3d Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 26 Aug 2020 11:15:39 +0800 Subject: [PATCH 063/111] [TD-1162] --- packaging/release.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index fe46c97928..bf355bdcd4 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -10,6 +10,7 @@ set -e # -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] # -V [stable | beta] # -l [full | lite] +# -s [static | dynamic] # -n [2.0.0.3] # set parameters by default value @@ -18,9 +19,10 @@ verType=stable # [stable, beta] cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] +soMode=dynamic # [static | dynamic] verNumber="" -while getopts "hv:V:c:o:l:n:" arg +while getopts "hv:V:c:o:l:s:n:" arg do case $arg in v) @@ -39,6 +41,10 @@ do #echo "pagMode=$OPTARG" pagMode=$(echo $OPTARG) ;; + s) + #echo "soMode=$OPTARG" + soMode=$(echo $OPTARG) + ;; n) #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) @@ -53,6 +59,7 @@ do echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " echo " -V [stable | beta] " echo " -l [full | lite] " + echo " -s [static | dynamic] " echo " -n [version number] " exit 0 ;; @@ -63,7 +70,7 @@ do esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} verNumber=${verNumber}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber}" curr_dir=$(pwd) @@ -223,9 +230,9 @@ cd ${compile_dir} # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DOSTYPE=${osType} + cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DOSTYPE=${osType} -DSOMODE=${soMode} else - cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} fi else echo "input cpuType=${cpuType} error!!!" From 6d7faf97f1d2bdca4237862c9dc9c25cc53244e8 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 26 Aug 2020 12:09:54 +0800 Subject: [PATCH 064/111] [update release ver info] --- src/util/src/version.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/util/src/version.c b/src/util/src/version.c index 6f7b29ebb9..746554ab6e 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,7 +1,7 @@ -char version[12] = "2.0.2.0"; +char version[12] = "2.0.2.1"; char compatible_version[12] = "2.0.0.0"; -char gitinfo[48] = "d711657139620f6c50f362597020705b8ad26bd2"; -char gitinfoOfInternal[48] = "1d74ae24c541ffbb280e8630883c0236cd45f8c7"; -char buildinfo[64] = "Built by root at 2020-08-24 16:31"; +char gitinfo[48] = "1527828603722a0ed8ea706439e8c172a13b7eb1"; +char gitinfoOfInternal[48] = "490a983428d02b00d06b2a067b3f6e74f4b8d1a8"; +char buildinfo[64] = "Built by root at 2020-08-26 12:00"; -void libtaos_2_0_2_0_Linux_x64_beta() {}; +void libtaos_2_0_2_1_Linux_x64() {}; From 5e17894fbae8932ba8a70c0caad1d2ff0f5847e8 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 26 Aug 2020 14:33:03 +0800 Subject: [PATCH 065/111] Update architecture-ch.md --- documentation20/webdocs/markdowndocs/architecture-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index bfe3b55bd2..7ab4b5d096 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -82,7 +82,7 @@ TDengine 分布式架构的逻辑结构图如下： ### 节点之间的通讯 **通讯方式：**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。 -**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过选项“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接使用IP地址作为FQDN，但不建议使用，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。 +**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。 **端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP链接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 From f9f9580f9a1b1abc4a3422c79c79996198a5c316 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 26 Aug 2020 06:45:35 +0000 Subject: [PATCH 066/111] TD-1238 add error code while unable to resolve fqdn --- src/inc/taoserror.h | 1 + src/rpc/src/rpcMain.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 57c2b322fa..1e7f8be564 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -66,6 +66,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid me TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Invalid timestamp") TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x000C, "Unable to resolve FQDN") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 7913b1c5b0..3aecd127ef 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -563,7 +563,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); - terrno = TSDB_CODE_RPC_APP_ERROR; + terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } From 06d188e98286c5ccf7ebf499e69d4762e64379b2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 26 Aug 2020 06:47:19 +0000 Subject: [PATCH 067/111] minor changes --- src/inc/taoserror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 1e7f8be564..df69d1d19a 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -66,7 +66,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid me TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Invalid timestamp") TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x000C, "Unable to resolve FQDN") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") From 53a7e3d62bc26fd66e190ebead996f4894e4b6df Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 15:51:52 +0800 Subject: [PATCH 068/111] [td-225] add test case for join query. --- src/client/inc/tsclient.h | 1 - src/client/src/tscAsync.c | 3 +- src/client/src/tscServer.c | 9 ++-- src/client/src/tscSql.c | 2 - src/client/src/tscSubquery.c | 68 +++++++++++++++------------- src/client/src/tscUtil.c | 4 +- src/util/inc/tlockfree.h | 8 ++++ tests/pytest/crash_gen.sh | 2 +- tests/script/general/parser/join.sim | 5 +- 9 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index aecbdd8207..004c5bc21c 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -324,7 +324,6 @@ typedef struct SSqlObj { SSqlRes res; uint16_t numOfSubs; struct SSqlObj **pSubs; - tsem_t subReadySem; struct SSqlObj * prev, *next; } SSqlObj; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b05aef76eb..4643d255dc 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -433,7 +433,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); goto _error; } else { - tscDebug("%p get tableMeta successfully", pSql); + const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; + tscDebug("%p get %s successfully", pSql, msg); } if (pSql->pStream == NULL) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e33bfe44d1..81be0625ea 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -454,6 +454,8 @@ void tscKillSTableQuery(SSqlObj *pSql) { return; } + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + for (int i = 0; i < pSql->numOfSubs; ++i) { // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; @@ -466,7 +468,7 @@ void tscKillSTableQuery(SSqlObj *pSql) { rpcCancelRequest(pSub->pRpcCtx); } - tscQueueAsyncRes(pSub); + tscQueueAsyncRes(pSub); // async res? not other functions? } tscDebug("%p super table query cancelled", pSql); @@ -1436,11 +1438,6 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { return code; } - // all subquery have completed already - if (pRes->pLocalReducer == NULL) { - sem_wait(&pSql->subReadySem); - } - pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1a774e8784..8d33368280 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -295,8 +295,6 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { } tsem_init(&pSql->rspSem, 0, 0); - tsem_init(&pSql->subReadySem, 0, 0); - doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); tsem_wait(&pSql->rspSem); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c50b663e02..4e2edde31e 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1146,7 +1146,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { ///////////////////////////////////////////////////////////////////////////////////////// static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; @@ -1411,7 +1411,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSql = pSql; trs->pFinalColModel = pModel; - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); + SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); taosTFree(trs->localBuffer); @@ -1451,18 +1451,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscDebug("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); tscProcessSql(pSub); } - - // set the command flag must be after the semaphore been correctly set. - pSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, 0); - } return TSDB_CODE_SUCCESS; } static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { - tscDebug("%p start to free subquery result", pSql); + tscDebug("%p start to free subquery obj", pSql); int32_t index = trsupport->subqueryIndex; SSqlObj *pParentSql = trsupport->pParentSql; @@ -1503,10 +1497,10 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - tscDebug("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscError("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); - SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); + SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); @@ -1584,9 +1578,17 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes - tsem_post(&pParentSql->subReadySem); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); + + if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { + (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); + } else { // regular super table query + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pParentSql); + } + } } static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) { @@ -1659,8 +1661,13 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - // all subqueries are completed, retrieve from local can be proceeded. - tsem_post(&pParentSql->subReadySem); + // set the command flag must be after the semaphore been correctly set. + pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; + if (pParentSql->res.code == TSDB_CODE_SUCCESS) { + (*pParentSql->fp)(pParentSql->param, pParentSql, 0); + } else { + tscQueueAsyncRes(pParentSql); + } } static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) { @@ -1669,21 +1676,22 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t idx = trsupport->subqueryIndex; SSqlObj * pParentSql = trsupport->pParentSql; + assert(tres != NULL); SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // sql object has been released in error process, return immediately - tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); - return; - } - +// if (pSql == NULL) { // sql object has been released in error process, return immediately +// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); +// return; +// } + SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscDebug("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscDebug("%p query cancelled/failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, numOfRows); @@ -1694,7 +1702,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(numOfRows == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscDebug("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; @@ -1745,11 +1753,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR (int32_t)pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); if (ret != 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - } else if (pRes->completed) { tscAllDataRetrievedFromDnode(trsupport, pSql); - return; - } else { // continue fetch data from dnode taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } @@ -1759,15 +1764,15 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } } -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { const int32_t table_index = 0; SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1); + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->numOfSubs); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); @@ -1784,7 +1789,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj* pParentSql = trsupport->pParentSql; SSqlObj* pSql = (SSqlObj *) tres; - + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); @@ -1812,7 +1817,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscWarn("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } @@ -2099,7 +2104,6 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlRes *pRes = &pSql->res; assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker taosTFree(pRes->tsrow); return pRes->tsrow; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 788283a1ad..9f23ac2bdc 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -373,7 +373,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } - + tscDebug("%p start to free sql object", pSql); tscPartiallyFreeSqlObj(pSql); @@ -388,7 +388,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); - tsem_destroy(&pSql->subReadySem); free(pSql); } @@ -1759,6 +1758,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; + SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); diff --git a/src/util/inc/tlockfree.h b/src/util/inc/tlockfree.h index a81f597832..e960b601ca 100644 --- a/src/util/inc/tlockfree.h +++ b/src/util/inc/tlockfree.h @@ -36,6 +36,14 @@ typedef void (*_ref_fn_t)(const void* pObj); _ref_fn_t end; \ } _ref_func = {.begin = (s), .end = (e)}; +// set the initial reference count value +#define T_REF_INIT_VAL(x, _v) \ + do { \ + assert(_v >= 0); \ + atomic_store_32(&((x)->_ref.val), (_v)); \ + } while (0) + +// increase the reference count by 1 #define T_REF_INC(x) (atomic_add_fetch_32(&((x)->_ref.val), 1)) #define T_REF_INC_WITH_CB(x, p) \ diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index f6be6aae49..df1a9f595b 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -49,4 +49,4 @@ export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR # Now we are all let, and let's see if we can find a crash. Note we pass all params -python3 ./crash_gen.py $@ +python3.8 ./crash_gen.py $@ diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 882f561ae1..c4e490190d 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -330,7 +330,10 @@ sql_error select join_tb1.* from $tb1 , $tb2 where join_tb1.ts != join_tb0.ts an sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts and join_tb1.ts >= 100000; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by ts; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by join_tb1.c7; - +sql_error select * from $tb1, $tb2; +sql_error select last_row(*) from $tb1, $tb2 +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts < now +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts = join_tb2.ts print ==================================super table join ============================== # select duplicate columns From 3326df48736da2a86c0246617778ed278372be2e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 16:01:53 +0800 Subject: [PATCH 069/111] [td-225] change python script version. --- tests/pytest/crash_gen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index df1a9f595b..f6be6aae49 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -49,4 +49,4 @@ export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR # Now we are all let, and let's see if we can find a crash. Note we pass all params -python3.8 ./crash_gen.py $@ +python3 ./crash_gen.py $@ From f3e15cc97982d31157bf71495b6b3a38572bcd11 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 26 Aug 2020 16:15:37 +0800 Subject: [PATCH 070/111] [add arbitrator default port in docker] --- packaging/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 3fb34e8286..668d5a49eb 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -12,6 +12,6 @@ ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 -EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 +EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 CMD ["taosd"] VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] From bd16bb3d230bd6189ecd8319f58dc01d3f665c89 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 26 Aug 2020 17:36:57 +0800 Subject: [PATCH 071/111] TD-1214 change csharp driver in win32 --- src/mnode/src/mnodeDb.c | 2 +- src/mnode/src/mnodeDnode.c | 2 +- tests/examples/C#/TDengineDriver.cs | 54 ++++++++++++------------ tests/examples/C#/TDengineTest.cs | 65 ++++++++++++++++------------- 4 files changed, 65 insertions(+), 58 deletions(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 0367ce21fd..cdea9eda60 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -354,7 +354,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg mDebug("db:%s, already exist, ignore exist is set", pCreate->db); return TSDB_CODE_SUCCESS; } else { - mError("db:%s, is already exist, ignore exist not set", pCreate->db); + mError("db:%s, already exist, ignore exist not set", pCreate->db); return TSDB_CODE_MND_DB_ALREADY_EXIST; } } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 5da57a3093..ccb85c3301 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -518,7 +518,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); if (pDnode != NULL) { mnodeDecDnodeRef(pDnode); - mError("dnode:%d is already exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); + mError("dnode:%d, already exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); return TSDB_CODE_MND_DNODE_ALREADY_EXIST; } diff --git a/tests/examples/C#/TDengineDriver.cs b/tests/examples/C#/TDengineDriver.cs index fcd5723405..b6f143e181 100644 --- a/tests/examples/C#/TDengineDriver.cs +++ b/tests/examples/C#/TDengineDriver.cs @@ -80,47 +80,47 @@ namespace TDengineDriver class TDengine { public const int TSDB_CODE_SUCCESS = 0; - - [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.StdCall)] + + [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] static extern public void Init(); - [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.StdCall)] + [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] static extern public void Cleanup(); - [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.StdCall)] + [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] static extern public void Options(int option, string value); - [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.StdCall)] - static extern public long Connect(string ip, string user, string password, string db, short port); + [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); - [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.StdCall)] - static extern private IntPtr taos_errstr(long res); - static public string Error(long res) + [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) { IntPtr errPtr = taos_errstr(res); return Marshal.PtrToStringAnsi(errPtr); } - [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.StdCall)] - static extern public int ErrorNo(long res); + [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.StdCall)] - static extern public long Query(long conn, string sqlstr); + [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); - [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.StdCall)] - static extern public int AffectRows(long res); + [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.StdCall)] - static extern public int FieldCount(long res); + [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.StdCall)] - static extern private IntPtr taos_fetch_fields(long res); - static public List FetchFields(long res) + [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) { const int fieldSize = 68; List metas = new List(); - if (res == 0) + if (res == IntPtr.Zero) { return metas; } @@ -142,13 +142,13 @@ namespace TDengineDriver return metas; } - [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.StdCall)] - static extern public IntPtr FetchRows(long res); + [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.StdCall)] - static extern public IntPtr FreeResult(long res); + [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.StdCall)] - static extern public int Close(long taos); + [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); } } \ No newline at end of file diff --git a/tests/examples/C#/TDengineTest.cs b/tests/examples/C#/TDengineTest.cs index 3f8161cadf..6b3f1160ad 100644 --- a/tests/examples/C#/TDengineTest.cs +++ b/tests/examples/C#/TDengineTest.cs @@ -43,7 +43,7 @@ namespace TDengineDriver private long batchRows; private long beginTimestamp = 1551369600000L; - private long conn = 0; + private IntPtr conn = IntPtr.Zero; private long rowsInserted = 0; static void Main(string[] args) @@ -191,7 +191,7 @@ namespace TDengineDriver { string db = ""; this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); - if (this.conn == 0) + if (this.conn == IntPtr.Zero) { Console.WriteLine("Connect to TDengine failed"); ExitProgram(); @@ -211,8 +211,8 @@ namespace TDengineDriver StringBuilder sql = new StringBuilder(); sql.Append("create database if not exists ").Append(this.dbName); - long res = TDengine.Query(this.conn, sql.ToString()); - if (res != 0) + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } @@ -226,7 +226,7 @@ namespace TDengineDriver sql.Clear(); sql.Append("use ").Append(this.dbName); res = TDengine.Query(this.conn, sql.ToString()); - if (res != 0) + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } @@ -240,7 +240,7 @@ namespace TDengineDriver sql.Clear(); sql.Append("create table if not exists ").Append(this.stableName).Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); res = TDengine.Query(this.conn, sql.ToString()); - if (res != 0) + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } @@ -257,7 +257,7 @@ namespace TDengineDriver sql = sql.Append("create table if not exists ").Append(this.tablePrefix).Append(i) .Append(" using ").Append(this.stableName).Append(" tags(").Append(i).Append(")"); res = TDengine.Query(this.conn, sql.ToString()); - if (res != 0) + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } @@ -297,8 +297,8 @@ namespace TDengineDriver .Append(rows) .Append(", 5, 6, 7, 'abc', 'def')"); } - long res = TDengine.Query(conn, sql.ToString()); - if (res == 0) + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res == IntPtr.Zero) { Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); } @@ -326,14 +326,14 @@ namespace TDengineDriver System.DateTime start = new System.DateTime(); long queryRows = 0; - - for (int i = 0; i < this.tableCount; ++i) + + for (int i = 0; i < 1/*this.tableCount*/; ++i) { String sql = "select * from " + this.dbName + "." + tablePrefix + i; Console.WriteLine(sql); - long res = TDengine.Query(conn, sql); - if (res == 0) + IntPtr res = TDengine.Query(conn, sql); + if (res == IntPtr.Zero) { Console.WriteLine(sql + " failure, reason: " + TDengine.Error(res)); ExitProgram(); @@ -350,20 +350,21 @@ namespace TDengineDriver } IntPtr rowdata; + StringBuilder builder = new StringBuilder(); while ((rowdata = TDengine.FetchRows(res)) != IntPtr.Zero) { queryRows++; for (int fields = 0; fields < fieldCount; ++fields) { TDengineMeta meta = metas[fields]; - int offset = 8 * fields; + int offset = IntPtr.Size * fields; IntPtr data = Marshal.ReadIntPtr(rowdata, offset); - //Console.Write("---"); + builder.Append("---"); if (data == IntPtr.Zero) { - //Console.Write("NULL"); + builder.Append("NULL"); continue; } @@ -371,47 +372,53 @@ namespace TDengineDriver { case TDengineDataType.TSDB_DATA_TYPE_BOOL: bool v1 = Marshal.ReadByte(data) == 0 ? false : true; - //Console.Write(v1); + builder.Append(v1); break; case TDengineDataType.TSDB_DATA_TYPE_TINYINT: byte v2 = Marshal.ReadByte(data); - //Console.Write(v2); + builder.Append(v2); break; case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: short v3 = Marshal.ReadInt16(data); - //Console.Write(v3); + builder.Append(v3); break; case TDengineDataType.TSDB_DATA_TYPE_INT: int v4 = Marshal.ReadInt32(data); - //Console.Write(v4); + builder.Append(v4); break; case TDengineDataType.TSDB_DATA_TYPE_BIGINT: long v5 = Marshal.ReadInt64(data); - //Console.Write(v5); + builder.Append(v5); break; case TDengineDataType.TSDB_DATA_TYPE_FLOAT: float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); - //Console.Write(v6); + builder.Append(v6); break; case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); - //Console.Write(v7); + builder.Append(v7); break; case TDengineDataType.TSDB_DATA_TYPE_BINARY: string v8 = Marshal.PtrToStringAnsi(data); - //Console.Write(v8); + builder.Append(v8); break; case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: long v9 = Marshal.ReadInt64(data); - //Console.Write(v9); + builder.Append(v9); break; case TDengineDataType.TSDB_DATA_TYPE_NCHAR: string v10 = Marshal.PtrToStringAnsi(data); - //Console.Write(v10); + builder.Append(v10); break; } } - //Console.WriteLine("---"); + builder.Append("---"); + + if (queryRows <= 10) + { + Console.WriteLine(builder.ToString()); + } + builder.Clear(); } if (TDengine.ErrorNo(res) != 0) @@ -431,9 +438,9 @@ namespace TDengineDriver public void CloseConnection() { - if (conn != 0) + if (this.conn != IntPtr.Zero) { - TDengine.Close(conn); + TDengine.Close(this.conn); } } From 6e43fe5217e9aea45ece66d659dd1e61c0c7a761 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Aug 2020 18:24:25 +0800 Subject: [PATCH 072/111] [td-621] --- src/client/src/tscSubquery.c | 10 +++++++++- src/client/src/tscUtil.c | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1c7047996c..e222c9748c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1912,6 +1912,10 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { assert(size > 0); pSql->pSubs = calloc(size, POINTER_BYTES); + if (pSql->pSubs == NULL) { + goto _error; + } + pSql->numOfSubs = (uint16_t)size; tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); @@ -1925,6 +1929,10 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); + if (pSupporter == NULL) { + goto _error; + } + pSupporter->pSql = pSql; pSupporter->pState = pState; pSupporter->index = numOfSub; @@ -1957,7 +1965,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { if (numOfSub < pSql->numOfSubs) { tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; // free all allocated resource + goto _error; } pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9f23ac2bdc..6dd8973f65 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1526,8 +1526,11 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + if (pQueryInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tscInitQueryInfo(pQueryInfo); - pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; From dd1278c6f9d3f6f18b205a178bc867f5d5459b6c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 26 Aug 2020 19:08:07 +0800 Subject: [PATCH 073/111] TD-1090 remove version.c and add version to taos.dll --- cmake/version.inc | 42 +++++++++++++++++++++++++++++++++++++++ src/client/CMakeLists.txt | 6 ++++-- src/client/src/taos.rc.in | 31 +++++++++++++++++++++++++++++ src/util/src/version.c | 7 ------- src/util/src/version.c.in | 7 +++++++ 5 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 cmake/version.inc create mode 100644 src/client/src/taos.rc.in delete mode 100644 src/util/src/version.c create mode 100644 src/util/src/version.c.in diff --git a/cmake/version.inc b/cmake/version.inc new file mode 100644 index 0000000000..8c02520142 --- /dev/null +++ b/cmake/version.inc @@ -0,0 +1,42 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +SET(TD_VER_1 "2") +SET(TD_VER_2 "0") +SET(TD_VER_3 "2") +SET(TD_VER_4 "3") +SET(TD_VER_GIT "d711657139620f6c50f362597020705b8ad26bd2") +SET(TD_VER_GIT_INTERNAL "1d74ae24c541ffbb280e8630883c0236cd45f8c7") + +SET(TD_VER_VERTYPE "stable") +SET(TD_VER_CPUTYPE "x64") +SET(TD_VER_OSTYPE "Linux") + +SET(TD_VER_COMPATIBLE "2.0.0.0") +STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") + +IF (TD_LINUX_64) + SET(TD_VER_CPUTYPE "x64") +ENDIF () + +IF (TD_LINUX_32) + SET(TD_VER_CPUTYPE "x86") +ENDIF () + +IF (TD_ARM_64) + SET(TD_VER_CPUTYPE "aarch64") +ENDIF () + +IF (TD_ARM_32) + SET(TD_VER_CPUTYPE "aarch32") +ENDIF () + +IF (TD_WINDOWS_64) + SET(TD_VER_CPUTYPE "x64") +ENDIF () + +IF (TD_WINDOWS_32) + SET(TD_VER_CPUTYPE "x86") +ENDIF () + +CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/util/src/version.c.in" "${TD_COMMUNITY_DIR}/src/util/src/version.c") diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index d5cbf3cab6..5b5fb3435d 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -35,12 +35,14 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) - + + CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/client/src/taos.rc.in" "${TD_COMMUNITY_DIR}/src/client/src/taos.rc") + ADD_LIBRARY(taos_static STATIC ${SRC}) TARGET_LINK_LIBRARIES(taos_static trpc tutil query) # generate dynamic library (*.dll) - ADD_LIBRARY(taos SHARED ${SRC}) + ADD_LIBRARY(taos SHARED ${SRC} ${TD_COMMUNITY_DIR}/src/client/src/taos.rc) IF (NOT TD_GODLL) SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) ENDIF () diff --git a/src/client/src/taos.rc.in b/src/client/src/taos.rc.in new file mode 100644 index 0000000000..05dbd9bb7d --- /dev/null +++ b/src/client/src/taos.rc.in @@ -0,0 +1,31 @@ +1 VERSIONINFO + FILEVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} + PRODUCTVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} + FILEFLAGSMASK 0x17L +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x0L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "Native C Driver for TDengine" + VALUE "FileVersion", "${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3}" + VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" + VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" + VALUE "OriginalFilename", "" + VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" + VALUE "ProductVersion", "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END \ No newline at end of file diff --git a/src/util/src/version.c b/src/util/src/version.c deleted file mode 100644 index 6f7b29ebb9..0000000000 --- a/src/util/src/version.c +++ /dev/null @@ -1,7 +0,0 @@ -char version[12] = "2.0.2.0"; -char compatible_version[12] = "2.0.0.0"; -char gitinfo[48] = "d711657139620f6c50f362597020705b8ad26bd2"; -char gitinfoOfInternal[48] = "1d74ae24c541ffbb280e8630883c0236cd45f8c7"; -char buildinfo[64] = "Built by root at 2020-08-24 16:31"; - -void libtaos_2_0_2_0_Linux_x64_beta() {}; diff --git a/src/util/src/version.c.in b/src/util/src/version.c.in new file mode 100644 index 0000000000..c7aea2afb1 --- /dev/null +++ b/src/util/src/version.c.in @@ -0,0 +1,7 @@ +char version[12] = "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}"; +char compatible_version[12] = "${TD_VER_COMPATIBLE}"; +char gitinfo[48] = "${TD_VER_GIT}"; +char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; +char buildinfo[64] = "Built at ${TD_VER_DATE}"; + +void libtaos_${TD_VER_1}_${TD_VER_2}_${TD_VER_3}_${TD_VER_4}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; From 9f8e81a5de4adcb778e039b1d0ca07ade8e2a15d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 26 Aug 2020 19:31:40 +0800 Subject: [PATCH 074/111] minor changes --- .gitignore | 2 ++ CMakeLists.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c5d90eea50..e6e327327c 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ CMakeError.log /test/cfg /src/.vs *.o +version.c +taos.rc src/connector/jdbc/.settings/ tests/comparisonTest/cassandra/cassandratest/.classpath tests/comparisonTest/cassandra/cassandratest/.project diff --git a/CMakeLists.txt b/CMakeLists.txt index bfb8e902a8..565ab32f00 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) INCLUDE(cmake/define.inc) INCLUDE(cmake/env.inc) +INCLUDE(cmake/version.inc) INCLUDE(cmake/install.inc) ADD_SUBDIRECTORY(deps) From 942c70c5e0360b5e23005c5295c5367e42b526d8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 26 Aug 2020 13:27:43 +0000 Subject: [PATCH 075/111] minor changes --- src/client/src/tscSql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1bd885466c..de998a6b59 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -201,7 +201,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db); + tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port != 0 ? port : tsServerPort , user, db); if (user == NULL) user = TSDB_DEFAULT_USER; if (pass == NULL) pass = TSDB_DEFAULT_PASS; From a393cc7524a56a2e16ac6935116d034a426506f7 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 27 Aug 2020 09:26:31 +0800 Subject: [PATCH 076/111] fix td-1242 --- src/dnode/src/dnodeSystem.c | 4 ++-- src/sync/src/tarbitrator.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 6f32bc0f7a..b286c0f219 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -120,8 +120,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { - if (res != EINTR) { - syslog(LOG_ERR, "failed to wait exit semphore: %d", res); + if (errno != EINTR) { + syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); break; } } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 625c0d6838..79ddb073de 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort); for (int res = tsem_wait(&tsArbSem); res != 0; res = tsem_wait(&tsArbSem)) { - if (res != EINTR) break; + if (errno != EINTR) break; } taosCloseTcpThreadPool(tsArbTcpPool); From 829caf39d38be113774ae236ba737c3d902cff52 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 27 Aug 2020 10:07:58 +0800 Subject: [PATCH 077/111] td-1242: wait again if sem_wait returns EINTR --- src/dnode/src/dnodeSystem.c | 7 ++----- src/os/inc/osSemphone.h | 2 +- src/os/src/detail/osSemphone.c | 12 ++++++++++++ src/sync/src/tarbitrator.c | 4 +--- src/util/src/tsched.c | 13 ++----------- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index b286c0f219..56316e9619 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -119,11 +119,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); - for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { - if (errno != EINTR) { - syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); - break; - } + if (tsem_wait(&exitSem) != 0) { + syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); } dnodeCleanUpSystem(); diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index fd88d2d798..4280b458a6 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -23,7 +23,7 @@ extern "C" { #ifndef TAOS_OS_FUNC_SEMPHONE #define tsem_t sem_t #define tsem_init sem_init - #define tsem_wait sem_wait + int tsem_wait(tsem_t* sem); #define tsem_post sem_post #define tsem_destroy sem_destroy #endif diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 74f8859029..b91888845e 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -16,6 +16,18 @@ #define _DEFAULT_SOURCE #include "os.h" +#ifndef TAOS_OS_FUNC_SEMPHONE + +int tsem_wait(tsem_t* sem) { + int ret = 0; + do { + ret = sem_wait(sem); + } while (ret != 0 && errno == EINTR); + return ret; +} + +#endif + #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 79ddb073de..eea46a2495 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -103,9 +103,7 @@ int main(int argc, char *argv[]) { sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort); - for (int res = tsem_wait(&tsArbSem); res != 0; res = tsem_wait(&tsArbSem)) { - if (errno != EINTR) break; - } + tsem_wait(&tsArbSem); taosCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down\n"); diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index cf7f5c10d4..f014dd0fab 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -123,11 +123,6 @@ void *taosProcessSchedQueue(void *param) { while (1) { if (tsem_wait(&pSched->fullSem) != 0) { - if (errno == EINTR) { - /* sem_wait is interrupted by interrupt, ignore and continue */ - uDebug("wait %s fullSem was interrupted", pSched->label); - continue; - } uError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); } if (pSched->stop) { @@ -163,12 +158,8 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { return 0; } - while (tsem_wait(&pSched->emptySem) != 0) { - if (errno != EINTR) { - uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); - break; - } - uDebug("wait %s emptySem was interrupted", pSched->label); + if (tsem_wait(&pSched->emptySem) != 0) { + uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) From 43bbbfd8669de91298f11a616312d8eb40937719 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 27 Aug 2020 03:58:53 +0000 Subject: [PATCH 078/111] Crash_gen tool minor tweaks, hoping to run 10x5000 with -a option --- tests/pytest/crash_gen.py | 55 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 7d3eb959c0..f67aeaf00e 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -161,6 +161,18 @@ class WorkerThread: logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...") break + # Before we fetch the task and run it, let's ensure we properly "use" the database + try: + self.useDb() # might encounter exceptions. TODO: catch + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno == 0x383 : # invalid database + # ignore + dummy = 0 + else: + print("\nCaught programming error. errno=0x{:X}, msg={} ".format(errno, err.msg)) + raise + # Fetch a task from the Thread Coordinator logger.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid)) task = tc.fetchTask() @@ -324,10 +336,12 @@ class ThreadCoordinator: logger.debug("[STT] transition ended") # Due to limitation (or maybe not) of the Python library, # we cannot share connections across threads - if sm.hasDatabase(): - for t in self._pool.threadList: - logger.debug("[DB] use db for all worker threads") - t.useDb() + # Here we are in main thread, we cannot operate the connections created in workers + # Moving below to task loop + # if sm.hasDatabase(): + # for t in self._pool.threadList: + # logger.debug("[DB] use db for all worker threads") + # t.useDb() # t.execSql("use db") # main thread executing "use # db" on behalf of every worker thread except taos.error.ProgrammingError as err: @@ -387,7 +401,7 @@ class ThreadCoordinator: transitionFailed = self._doTransition() # To start, we end step -1 first except taos.error.ProgrammingError as err: transitionFailed = True - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme + errno2 = Helper.convertErrno(err.errno) # correct error scheme errMsg = "Transition failed: errno=0x{:X}, msg: {}".format(errno2, err) logger.info(errMsg) self._execStats.registerFailure(errMsg) @@ -468,6 +482,10 @@ class ThreadCoordinator: # We define a class to run a number of threads in locking steps. +class Helper: + @classmethod + def convertErrno(cls, errno): + return errno if (errno > 0) else 0x80000000 + errno class ThreadPool: def __init__(self, numThreads, maxSteps): @@ -613,8 +631,7 @@ class DbConn: def resetDb(self): # reset the whole database, etc. if (not self.isOpen): - raise RuntimeError( - "Cannot reset database until connection is open") + raise RuntimeError("Cannot reset database until connection is open") # self._tdSql.prepare() # Recreate database, etc. self.execute('drop database if exists db') @@ -681,8 +698,7 @@ class DbConnRest(DbConn): def close(self): if (not self.isOpen): - raise RuntimeError( - "Cannot clean up database until connection is open") + raise RuntimeError("Cannot clean up database until connection is open") # Do nothing for REST logger.debug("[DB] REST Database connection closed") self.isOpen = False @@ -796,8 +812,7 @@ class DbConnNative(DbConn): super().__init__() self._type = self.TYPE_NATIVE self._conn = None - self._cursor = None - + self._cursor = None def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -814,7 +829,7 @@ class DbConnNative(DbConn): buildPath = root[:len(root) - len("/build/bin")] break if buildPath == None: - raise RuntimeError("Failed to determine buildPath, selfPath={}".format(selfPath)) + raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}".format(selfPath, projPath)) return buildPath @@ -839,8 +854,7 @@ class DbConnNative(DbConn): def close(self): if (not self.isOpen): - raise RuntimeError( - "Cannot clean up database until connection is open") + raise RuntimeError("Cannot clean up database until connection is open") self._tdSql.close() logger.debug("[DB] Database connection closed") self.isOpen = False @@ -1528,7 +1542,7 @@ class Task(): try: self._executeInternal(te, wt) # TODO: no return value? except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme + errno2 = Helper.convertErrno(err.errno) if (gConfig.continue_on_exception): # user choose to continue self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format( errno2, err, wt.getDbConn().getLastSql())) @@ -1789,7 +1803,7 @@ class TdSuperTable: try: dbc.query("select TBNAME from db.{}".format(self._stName)) # TODO: analyze result set later except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno + errno2 = Helper.convertErrno(err.errno) logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err)) raise @@ -1891,7 +1905,7 @@ class TaskReadData(StateTransitionTask): if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?! dbc.execute("select {} from db.{}".format(aggExpr, sTable.getName())) except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno + errno2 = Helper.convertErrno(err.errno) logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql())) raise @@ -1920,9 +1934,8 @@ class TaskDropSuperTable(StateTransitionTask): self.execWtSql(wt, "drop table {}".format( regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = err.errno if ( - err.errno > 0) else 0x80000000 + err.errno + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) if (errno2 in [0x362]): # mnode invalid table name isSuccess = False logger.debug( @@ -2437,7 +2450,7 @@ class ServiceManagerThread: if self._status == MainExec.STATUS_STARTING: # we are starting, let's see if we have started if line.find(self.TD_READY_MSG) != -1: # found logger.info("Waiting for the service to become FULLY READY") - time.sleep(1.0) # wait for the server to truly start. TODO: remove this + time.sleep(5.0) # wait for the server to truly start. TODO: remove this logger.info("Service is now FULLY READY") self._status = MainExec.STATUS_RUNNING From 2b74699d3592ca1a521fae5ff6f0ca90a1fa421c Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 27 Aug 2020 04:47:43 +0000 Subject: [PATCH 079/111] Adjusted crash_gen tool to tolerate non UTF8 server output --- tests/pytest/crash_gen.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index f67aeaf00e..fd48edae05 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -166,7 +166,7 @@ class WorkerThread: self.useDb() # might encounter exceptions. TODO: catch except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno == 0x383 : # invalid database + if errno in [0x383, 0x386, 0x00B, 0x014] : # invalid database, dropping, Unable to establish connection, Database not ready # ignore dummy = 0 else: @@ -2442,7 +2442,11 @@ class ServiceManagerThread: for line in iter(out.readline, b''): # print("Finished reading a line: {}".format(line)) # print("Adding item to queue...") - line = line.decode("utf-8").rstrip() + try: + line = line.decode("utf-8").rstrip() + except UnicodeError: + print("\nNon-UTF8 server output: {}\n".format(line)) + # This might block, and then causing "out" buffer to block queue.put(line) self._printProgress("_i") From 4d122b7adbd486d2ffe2d0292061060af8a7440a Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 27 Aug 2020 05:04:27 +0000 Subject: [PATCH 080/111] Adjusted crash_gen tool to re-connect after server restart disruption --- tests/pytest/crash_gen.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index fd48edae05..3ffe77c6a4 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -163,6 +163,9 @@ class WorkerThread: # Before we fetch the task and run it, let's ensure we properly "use" the database try: + if (gConfig.per_thread_db_connection): # most likely TRUE + if not self._dbConn.isOpen: # might have been closed during server auto-restart + self._dbConn.open() self.useDb() # might encounter exceptions. TODO: catch except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) @@ -861,8 +864,7 @@ class DbConnNative(DbConn): def execute(self, sql): if (not self.isOpen): - raise RuntimeError( - "Cannot execute database commands until connection is open") + raise RuntimeError("Cannot execute database commands until connection is open") logger.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql nRows = self._tdSql.execute(sql) @@ -2454,7 +2456,7 @@ class ServiceManagerThread: if self._status == MainExec.STATUS_STARTING: # we are starting, let's see if we have started if line.find(self.TD_READY_MSG) != -1: # found logger.info("Waiting for the service to become FULLY READY") - time.sleep(5.0) # wait for the server to truly start. TODO: remove this + time.sleep(1.0) # wait for the server to truly start. TODO: remove this logger.info("Service is now FULLY READY") self._status = MainExec.STATUS_RUNNING From 158f7b2ed607696f4fdf92c9f28300060217306d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 27 Aug 2020 05:42:02 +0000 Subject: [PATCH 081/111] TD-1201 change filesize from 32bits to 64bits --- src/inc/tsdb.h | 2 +- src/inc/tsync.h | 2 +- src/mnode/src/mnodeSdb.c | 2 +- src/sync/inc/syncInt.h | 2 +- src/sync/src/syncRestore.c | 4 ++-- src/sync/src/syncRetrieve.c | 2 +- src/sync/test/syncServer.c | 2 +- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbFile.c | 4 ++-- src/tsdb/src/tsdbMain.c | 2 +- src/util/inc/tkvstore.h | 2 +- src/util/src/tkvstore.c | 4 ++-- src/vnode/src/vnodeMain.c | 4 ++-- 13 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 4776d1cda7..be73d8d383 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -115,7 +115,7 @@ int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); +uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); // the TSDB repository info typedef struct STsdbRepoInfo { diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 2dfac7ec32..ff9c9901bd 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -64,7 +64,7 @@ typedef struct { if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. */ -typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); +typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 3651aa8aad..9efb804734 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -224,7 +224,7 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeEpSet(); } -static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { +static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { sdbUpdateMnodeRoles(); return 0; } diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index be1e01cb23..cd1252f4b4 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -74,7 +74,7 @@ typedef struct { uint32_t magic; uint32_t index; uint64_t fversion; - int32_t size; + int64_t size; } SFileInfo; typedef struct { diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 505882b209..5c9b734365 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -28,7 +28,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind char fname[TSDB_FILENAME_LEN*3] = {0}; uint32_t magic; uint64_t fversion; - int32_t size; + int64_t size; uint32_t index = sindex; SSyncNode *pNode = pPeer->pSyncNode; @@ -113,7 +113,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) close(dfd); if (ret<0) break; - sDebug("%s, %s is received, size:%d", pPeer->id, minfo.name, minfo.size); + sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 1dd1cda343..0cc3e7326a 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -154,7 +154,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) close(sfd); if (ret <0) break; - sDebug("%s, %s is sent, size:%d", pPeer->id, name, fileInfo.size); + sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); fileInfo.index++; // check if processed files are modified diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 44019ad96e..b802905038 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -234,7 +234,7 @@ void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); } -uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) +uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { uint32_t magic; struct stat fstat; diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 7936ea423f..e7a86798ee 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -475,7 +475,7 @@ int tsdbUpdateFileHeader(SFile* pFile); int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); -void tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int32_t* size); +void tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size); void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); // ------------------ tsdbRWHelper.c diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 154d728024..29e46a88af 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -424,7 +424,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { } } -void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int32_t *size) { +void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) { char buf[TSDB_FILE_HEAD_SIZE] = "\0"; uint32_t version = 0; STsdbFileInfo info = {0}; @@ -445,7 +445,7 @@ void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int32_t *size) { close(fd); *magic = info.magic; - *size = (int32_t)offset; + *size = offset; return; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 827e5c3c6d..ea912ad1f4 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -212,7 +212,7 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * return 0; } -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) { +uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { STsdbRepo *pRepo = (STsdbRepo *)repo; // STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h index 3b4e8e3757..b2b0ff05f5 100644 --- a/src/util/inc/tkvstore.h +++ b/src/util/inc/tkvstore.h @@ -58,7 +58,7 @@ int tdKVStoreStartCommit(SKVStore *pStore); int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen); int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid); int tdKVStoreEndCommit(SKVStore *pStore); -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size); +void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size); #ifdef __cplusplus } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 696819e5d0..6ba1d87d92 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -332,7 +332,7 @@ int tdKVStoreEndCommit(SKVStore *pStore) { return 0; } -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { +void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; SStoreInfo info = {0}; @@ -349,7 +349,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { close(fd); *magic = info.magic; - *size = (int32_t)offset; + *size = offset; return; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3c9da7fd23..8a1880b36e 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -41,7 +41,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode); static int32_t vnodeSaveVersion(SVnodeObj *pVnode); static int32_t vnodeReadVersion(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeCtrlFlow(void *handle, int32_t mseconds); @@ -536,7 +536,7 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { SVnodeObj *pVnode = ahandle; *fversion = pVnode->fversion; return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); From 65f750a087e18b4ffcffdc3379cacb97c0994e5a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 27 Aug 2020 06:47:59 +0000 Subject: [PATCH 082/111] fixbug connection killing led floating point exception --- src/client/src/tscProfile.c | 2 +- src/client/src/tscServer.c | 10 +++++++--- src/inc/taoserror.h | 1 + src/mnode/src/mnodeProfile.c | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f48e7b7691..6ff97e9d00 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -285,9 +285,9 @@ void tscKillConnection(STscObj *pObj) { SSqlObj *pSql = pObj->sqlList; while (pSql) { - //taosStopRpcConn(pSql->thandle); pSql = pSql->next; } + SSqlStream *pStream = pObj->streamList; while (pStream) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9282fa74fb..4f179adf72 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -226,13 +226,17 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = &pSql->pRpcCtx, .code = 0 }; - // NOTE: the rpc context should be acquired before sending data to server. // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); - return TSDB_CODE_SUCCESS; + if (pObj != NULL && pObj->signature == pObj) { + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + return TSDB_CODE_SUCCESS; + } else { + //pObj->signature has been reset by other thread, ignore concurrency problem + return TSDB_CODE_TSC_CONN_KILLED; + } } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 57c2b322fa..e2ba7b03c1 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -96,6 +96,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Applicatio TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 85457d7a26..06f992c26a 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -100,7 +100,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { }; tstrncpy(connObj.user, user, sizeof(connObj.user)); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME * 1000); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; From 13cd54637b12aae0001dd69ba58c74ab5e42ef08 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 14:50:53 +0800 Subject: [PATCH 083/111] [td-255] fix compiler error --- src/client/src/tscSubquery.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e222c9748c..eaae026f06 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1911,13 +1911,12 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); + pSql->numOfSubs = (uint16_t)size; pSql->pSubs = calloc(size, POINTER_BYTES); if (pSql->pSubs == NULL) { goto _error; } - pSql->numOfSubs = (uint16_t)size; - tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); From d5abee89dde1bef4d3ff20dcfdc4966bc8154ab2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 15:04:19 +0800 Subject: [PATCH 084/111] [td-255] fix compiler error --- src/client/src/tscSubquery.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index eaae026f06..011940a0e6 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1911,6 +1911,9 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); + // the number of already initialized subqueries + int32_t numOfSub = 0; + pSql->numOfSubs = (uint16_t)size; pSql->pSubs = calloc(size, POINTER_BYTES); if (pSql->pSubs == NULL) { @@ -1924,7 +1927,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pState->numOfRemain = pSql->numOfSubs; pRes->code = TSDB_CODE_SUCCESS; - int32_t numOfSub = 0; while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); From 460ae42c096e29de699c311c7d0ffe22ea65cd0a Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 27 Aug 2020 16:20:48 +0800 Subject: [PATCH 085/111] [TD-1203] --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 6 +++++- src/sync/src/syncMain.c | 8 +++++--- src/sync/src/tarbitrator.c | 4 ---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 4ffc631566..ef0713c415 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -29,6 +29,7 @@ extern uint16_t tsServerPort; extern uint16_t tsDnodeShellPort; extern uint16_t tsDnodeDnodePort; extern uint16_t tsSyncPort; +extern uint16_t tsArbitratorPort; extern int32_t tsStatusInterval; extern int32_t tsNumOfMnodes; extern int32_t tsEnableVnodeBak; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index eb6fcde385..795585e5c9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -37,6 +37,7 @@ uint16_t tsServerPort = 6030; uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035] uint16_t tsDnodeDnodePort = 6035; // udp/tcp uint16_t tsSyncPort = 6040; +uint16_t tsArbitratorPort = 6042; int32_t tsStatusInterval = 1; // second int32_t tsNumOfMnodes = 3; int32_t tsEnableVnodeBak = 1; @@ -1331,7 +1332,10 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { *port = atoi(temp+1); } - if (*port == 0) *port = tsServerPort; + if (*port == 0) { + *port = tsServerPort; + return -1; + } return 0; } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index eaa073348a..b6babd41eb 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -413,9 +413,11 @@ static void syncAddArbitrator(SSyncNode *pNode) SNodeInfo nodeInfo; nodeInfo.nodeId = 0; - taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); - nodeInfo.nodePort += TSDB_PORT_SYNC; - + int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); + if (-1 == ret) { + nodeInfo.nodePort = tsArbitratorPort; + } + if (pPeer) { if ((strcmp(nodeInfo.nodeFqdn, pPeer->fqdn) == 0) && (nodeInfo.nodePort == pPeer->port)) { return; diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 625c0d6838..4e55cc475f 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -40,13 +40,9 @@ typedef struct { void *pConn; } SNodeConn; -uint16_t tsArbitratorPort = 0; - int main(int argc, char *argv[]) { char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; - tsArbitratorPort = tsServerPort + TSDB_PORT_ARBITRATOR; - for (int i=1; i Date: Thu, 27 Aug 2020 16:28:26 +0800 Subject: [PATCH 086/111] Update mnodeProfile.c Update the mnode connection activity time --- src/mnode/src/mnodeProfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 85457d7a26..34147f2e4b 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -100,7 +100,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { }; tstrncpy(connObj.user, user, sizeof(connObj.user)); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME*1000); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; From 76ad0bc70e4bc6f74d09c843e765cfd56970fdbe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 18:32:34 +0800 Subject: [PATCH 087/111] [td-255] add check for memory allocation failure. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 34 +++++++++ src/client/src/tscParseInsert.c | 13 ++-- src/client/src/tscSQLParser.c | 39 +++++++--- src/client/src/tscServer.c | 15 ++-- src/client/src/tscSql.c | 7 +- src/client/src/tscSubquery.c | 13 ++++ src/client/src/tscUtil.c | 107 ++++++++++++++++++--------- src/query/src/qParserImpl.c | 10 ++- tests/script/general/parser/join.sim | 4 +- 10 files changed, 174 insertions(+), 70 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 42959ade3c..228e54a1c7 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -217,7 +217,7 @@ STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); -int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); +SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 004c5bc21c..c59f305b58 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -35,6 +35,40 @@ extern "C" { #include "qTsbuf.h" #include "tcmdtype.h" +#if 0 +static UNUSED_FUNC void *u_malloc (size_t __size) { + uint32_t v = rand(); + + if (v % 5000 <= 0) { + return NULL; + } else { + return malloc(__size); + } +} + +static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return calloc(num, __size); + } +} + +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return realloc(p, __size); + } +} + +#define calloc u_calloc +#define malloc u_malloc +#define realloc u_realloc +#endif + // forward declaration struct SSqlInfo; struct SLocalReducer; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 939ebb6abe..47bfe0fcdc 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1031,11 +1031,11 @@ int tsParseInsertSql(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); assert(pQueryInfo != NULL); - STableMetaInfo *pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = terrno; + return code; } if ((code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -1292,8 +1292,7 @@ int tsInsertInitialCheck(SSqlObj *pSql) { pCmd->command = TSDB_SQL_INSERT; pSql->res.numOfRows = 0; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e9fc27bb4e..5600ee3be9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -179,20 +179,24 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_TSC_APP_ERROR; } - SSqlCmd* pCmd = &(pSql->cmd); - SQueryInfo* pQueryInfo = NULL; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), pInfo->pzErrMsg); } - int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pRes->code = terrno; + return pRes->code; + } - STableMetaInfo* pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + STableMetaInfo* pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo) : pQueryInfo->pTableMetaInfo[0]; + if (pTableMetaInfo == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; } pCmd->command = pInfo->type; @@ -487,9 +491,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg1 = "columns in select clause not identical"; for (int32_t i = pCmd->numOfClause; i < pInfo->subclauseInfo.numOfClause; ++i) { - SQueryInfo* pqi = NULL; - if ((code = tscGetQueryInfoDetailSafely(pCmd, i, &pqi)) != TSDB_CODE_SUCCESS) { - return code; + SQueryInfo* pqi = tscGetQueryInfoDetailSafely(pCmd, i); + if (pqi == NULL) { + pRes->code = terrno; + return pRes->code; } } @@ -2678,9 +2683,12 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { } int32_t size = pColumn->numOfFilters + 1; - char* tmp = (char*)realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); + + char* tmp = (char*) realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; + } else { + return NULL; } pColumn->numOfFilters++; @@ -2964,9 +2972,16 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } else { // update the existed column filter information, find the filter info here pColFilter = &pColumn->filterInfo[0]; } + + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else if (sqlOptr == TK_OR) { // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" pColFilter = addColumnFilterInfo(pColumn); + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else { // error; return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7903ff9562..6cdb39040c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1956,8 +1956,12 @@ static void createHBObj(STscObj* pObj) { pSql->fp = tscProcessHeartBeatRsp; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pSql->cmd, 0, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return; + } + pQueryInfo->command = TSDB_SQL_HB; pSql->cmd.command = pQueryInfo->command; @@ -2142,8 +2146,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf tscAddSubqueryInfo(&pNew->cmd); - SQueryInfo *pNewQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo); + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { @@ -2246,8 +2249,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - SQueryInfo *pNewQueryInfo = NULL; - if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); + if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); return code; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 8d33368280..e5db752458 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -827,8 +827,11 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; char *str = (char *)tblNameList; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return terrno; + } STableMetaInfo *pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 011940a0e6..2fb264c756 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -437,6 +437,7 @@ int32_t tscCompareTidTags(const void* p1, const void* p2) { if (t1->vgId != t2->vgId) { return (t1->vgId > t2->vgId) ? 1 : -1; } + if (t1->tid != t2->tid) { return (t1->tid > t2->tid) ? 1 : -1; } @@ -543,6 +544,7 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, for(int32_t i = 1; i < p1->num; ++i) { STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); + assert(prev->vgId >= 1 && p->vgId >= 1); if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { tscError("%p join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj); @@ -579,6 +581,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar while(i < p1->num && j < p2->num) { STidTags* pp1 = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); STidTags* pp2 = (STidTags*) varDataVal(p2->pIdTagList + j * p2->tagSize); + assert(pp1->tid != 0 && pp2->tid != 0); int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes); if (ret == 0) { @@ -1221,6 +1224,16 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + // get the tag colId column index + int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + for(int32_t i = 0; i < numOfTags; ++i) { + if (pSchema[i].colId == tagColId) { + index.columnIndex = i; + break; + } + } + int16_t bytes = 0; int16_t type = 0; int32_t inter = 0; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6dd8973f65..5bb6d3743d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -486,15 +486,6 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { return TSDB_CODE_SUCCESS; } -//void tscFreeUnusedDataBlocks(SDataBlockList* pList) { -// /* release additional memory consumption */ -// for (int32_t i = 0; i < pList->nSize; ++i) { -// STableDataBlocks* pDataBlock = pList->pData[i]; -// pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size); -// pDataBlock->nAllocSize = (uint32_t)pDataBlock->size; -// } -//} - /** * create the in-memory buffer for each table to keep the submitted data block * @param initialSize @@ -519,6 +510,11 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff } dataBuf->pData = calloc(1, dataBuf->nAllocSize); + if (dataBuf->pData == NULL) { + tscError("failed to allocated memory, reason:%s", strerror(errno)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; @@ -931,8 +927,12 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); + if (pExpr == NULL) { + return NULL; + } + pExpr->functionId = functionId; - + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; @@ -1063,8 +1063,11 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) if (deepcopy) { SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + if (p1 == NULL) { + assert(0); + } + *p1 = *pExpr; - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { tVariantAssign(&p1->param[j], &pExpr->param[j]); } @@ -1100,16 +1103,22 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { if (i >= numOfCols || numOfCols == 0) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } else { SColumn* pCol = taosArrayGetP(pColumnList, i); if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } } @@ -1131,7 +1140,10 @@ SColumn* tscColumnClone(const SColumn* src) { assert(src != NULL); SColumn* dst = calloc(1, sizeof(SColumn)); - + if (dst == NULL) { + return NULL; + } + dst->colIndex = src->colIndex; dst->numOfFilters = src->numOfFilters; dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); @@ -1341,6 +1353,10 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { if (pCond->len > 0) { assert(pCond->cond != NULL); c.cond = malloc(c.len); + if (c.cond == NULL) { + assert(0); + } + memcpy(c.cond, pCond->cond, c.len); } @@ -1466,20 +1482,20 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { return pQueryInfo->pTableMetaInfo[tableIndex]; } -int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo) { +SQueryInfo* tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); int32_t ret = TSDB_CODE_SUCCESS; - *pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); - - while ((*pQueryInfo) == NULL) { + while ((pQueryInfo) == NULL) { if ((ret = tscAddSubqueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - return ret; + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; } - (*pQueryInfo) = tscGetQueryInfoDetail(pCmd, subClauseIndex); + pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); } - return TSDB_CODE_SUCCESS; + return pQueryInfo; } STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { @@ -1591,14 +1607,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST SVgroupsInfo* vgroupList, SArray* pTagCols) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } pQueryInfo->pTableMetaInfo = pAlloc; - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = calloc(1, sizeof(STableMetaInfo)); + STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables]; - assert(pTableMetaInfo != NULL); + pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; if (name != NULL) { tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name)); @@ -1609,10 +1629,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST if (vgroupList != NULL) { size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; pTableMetaInfo->vgroupList = malloc(size); + if (pTableMetaInfo->vgroupList == NULL) { + return NULL; + } + memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + if (pTableMetaInfo->tagColList == NULL) { + return NULL; + } + if (pTagCols != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } @@ -1678,8 +1706,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return NULL; } - SQueryInfo* pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); @@ -1777,10 +1804,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - - free(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SSqlCmd* pnCmd = &pNew->cmd; @@ -1797,9 +1822,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->parseFinished = 1; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pnCmd, 0); @@ -1824,20 +1848,28 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } } tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->fillType != TSDB_FILL_NONE) { pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + if (pNewQueryInfo->fillVal == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); @@ -1880,16 +1912,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pFinalInfo->pTableMeta == NULL) { tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); - tscFreeSqlObj(pNew); - if (pPrevSql != NULL) { + if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); terrno = pPrevSql->res.code; } else { terrno = TSDB_CODE_TSC_APP_ERROR; } - return NULL; + goto _error; } assert(pNewQueryInfo->numOfTables == 1); @@ -1914,6 +1945,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } return pNew; + +_error: + tscFreeSqlObj(pNew); + return NULL; } /** diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 733459c6fe..c09e244643 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -179,10 +179,12 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - char* endPos = pRight->token.z + pRight->token.n; - pExpr->token.z = pLeft->token.z; - pExpr->token.n = endPos - pExpr->token.z; - pExpr->token.type = pLeft->token.type; + if (pRight != NULL && pLeft != NULL) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = endPos - pExpr->token.z; + pExpr->token.type = pLeft->token.type; + } if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { /* diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index c4e490190d..1bb0ff5448 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -330,8 +330,8 @@ sql_error select join_tb1.* from $tb1 , $tb2 where join_tb1.ts != join_tb0.ts an sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts and join_tb1.ts >= 100000; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by ts; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by join_tb1.c7; -sql_error select * from $tb1, $tb2; -sql_error select last_row(*) from $tb1, $tb2 +sql_error select * from join_tb0, join_tb1 +sql_error select last_row(*) from join_tb0, join_tb1 sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts < now sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts = join_tb2.ts From fdbd8fe09970d7261247a478686361e0b5502fff Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 18:40:23 +0800 Subject: [PATCH 088/111] [td-255] update the sim --- tests/script/general/parser/nchar.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 8d4e468f0e..bdac5ace55 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -223,6 +223,7 @@ if $rows != 2 then endi print data00 = $data00 if $data00 != 5 then + print expect 5, actual: $data00 return -1 endi if $data10 != 5 then From df013fc336336230391013fdc635a9dbdeca809d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 27 Aug 2020 11:13:20 +0000 Subject: [PATCH 089/111] TD-857 --- src/util/src/tcompare.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 9564588254..ba711ced8f 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -220,8 +220,14 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { char pattern[128] = {0}; memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); assert(varDataLen(pRight) < 128); - - int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } From 37c2d3eb7d6c426d9c51878c54aae7d7bd48ea8b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 27 Aug 2020 13:06:19 +0000 Subject: [PATCH 090/111] minor changes --- src/sync/src/syncMain.c | 170 +++++++++++++++--------------------- src/sync/src/syncRestore.c | 29 +++--- src/sync/src/syncRetrieve.c | 90 +++++++++---------- src/sync/src/taosTcpPool.c | 14 ++- src/sync/src/tarbitrator.c | 7 +- src/vnode/src/vnodeMain.c | 2 + 6 files changed, 130 insertions(+), 182 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index eaa073348a..8e7f18dc3b 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -108,8 +108,7 @@ static void syncModuleInitFunc() { tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn)); } -void *syncStart(const SSyncInfo *pInfo) -{ +void *syncStart(const SSyncInfo *pInfo) { const SSyncCfg *pCfg = &pInfo->syncCfg; SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1); @@ -189,9 +188,8 @@ void *syncStart(const SSyncInfo *pInfo) return pNode; } -void syncStop(void *param) -{ - SSyncNode *pNode = param; +void syncStop(void *param) { + SSyncNode * pNode = param; SSyncPeer *pPeer; if (pNode == NULL) return; @@ -215,9 +213,8 @@ void syncStop(void *param) syncDecNodeRef(pNode); } -int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) -{ - SSyncNode *pNode = param; +int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { + SSyncNode * pNode = param; int i, j; if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; @@ -283,10 +280,9 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) return 0; } -int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) -{ - SSyncNode *pNode = param; - SSyncPeer *pPeer; +int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) { + SSyncNode * pNode = param; + SSyncPeer * pPeer; SSyncHead *pSyncHead; SWalHead *pWalHead = data; int fwdLen; @@ -334,9 +330,8 @@ int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) return code; } -void syncConfirmForward(void *param, uint64_t version, int32_t code) -{ - SSyncNode *pNode = param; +void syncConfirmForward(void *param, uint64_t version, int32_t code) { + SSyncNode *pNode = param; if (pNode == NULL) return; if (pNode->quorum <= 1) return; @@ -387,10 +382,9 @@ void syncRecover(void *param) { pthread_mutex_unlock(&(pNode->mutex)); } -int syncGetNodesRole(void *param, SNodesRole *pNodesRole) -{ +int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { SSyncNode *pNode = param; - + pNodesRole->selfIndex = pNode->selfIndex; for (int i=0; ireplica; ++i) { pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId; @@ -400,8 +394,7 @@ int syncGetNodesRole(void *param, SNodesRole *pNodesRole) return 0; } -static void syncAddArbitrator(SSyncNode *pNode) -{ +static void syncAddArbitrator(SSyncNode *pNode) { SSyncPeer *pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; // if not configured, return right away @@ -454,13 +447,11 @@ static void syncDecNodeRef(SSyncNode *pNode) } } -void syncAddPeerRef(SSyncPeer *pPeer) -{ +void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } -int syncDecPeerRef(SSyncPeer *pPeer) -{ +int syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { syncDecNodeRef(pPeer->pSyncNode); @@ -473,18 +464,16 @@ int syncDecPeerRef(SSyncPeer *pPeer) return 1; } -static void syncClosePeerConn(SSyncPeer *pPeer) -{ +static void syncClosePeerConn(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); taosClose(pPeer->syncFd); - if (pPeer->peerFd >=0) { + if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; taosFreeTcpConn(pPeer->pConn); } } -static void syncRemovePeer(SSyncPeer *pPeer) -{ +static void syncRemovePeer(SSyncPeer *pPeer) { sInfo("%s, it is removed", pPeer->id); pPeer->ip = 0; @@ -492,8 +481,7 @@ static void syncRemovePeer(SSyncPeer *pPeer) syncDecPeerRef(pPeer); } -static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) -{ +static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn); if (ip == -1) return NULL; @@ -523,25 +511,24 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) return pPeer; } -void syncBroadcastStatus(SSyncNode *pNode) -{ +void syncBroadcastStatus(SSyncNode *pNode) { SSyncPeer *pPeer; for (int i = 0; i < pNode->replica; ++i) { - if ( i == pNode->selfIndex ) continue; + if (i == pNode->selfIndex) continue; pPeer = pNode->peerInfo[i]; syncSendPeersStatusMsgToPeer(pPeer, 1); } -} +} static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int i = 0; i < pNode->replica; ++i) { pNode->peerInfo[i]->numOfRetrieves = 0; } - if (pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + if (pNode->notifyFlowCtrl) { + (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + } } static void syncChooseMaster(SSyncNode *pNode) { @@ -598,9 +585,9 @@ static void syncChooseMaster(SSyncNode *pNode) { } else { sDebug("vgId:%d, failed to choose master", pNode->vgId); } -} - -static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { +} + +static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { int onlineNum = 0; int index = -1; int replica = pNode->replica; @@ -617,7 +604,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { replica = pNode->replica + 1; } - if (onlineNum <= replica*0.5) { + if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; pNode->peerInfo[pNode->selfIndex]->role = nodeRole; @@ -625,13 +612,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { - for (int i=0; ireplica; ++i) { + for (int i = 0; i < pNode->replica; ++i) { SSyncPeer *pTemp = pNode->peerInfo[i]; - if ( pTemp->role != TAOS_SYNC_ROLE_MASTER ) continue; - if ( index < 0 ) { + if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; + if (index < 0) { index = i; - } else { // multiple masters, it shall not happen - if ( i == pNode->selfIndex ) { + } else { // multiple masters, it shall not happen + if (i == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); @@ -640,7 +627,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { } } - SSyncPeer *pMaster = (index>=0) ? pNode->peerInfo[index]:NULL; + SSyncPeer *pMaster = (index >= 0) ? pNode->peerInfo[index] : NULL; return pMaster; } @@ -649,7 +636,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) { int code = 0; if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { - sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); + sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; (*pNode->notifyRole)(pNode->ahandle, nodeRole); code = -1; @@ -658,13 +645,12 @@ static int syncValidateMaster(SSyncPeer *pPeer) { if ( i == pNode->selfIndex ) continue; syncRestartPeer(pNode->peerInfo[i]); } - } + } return code; } -static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) -{ +static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) { SSyncNode *pNode = pPeer->pSyncNode; int8_t peerOldRole = pPeer->role; int8_t selfOldRole = nodeRole; @@ -686,14 +672,14 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne if (syncValidateMaster(pPeer) < 0) return; if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) { - if ( nodeVersion < pMaster->version) { + if (nodeVersion < pMaster->version) { syncRequired = 1; } else { sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); } - } else if ( nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { + } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { // nodeVersion = pMaster->version; } } else { @@ -734,20 +720,18 @@ static void syncRestartPeer(SSyncPeer *pPeer) { pPeer->sstatus = TAOS_SYNC_STATUS_INIT; int ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort) ) - taosTmrReset(syncCheckPeerConnection, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer); + if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); } -void syncRestartConnection(SSyncPeer *pPeer) -{ +void syncRestartConnection(SSyncPeer *pPeer) { if (pPeer->ip == 0) return; syncRestartPeer(pPeer); syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE); } -static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) -{ +static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; sDebug("%s, sync-req is received", pPeer->id); @@ -782,8 +766,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) } } -static void syncNotStarted(void *param, void *tmrId) -{ +static void syncNotStarted(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; @@ -803,14 +786,13 @@ static void syncTryRecoverFromMaster(void *param, void *tmrId) { pthread_mutex_unlock(&(pNode->mutex)); } -static void syncRecoverFromMaster(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncRecoverFromMaster(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; - if ( nodeSStatus != TAOS_SYNC_STATUS_INIT) { + if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus); return; - } + } taosTmrStopA(&pPeer->timer); if (tsSyncNum >= tsMaxSyncNum) { @@ -840,9 +822,8 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) return; } -static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SFwdRsp *pFwdRsp = (SFwdRsp *) cont; SSyncFwds *pSyncFwds = pNode->pSyncFwds; SFwdInfo *pFwdInfo; @@ -862,10 +843,8 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) } } - -static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SWalHead *pHead = (SWalHead *)cont; sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version); @@ -884,9 +863,8 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) return; } -static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id, @@ -909,10 +887,10 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { } // head.len = htonl(head.len); - if (pHead->len <0) { + if (pHead->len < 0) { sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len); return -1; - } + } int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); if (bytes != pHead->len) { @@ -923,9 +901,8 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return 0; } -static int syncProcessPeerMsg(void *param, void *buffer) -{ - SSyncPeer *pPeer = param; +static int syncProcessPeerMsg(void *param, void *buffer) { + SSyncPeer * pPeer = param; SSyncHead head; char *cont = (char *)buffer; @@ -953,8 +930,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) #define statusMsgLen sizeof(SSyncHead)+sizeof(SPeersStatus)+sizeof(SPeerStatus)*TAOS_SYNC_MAX_REPLICA -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) -{ +static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { SSyncNode *pNode = pPeer->pSyncNode; char msg[statusMsgLen] = {0}; @@ -1011,7 +987,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { firstPkt.port = tsSyncPort; firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId - if ( write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { + if (write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { sDebug("%s, connection to peer server is setup", pPeer->id); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; @@ -1024,8 +1000,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { } } -static void syncCheckPeerConnection(void *param, void *tmrId) -{ +static void syncCheckPeerConnection(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; @@ -1037,8 +1012,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId) pthread_mutex_unlock(&(pNode->mutex)); } -static void syncCreateRestoreDataThread(SSyncPeer *pPeer) -{ +static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); pthread_attr_t thattr; @@ -1059,8 +1033,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) } } -static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) -{ +static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { char ipstr[24]; int i; @@ -1137,8 +1110,7 @@ static void syncProcessBrokenLink(void *param) { syncDecNodeRef(pNode); } -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) -{ +static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; uint64_t time = taosGetTimestampMs(); @@ -1160,8 +1132,7 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); } -static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) -{ +static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; int fwds = pSyncFwds->fwds; @@ -1178,8 +1149,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) } } -static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) -{ +static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) { int confirm = 0; if (pFwdInfo->code == 0) pFwdInfo->code = code; @@ -1200,8 +1170,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } } -static void syncMonitorFwdInfos(void *param, void *tmrId) -{ +static void syncMonitorFwdInfos(void *param, void *tmrId) { SSyncNode *pNode = param; SSyncFwds *pSyncFwds = pNode->pSyncFwds; uint64_t time = taosGetTimestampMs(); @@ -1220,6 +1189,3 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); } - - - diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 5c9b734365..2a0bee3726 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -48,8 +48,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind } } -static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) -{ +static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info @@ -130,8 +129,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) return code; } -static int syncRestoreWal(SSyncPeer *pPeer) -{ +static int syncRestoreWal(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; int ret, code = -1; @@ -172,8 +170,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) return offset; } -static int syncProcessBufferedFwd(SSyncPeer *pPeer) -{ +static int syncProcessBufferedFwd(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; int forwards = 0; @@ -201,8 +198,7 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) return pRecv->code; } -int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) -{ +int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { SSyncNode *pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; @@ -222,8 +218,7 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) return pRecv->code; } -static void syncCloseRecvBuffer(SSyncNode *pNode) -{ +static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { taosTFree(pNode->pRecv->buffer); } @@ -231,8 +226,7 @@ static void syncCloseRecvBuffer(SSyncNode *pNode) taosTFree(pNode->pRecv); } -static int syncOpenRecvBuffer(SSyncNode *pNode) -{ +static int syncOpenRecvBuffer(SSyncNode *pNode) { syncCloseRecvBuffer(pNode); SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1); @@ -253,8 +247,7 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) return 0; } -static int syncRestoreDataStepByStep(SSyncPeer *pPeer) -{ +static int syncRestoreDataStepByStep(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; @@ -292,10 +285,9 @@ static int syncRestoreDataStepByStep(SSyncPeer *pPeer) return 0; } -void *syncRestoreData(void *param) -{ - SSyncPeer *pPeer = (SSyncPeer *)param; - SSyncNode *pNode = pPeer->pSyncNode; +void *syncRestoreData(void *param) { + SSyncPeer *pPeer = (SSyncPeer *)param; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); @@ -326,4 +318,3 @@ void *syncRestoreData(void *param) return NULL; } - diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 0cc3e7326a..8aa317b1ac 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -27,11 +27,10 @@ #include "tsync.h" #include "syncInt.h" -static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) -{ +static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { sDebug("%s, start to monitor:%s", pPeer->id, name); - if (pPeer->notifyFd <=0) { + if (pPeer->notifyFd <= 0) { pPeer->watchNum = 0; pPeer->notifyFd = inotify_init1(IN_NONBLOCK); if (pPeer->notifyFd < 0) { @@ -70,9 +69,8 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) return 0; } -static int syncAreFilesModified(SSyncPeer *pPeer) -{ - if (pPeer->notifyFd <=0) return 0; +static int syncAreFilesModified(SSyncPeer *pPeer) { + if (pPeer->notifyFd <= 0) return 0; char buf[2048]; int len = read(pPeer->notifyFd, buf, sizeof(buf)); @@ -96,12 +94,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) } } - return code; + return code; } -static int syncRetrieveFile(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveFile(SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SFileInfo fileInfo; SFileAck fileAck; int code = -1; @@ -128,7 +125,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) // wait for the ack from peer ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret <0) break; + if (ret < 0) break; // set the peer sync version pPeer->sversion = fileInfo.fversion; @@ -148,11 +145,11 @@ static int syncRetrieveFile(SSyncPeer *pPeer) // send the file to peer int sfd = open(name, O_RDONLY); - if ( sfd < 0 ) break; + if (sfd < 0) break; ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); - close(sfd); - if (ret <0) break; + close(sfd); + if (ret < 0) break; sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); fileInfo.index++; @@ -170,8 +167,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) /* if only a partial record is read out, set the IN_MODIFY flag in event, so upper layer will reload the file to get a complete record */ -static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) -{ +static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) { int ret; ret = read(sfd, pHead, sizeof(SWalHead)); @@ -185,7 +181,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) } ret = read(sfd, pHead->cont, pHead->len); - if (ret <0) return -1; + if (ret < 0) return -1; if (ret != pHead->len) { // file is not at end yet, it shall be reloaded @@ -194,10 +190,9 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) } return sizeof(SWalHead) + pHead->len; -} +} -static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) -{ +static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { pPeer->watchNum = 0; taosClose(pPeer->notifyFd); pPeer->notifyFd = inotify_init1(IN_NONBLOCK); @@ -221,18 +216,17 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) return -1; } - return 0; + return 0; } -static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) -{ +static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { char buf[2048]; int len = read(pPeer->notifyFd, buf, sizeof(buf)); - if (len <0 && errno != EAGAIN) { - sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); + if (len < 0 && errno != EAGAIN) { + sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; } - + if (len == 0) return 0; struct inotify_event *event; @@ -248,8 +242,7 @@ static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) return 0; } -static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) -{ +static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { SWalHead *pHead = (SWalHead *) malloc(640000); int code = -1; int32_t bytes = 0; @@ -261,9 +254,12 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion); while (1) { - int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); - if (wsize <0) break; - if (wsize == 0) { code = 0; break; } + int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); + if (wsize < 0) break; + if (wsize == 0) { + code = 0; + break; + } sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version); int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); @@ -286,8 +282,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, return -1; } -static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) -{ +static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { SSyncNode *pNode = pPeer->pSyncNode; int code = -1; char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file @@ -350,12 +345,16 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) } if (code < 0) break; - if (pPeer->sversion >= fversion && fversion > 0) break; + if (pPeer->sversion >= fversion && fversion > 0) break; - index++; wname[0] = 0; + index++; + wname[0] = 0; code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); - if ( code < 0) break; - if ( wname[0] == 0 ) {code = 0; break;} + if (code < 0) break; + if (wname[0] == 0) { + code = 0; + break; + } // current last wal is closed, there is a new one sDebug("%s, last wal is closed, try new one", pPeer->id); @@ -366,9 +365,8 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) return code; } -static int syncRetrieveWal(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveWal(SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; @@ -396,7 +394,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) // send wal file, // inotify is not required, old wal file won't be modified, even remove is ok - if ( stat(fname, &fstat) < 0 ) break; + if (stat(fname, &fstat) < 0) break; size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); @@ -425,9 +423,8 @@ static int syncRetrieveWal(SSyncPeer *pPeer) return code; } -static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; SFirstPkt firstPkt; memset(&firstPkt, 0, sizeof(firstPkt)); @@ -462,9 +459,8 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) return 0; } -void *syncRetrieveData(void *param) -{ - SSyncPeer *pPeer = (SSyncPeer *)param; +void *syncRetrieveData(void *param) { + SSyncPeer * pPeer = (SSyncPeer *)param; SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index b523728bf9..2f064ceb36 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -48,8 +48,7 @@ static void *taosProcessTcpData(void *param); static SThreadObj *taosGetTcpThread(SPoolObj *pPool); static void taosStopPoolThread(SThreadObj* pThread); -void *taosOpenTcpThreadPool(SPoolInfo *pInfo) -{ +void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_t thattr; SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); @@ -89,8 +88,7 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) return pPool; } -void taosCloseTcpThreadPool(void *param) -{ +void taosCloseTcpThreadPool(void *param) { SPoolObj *pPool = (SPoolObj *)param; SThreadObj *pThread; @@ -107,8 +105,7 @@ void taosCloseTcpThreadPool(void *param) uDebug("%p TCP pool is closed", pPool); } -void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) -{ +void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { struct epoll_event event; SPoolObj *pPool = (SPoolObj *)param; @@ -145,9 +142,8 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) return pConn; } -void taosFreeTcpConn(void *param) -{ - SConnObj *pConn = (SConnObj *)param; +void taosFreeTcpConn(void *param) { + SConnObj * pConn = (SConnObj *)param; SThreadObj *pThread = pConn->pThread; uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 625c0d6838..d281642a47 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -156,9 +156,8 @@ static void arbProcessBrokenLink(void *param) { taosTFree(pNode); } -static int arbProcessPeerMsg(void *param, void *buffer) -{ - SNodeConn *pNode = param; +static int arbProcessPeerMsg(void *param, void *buffer) { + SNodeConn * pNode = param; SSyncHead head; int bytes = 0; char *cont = (char *)buffer; @@ -180,7 +179,6 @@ static int arbProcessPeerMsg(void *param, void *buffer) } static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { - struct sigaction act = {{0}}; act.sa_handler = SIG_IGN; sigaction(SIGTERM, &act, NULL); @@ -192,4 +190,3 @@ static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) // inform main thread to exit tsem_post(&tsArbSem); } - diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 6a94c9c6d6..176a451168 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -290,6 +290,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->sync = syncStart(&syncInfo); if (pVnode->sync == NULL) { + vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, + tstrerror(terrno)); vnodeCleanUp(pVnode); return terrno; } From 58408892dabf69f1a1b2291ff2b4a0efc15ac808 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 27 Aug 2020 21:11:57 +0800 Subject: [PATCH 091/111] minor changes --- tests/script/tmp/prepare.sim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index f59eebede0..ab2ad01c32 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -32,10 +32,16 @@ system sh/cfg.sh -n dnode2 -c http -v 1 system sh/cfg.sh -n dnode3 -c http -v 1 system sh/cfg.sh -n dnode4 -c http -v 1 - +# for crash_gen +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/cfg.sh -n dnode1 -c rpcMaxTime -v 101 +system sh/cfg.sh -n dnode1 -c cache -v 2 +system sh/cfg.sh -n dnode1 -c keep -v 36500 system sh/cfg.sh -n dnode1 -c walLevel -v 2 +# for windows + system sh/cfg.sh -n dnode1 -c firstEp -v 152.136.17.116:6030 system sh/cfg.sh -n dnode1 -c secondEp -v 152.136.17.116:6030 system sh/cfg.sh -n dnode1 -c serverPort -v 6030 From 3c45dcdee9481ff813bf6d5beb52606881634fac Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Aug 2020 22:26:37 +0800 Subject: [PATCH 092/111] [td-255] replace wcsncpy with memcpy --- src/common/src/tvariant.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 4036e578b4..069d8eb251 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -100,10 +100,9 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; + pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); - - wcsncpy(pVar->wpz, (wchar_t *)pz, lenInwchar); - pVar->wpz[lenInwchar] = 0; + memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; From 90367ca788e8501340ceb049fe65cdc33e4513f3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 27 Aug 2020 15:21:37 +0000 Subject: [PATCH 093/111] TD-1257 mnode module startup check condition is wrong --- src/mnode/src/mnodeMain.c | 21 +++++++++---- src/mnode/src/mnodeSdb.c | 5 ++++ tests/script/jenkins/basic.txt | 1 + tests/script/jenkins/unique.txt | 40 +++++-------------------- tests/script/unique/mnode/mgmt21.sim | 44 ++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 37 deletions(-) create mode 100644 tests/script/unique/mnode/mgmt21.sim diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 53f3474c3c..84aab70117 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -21,6 +21,7 @@ #include "tgrant.h" #include "ttimer.h" #include "tglobal.h" +#include "mnode.h" #include "dnode.h" #include "mnodeDef.h" #include "mnodeInt.h" @@ -107,13 +108,18 @@ int32_t mnodeStartSystem() { tsMgmtIsRunning = true; mInfo("mnode is initialized successfully"); + + sdbUpdateSync(); + return 0; } int32_t mnodeInitSystem() { mnodeInitTimer(); - if (!mnodeNeedStart()) return 0; - return mnodeStartSystem(); + if (mnodeNeedStart()) { + return mnodeStartSystem(); + } + return 0; } void mnodeCleanupSystem() { @@ -159,14 +165,19 @@ static void mnodeCleanupTimer() { static bool mnodeNeedStart() { struct stat dirstat; - bool fileExist = (stat(tsMnodeDir, &dirstat) == 0); + char mnodeFileName[TSDB_FILENAME_LEN * 2] = {0}; + sprintf(mnodeFileName, "%s/wal0", tsMnodeDir); + + bool fileExist = (stat(mnodeFileName, &dirstat) == 0); bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0); if (asMaster || fileExist) { + mDebug("mnode module start, asMaster:%d fileExist:%d", asMaster, fileExist); return true; + } else { + mDebug("mnode module won't start, asMaster:%d fileExist:%d", asMaster, fileExist); + return false; } - - return false; } bool mnodeIsRunning() { diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 9efb804734..e3ed7daf8c 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -291,6 +291,11 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { } void sdbUpdateSync() { + if (!mnodeIsRunning()) { + mDebug("mnode not start yet, update sync info later"); + return; + } + SSyncCfg syncCfg = {0}; int32_t index = 0; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a48584b0ed..9baa42a8d5 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -294,6 +294,7 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index 06edb8890a..b271f5b726 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -1,22 +1,11 @@ cd ../../../debug; cmake .. cd ../../../debug; make -./test.sh -f unique/account/account_create.sim -./test.sh -f unique/account/account_delete.sim -./test.sh -f unique/account/account_len.sim -./test.sh -f unique/account/authority.sim -./test.sh -f unique/account/basic.sim -./test.sh -f unique/account/paras.sim -./test.sh -f unique/account/pass_alter.sim -./test.sh -f unique/account/pass_len.sim -./test.sh -f unique/account/usage.sim -./test.sh -f unique/account/user_create.sim -./test.sh -f unique/account/user_len.sim - ./test.sh -f unique/big/balance.sim ./test.sh -f unique/big/maxvnodes.sim ./test.sh -f unique/big/tcp.sim +./test.sh -f unique/cluster/alter.sim ./test.sh -f unique/cluster/balance1.sim ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim @@ -61,6 +50,7 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim @@ -77,31 +67,16 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -./test.sh -f general/stream/metrics_1.sim -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -#./test.sh -f general/stream/new_stream.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_1.sim -./test.sh -f general/stream/stream_2.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +# lower the priority while file corruption #./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -126,6 +101,7 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim + ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim ./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim diff --git a/tests/script/unique/mnode/mgmt21.sim b/tests/script/unique/mnode/mgmt21.sim new file mode 100644 index 0000000000..53ad0eebe7 --- /dev/null +++ b/tests/script/unique/mnode/mgmt21.sim @@ -0,0 +1,44 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 + +print ============== step1 +system sh/exec.sh -n dnode2 -s start +sleep 10000 + +system sh/exec.sh -n dnode1 -s start +sql connect + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + return -1 +endi + +print ============== step2 +sql create dnode $hostname2 + +$x = 0 +show2: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + goto show2 +endi +if $data2_2 != slave then + goto show2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file From 5242848a8280883c0f33b8441956733cb470b150 Mon Sep 17 00:00:00 2001 From: zwllxs Date: Fri, 28 Aug 2020 10:06:01 +0800 Subject: [PATCH 094/111] Update administrator-ch.md --- documentation20/webdocs/markdowndocs/administrator-ch.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index c5ecb862e3..64cadf69cd 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -153,10 +153,10 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下： ``` -CREATE USER PASS <‘password’>; +CREATE USER PASS <'password'>; ``` -创建用户，并指定用户名和密码，密码需要用单引号引起来 +创建用户，并指定用户名和密码，密码需要用单引号引起来,单引号为英文半角 ``` DROP USER ; @@ -165,10 +165,10 @@ DROP USER ; 删除用户，限root用户使用 ``` -ALTER USER PASS <‘password’>; +ALTER USER PASS <'password'>; ``` -修改用户密码, 为避免被转换为小写，密码需要用单引号引用 +修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 ``` SHOW USERS; From 9f158b9d482de7a49a2adf4fa8c1ed5686b71cc0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 10:28:12 +0800 Subject: [PATCH 095/111] minor changes --- tests/script/jenkins/basic.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a48584b0ed..c9e081a70b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -147,7 +147,6 @@ cd ../../../debug; make ./test.sh -f general/parser/join.sim ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/bug.sim ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim From ae8fbb6b212d3d760b90ac638eab92a2f6d7456f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 11:57:40 +0800 Subject: [PATCH 096/111] TD-1090 compile error in windows --- src/client/src/tscFunctionImpl.c | 6 +++--- src/client/src/tscLocalMerge.c | 2 +- src/client/src/tscSQLParser.c | 2 +- src/common/src/tname.c | 2 +- src/query/src/qExecutor.c | 18 +++++++++--------- src/query/src/qParserImpl.c | 2 +- src/query/src/qResultbuf.c | 2 +- src/query/src/qTsbuf.c | 8 ++++---- tests/script/wtest.bat | 6 +++--- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index bd4aa70ee4..e74fcba246 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2034,7 +2034,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { tValuePair **tvp = pRes->res; int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = GET_RES_INFO(pCtx)->numOfRes; + int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); switch (type) { case TSDB_DATA_TYPE_INT: { @@ -2408,10 +2408,10 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } GET_TRUE_DATA_TYPE(); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index cabf2a6a11..563c9fa84e 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -966,7 +966,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); } pRes->numOfRowsGroup += pRes->numOfRows; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5600ee3be9..5ce4c7125f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1303,7 +1303,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); // add the timestamp column into the output columns - int32_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 960cc7d725..01945dbb00 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -53,7 +53,7 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const s.type = pVal->nType; if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { - s.bytes = pVal->nLen + VARSTR_HEADER_SIZE; + s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); } else { s.bytes = tDataTypeDesc[pVal->nType].nSize; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 19324bc2cb..4e2e31d269 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2771,14 +2771,14 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); - int32_t numOfRes = pData->num - pGroupResInfo->pos.rowId; + int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId); if (numOfRes > pQuery->rec.capacity - offset) { - numOfCopiedRows = pQuery->rec.capacity - offset; + numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); pGroupResInfo->pos.rowId += numOfCopiedRows; done = true; } else { - numOfCopiedRows = pData->num; + numOfCopiedRows = (int32_t)pData->num; pGroupResInfo->pos.pageId += 1; pGroupResInfo->pos.rowId = 0; @@ -2871,7 +2871,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - pGroupResInfo->numOfDataPages = taosArrayGetSize(pageList); + pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); pGroupResInfo->groupId = tid; pGroupResInfo->pos.pageId = 0; pGroupResInfo->pos.rowId = 0; @@ -3036,7 +3036,7 @@ int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupR char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; - memcpy(output, src, buf->num * bytes); + memcpy(output, src, (size_t)(buf->num * bytes)); } offset += rows; @@ -3211,7 +3211,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity)); } initCtxOutputBuf(pRuntimeEnv); @@ -3926,7 +3926,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } } @@ -4789,7 +4789,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } @@ -6356,7 +6356,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, s); + size_t sz = read(fd, data, (uint32_t)s); if(sz < s) { // todo handle error assert(0); } diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index c09e244643..9629f24cc2 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -182,7 +182,7 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { if (pRight != NULL && pLeft != NULL) { char* endPos = pRight->token.z + pRight->token.n; pExpr->token.z = pLeft->token.z; - pExpr->token.n = endPos - pExpr->token.z; + pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); pExpr->token.type = pLeft->token.type; } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 8d7730a75a..b3e97459d3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -142,7 +142,7 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { } - ret = fwrite(t, size, 1, pResultBuf->file); + ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); if (ret != size) { // todo handle the error case } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 518bb4083b..b264f6cdc9 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -242,13 +242,13 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * both side has the compressed length is used to support load data forwards/backwords. */ int32_t metaLen = 0; - metaLen += fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); - metaLen += fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); } fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 1574b5013e..6cdd63b42d 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,8 +6,8 @@ echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\build\bin" -set "TSIM=%~dp0..\..\debug\build\bin\tsim" +set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" +set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" echo BUILD_DIR: %BUILD_DIR% set "SIM_DIR=%~dp0..\..\sim" @@ -47,7 +47,7 @@ echo qdebugFlag 143 >> %TAOS_CFG% echo udebugFlag 143 >> %TAOS_CFG% set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=localhost" +set "FIRSTEP=192.168.1.182" if "%1" == "-f" set "FILE_NAME=%2" if "%1" == "-h" set "FIRSTEP=%2" if "%3" == "-f" set "FILE_NAME=%4" From 89744f4ba80c6d13433b6657adf2751c7001fef5 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 28 Aug 2020 13:00:53 +0800 Subject: [PATCH 097/111] [release version modify] --- cmake/version.inc | 79 ++++++++---- packaging/release.sh | 261 ++++++++++++++------------------------ src/util/src/version.c.in | 4 +- 3 files changed, 150 insertions(+), 194 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 8c02520142..ad399bfa86 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -1,42 +1,65 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -SET(TD_VER_1 "2") -SET(TD_VER_2 "0") -SET(TD_VER_3 "2") -SET(TD_VER_4 "3") -SET(TD_VER_GIT "d711657139620f6c50f362597020705b8ad26bd2") -SET(TD_VER_GIT_INTERNAL "1d74ae24c541ffbb280e8630883c0236cd45f8c7") +IF (DEFINED VERNUMBER) + SET(TD_VER_NUMBER ${VERNUMBER}) +ELSE () + SET(TD_VER_NUMBER "2.0.2.0") +ENDIF () -SET(TD_VER_VERTYPE "stable") -SET(TD_VER_CPUTYPE "x64") -SET(TD_VER_OSTYPE "Linux") +IF (DEFINED VERCOMPATIBLE) + SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) +ELSE () + SET(TD_VER_COMPATIBLE "2.0.0.0") +ENDIF () -SET(TD_VER_COMPATIBLE "2.0.0.0") -STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +IF (DEFINED GITINFO) + SET(TD_VER_GIT ${GITINFO}) +ELSE () + SET(TD_VER_GIT "community") +ENDIF () -IF (TD_LINUX_64) +IF (DEFINED GITINFOI) + SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSE () + SET(TD_VER_GIT_INTERNAL "internal") +ENDIF () + +IF (DEFINED VERDATE) + SET(TD_VER_DATE ${VERDATE}) +ELSE () + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +ENDIF () + +IF (DEFINED VERTYPE) + SET(TD_VER_VERTYPE ${VERTYPE}) +ELSE () + SET(TD_VER_VERTYPE "stable") +ENDIF () + +IF (DEFINED CPUTYPE) + SET(TD_VER_CPUTYPE ${CPUTYPE}) +ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () -IF (TD_LINUX_32) - SET(TD_VER_CPUTYPE "x86") +IF (DEFINED OSTYPE) + SET(TD_VER_OSTYPE ${OSTYPE}) +ELSE () + SET(TD_VER_OSTYPE "Linux") ENDIF () -IF (TD_ARM_64) - SET(TD_VER_CPUTYPE "aarch64") -ENDIF () +MESSAGE(STATUS "============= compile version parameter information start ============= ") +MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) +MESSAGE(STATUS "internal commit id:" ${TD_VER_GIT_INTERNAL}) +MESSAGE(STATUS "build date:" ${TD_VER_DATE}) +MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) +MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "============= compile version parameter information end ============= ") -IF (TD_ARM_32) - SET(TD_VER_CPUTYPE "aarch32") -ENDIF () - -IF (TD_WINDOWS_64) - SET(TD_VER_CPUTYPE "x64") -ENDIF () - -IF (TD_WINDOWS_32) - SET(TD_VER_CPUTYPE "x86") -ENDIF () +STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/util/src/version.c.in" "${TD_COMMUNITY_DIR}/src/util/src/version.c") diff --git a/packaging/release.sh b/packaging/release.sh index bf355bdcd4..3c0378042d 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -12,6 +12,7 @@ set -e # -l [full | lite] # -s [static | dynamic] # -n [2.0.0.3] +# -m [2.0.0.0] # set parameters by default value verMode=edge # [cluster, edge] @@ -21,8 +22,9 @@ osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ni pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] verNumber="" +verNumberComp="2.0.0.0" -while getopts "hv:V:c:o:l:s:n:" arg +while getopts "hv:V:c:o:l:s:n:m:" arg do case $arg in v) @@ -49,6 +51,10 @@ do #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) ;; + m) + #echo "verNumberComp=$OPTARG" + verNumberComp=$(echo $OPTARG) + ;; o) #echo "osType=$OPTARG" osType=$(echo $OPTARG) @@ -61,6 +67,7 @@ do echo " -l [full | lite] " echo " -s [static | dynamic] " echo " -n [version number] " + echo " -m [compatible version number] " exit 0 ;; ?) #unknow option @@ -70,216 +77,142 @@ do esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber} verNumberComp=${verNumberComp}" curr_dir=$(pwd) if [ "$osType" != "Darwin" ]; then - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/..)" else - script_dir=`dirname $0` - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/.. fi -versioninfo="${top_dir}/src/util/src/version.c" - csudo="" #if command -v sudo > /dev/null; then -# csudo="sudo" +# csudo="sudo" #fi function is_valid_version() { - [ -z $1 ] && return 1 || : + [ -z $1 ] && return 1 || : - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 + rx='^([0-9]+\.){3}(\*|[0-9]+)$' + if [[ $1 =~ $rx ]]; then + return 0 + fi + return 1 } function vercomp () { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i=0; i<${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done + if [[ $1 == $2 ]]; then echo 0 -} + exit 0 + fi + + local IFS=. + local i ver1=($1) ver2=($2) -# 1. Read version information -version=$(cat ${versioninfo} | grep " version" | cut -d '"' -f2) -compatible_version=$(cat ${versioninfo} | grep " compatible_version" | cut -d '"' -f2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done -if [ -z ${verNumber} ]; then - while true; do - read -p "Do you want to release a new version? [y/N]: " is_version_change - - if [[ ( "${is_version_change}" == "y") || ( "${is_version_change}" == "Y") ]]; then - read -p "Please enter the new version: " tversion - while true; do - if (! is_valid_version $tversion) || [ "$(vercomp $tversion $version)" = '2' ]; then - read -p "Please enter a correct version: " tversion - continue - fi - version=${tversion} - break - done - - echo - - read -p "Enter the oldest compatible version: " tversion - while true; do - - if [ -z $tversion ]; then - break - fi - - if (! is_valid_version $tversion) || [ "$(vercomp $version $tversion)" = '2' ]; then - read -p "enter correct compatible version: " tversion - else - compatible_version=$tversion - break - fi - done - - break - elif [[ ( "${is_version_change}" == "n") || ( "${is_version_change}" == "N") ]]; then - echo "Use old version: ${version} compatible version: ${compatible_version}." - break - else - continue + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + echo 1 + exit 0 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + echo 2 + exit 0 fi done -else - echo "old version: $version, new version: $verNumber" - #if ( ! is_valid_version $verNumber ) || [[ "$(vercomp $version $verNumber)" == '2' ]]; then - # echo "please enter correct version" - # exit 0 - #else - version=${verNumber} - #fi -fi + echo 0 +} -echo "=======================new version number: ${version}======================================" +# 1. check version information +if (( ! is_valid_version $verNumber ) || ( ! is_valid_version $verNumberComp ) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then + echo "please enter correct version" + exit 0 +fi + +echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" -# output the version info to the buildinfo file. build_time=$(date +"%F %R") -echo "char version[12] = \"${version}\";" > ${versioninfo} -echo "char compatible_version[12] = \"${compatible_version}\";" >> ${versioninfo} -echo "char gitinfo[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} -if [ "$verMode" != "cluster" ]; then - echo "char gitinfoOfInternal[48] = \"\";" >> ${versioninfo} -else - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - echo "char gitinfoOfInternal[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} - cd ${curr_dir} -fi -echo "char buildinfo[64] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} -echo "" >> ${versioninfo} -tmp_version=$(echo $version | tr -s "." "_") -if [ "$verMode" == "cluster" ]; then - libtaos_info=${tmp_version}_${osType}_${cpuType} -else - libtaos_info=edge_${tmp_version}_${osType}_${cpuType} -fi -if [ "$verType" == "beta" ]; then - libtaos_info=${libtaos_info}_${verType} -fi -echo "void libtaos_${libtaos_info}() {};" >> ${versioninfo} + +# get commint id from git +gitinfo=$(git rev-parse --verify HEAD) +enterprise_dir="${top_dir}/../enterprise" +cd ${enterprise_dir} +gitinfoOfInternal=$(git rev-parse --verify HEAD) +cd ${curr_dir} # 2. cmake executable file compile_dir="${top_dir}/debug" if [ -d ${compile_dir} ]; then - ${csudo} rm -rf ${compile_dir} + ${csudo} rm -rf ${compile_dir} fi if [ "$osType" != "Darwin" ]; then - ${csudo} mkdir -p ${compile_dir} + ${csudo} mkdir -p ${compile_dir} else - mkdir -p ${compile_dir} + mkdir -p ${compile_dir} fi cd ${compile_dir} # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then - if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DOSTYPE=${osType} -DSOMODE=${soMode} - else - cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} - fi + if [ "$verMode" != "cluster" ]; then + cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} + else + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} + fi else - echo "input cpuType=${cpuType} error!!!" - exit 1 + echo "input cpuType=${cpuType} error!!!" + exit 1 fi make cd ${curr_dir} -# 3. judge the operating system type, then Call the corresponding script for packaging -#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -#osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) -#echo "osinfo: ${osinfo}" - +# 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} - - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} + if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then + echo "====do deb package for the ubuntu system====" + output_dir="${top_dir}/debs" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/deb + ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + + echo "====do rpm package for the centos system====" + output_dir="${top_dir}/rpms" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/rpm + ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + fi - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo} ./makepkg.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makearbi.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + echo "====do tar.gz package for all systems====" + cd ${script_dir}/tools + + ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else - cd ${script_dir}/tools - ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} + cd ${script_dir}/tools + ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} fi -# 4. Clean up temporary compile directories -#${csudo} rm -rf ${compile_dir} - diff --git a/src/util/src/version.c.in b/src/util/src/version.c.in index c7aea2afb1..21c78a0eb4 100644 --- a/src/util/src/version.c.in +++ b/src/util/src/version.c.in @@ -1,7 +1,7 @@ -char version[12] = "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}"; +char version[12] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; char buildinfo[64] = "Built at ${TD_VER_DATE}"; -void libtaos_${TD_VER_1}_${TD_VER_2}_${TD_VER_3}_${TD_VER_4}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; +void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; From 28cef33a72147ec61a620ef1febe698ec87170fb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 13:08:34 +0800 Subject: [PATCH 098/111] minor changes --- src/client/src/taos.rc.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/taos.rc.in b/src/client/src/taos.rc.in index 05dbd9bb7d..751be85fd0 100644 --- a/src/client/src/taos.rc.in +++ b/src/client/src/taos.rc.in @@ -1,6 +1,6 @@ 1 VERSIONINFO - FILEVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} - PRODUCTVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} + FILEVERSION ${TD_VER_NUMBER} + PRODUCTVERSION ${TD_VER_NUMBER} FILEFLAGSMASK 0x17L #ifdef _DEBUG FILEFLAGS 0x1L @@ -16,12 +16,12 @@ BEGIN BLOCK "040904b0" BEGIN VALUE "FileDescription", "Native C Driver for TDengine" - VALUE "FileVersion", "${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3}" + VALUE "FileVersion", "${TD_VER_NUMBER}" VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" VALUE "OriginalFilename", "" VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" - VALUE "ProductVersion", "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}" + VALUE "ProductVersion", "${TD_VER_NUMBER}" END END BLOCK "VarFileInfo" From 413a9a657ebc0fac2046da43f6d34020c01efc0d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 28 Aug 2020 13:41:01 +0800 Subject: [PATCH 099/111] add docker repo link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 776d7a154b..8f29adf89b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine) +[![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine)](https://hub.docker.com/repository/docker/tdengine/tdengine) [![TDengine](TDenginelogo.png)](https://www.taosdata.com) From d09567c82b7526e72e3bf14c02ab4866b34c5d3d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 06:42:05 +0000 Subject: [PATCH 100/111] TD-1257 --- src/mnode/src/mnodeMain.c | 2 +- tests/script/unique/mnode/testSuite.sim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 84aab70117..d63a575868 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -166,7 +166,7 @@ static void mnodeCleanupTimer() { static bool mnodeNeedStart() { struct stat dirstat; char mnodeFileName[TSDB_FILENAME_LEN * 2] = {0}; - sprintf(mnodeFileName, "%s/wal0", tsMnodeDir); + sprintf(mnodeFileName, "%s/wal/wal0", tsMnodeDir); bool fileExist = (stat(mnodeFileName, &dirstat) == 0); bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0); diff --git a/tests/script/unique/mnode/testSuite.sim b/tests/script/unique/mnode/testSuite.sim index 33df24b860..b9adbe06a2 100644 --- a/tests/script/unique/mnode/testSuite.sim +++ b/tests/script/unique/mnode/testSuite.sim @@ -1,3 +1,4 @@ +run unique/mnode/mgmt21.sim run unique/mnode/mgmt22.sim run unique/mnode/mgmt23.sim run unique/mnode/mgmt24.sim From 2420c8c5c6f59f75dd54dfba76febe82a601eeb2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 15:48:03 +0800 Subject: [PATCH 101/111] minor changes --- cmake/version.inc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/version.inc b/cmake/version.inc index ad399bfa86..c620d753a6 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -40,7 +40,11 @@ ENDIF () IF (DEFINED CPUTYPE) SET(TD_VER_CPUTYPE ${CPUTYPE}) ELSE () - SET(TD_VER_CPUTYPE "x64") + IF (TD_WINDOWS_32) + SET(TD_VER_CPUTYPE "x86") + ELSE () + SET(TD_VER_CPUTYPE "x64") + ENDIF () ENDIF () IF (DEFINED OSTYPE) From feebfa626ef6d63f144b84926539a30b3c056be9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 28 Aug 2020 08:31:51 +0000 Subject: [PATCH 102/111] TD-1173 Invalid account permission settings --- src/inc/taoserror.h | 1 + tests/script/tmp/prepare.sim | 1 + tests/script/unique/account/usage.sim | 11 ++++++----- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index a789d002bf..951c511022 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -140,6 +140,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, 0, 0x033C, "Cluster no TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "Account already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "Invalid account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, 0, 0x0342, "Invalid account options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, 0, 0x0343, "Account authorization has expired") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, 0, 0x0350, "User already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, 0, 0x0351, "Invalid user") diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index ab2ad01c32..8b8f206233 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -32,6 +32,7 @@ system sh/cfg.sh -n dnode2 -c http -v 1 system sh/cfg.sh -n dnode3 -c http -v 1 system sh/cfg.sh -n dnode4 -c http -v 1 +return # for crash_gen system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/cfg.sh -n dnode1 -c rpcMaxTime -v 101 diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index 3f12c46137..8d611d6fb6 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -112,22 +112,23 @@ print =============== step4 sql insert into d1.t1 values(now + 1s, 1) sql insert into d1.t1 values(now + 2s, 2) -# no write auth -sleep 3000 +sleep 10000 +print no write auth sql_error insert into d1.t1 values(now + 3s, 2) sql_error insert into d1.t1 values(now + 4s, 2) sql alter account root pass "taosdata" tseries 10 storage 36 streams 10 dbs 5 users 5 -sleep 3000 +sleep 10000 +print has write auth sql insert into d1.t1 values(now + 5s, 1) sql insert into d1.t1 values(now + 6s, 2) # no write auth -sleep 3000 +sleep 10000 +print no write auth sql_error insert into d1.t1 values(now + 7s, 2) sql_error insert into d1.t1 values(now + 8s, 2) - print =============== check grant sql_error create database d6 From 544562af01053861cd228628a93cf0be26e2bd14 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 29 Aug 2020 03:43:53 +0000 Subject: [PATCH 103/111] TD-1248 TD-1044 --- src/mnode/src/mnodeShow.c | 1 + src/mnode/src/mnodeTable.c | 7 +++++++ src/mnode/src/mnodeVgroup.c | 13 +++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index e553853658..f66ef6b7a3 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -313,6 +313,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); code = TSDB_CODE_MND_DB_IN_DROPPING; + mnodeDecDbRef(pDb); goto connect_over; } mnodeDecDbRef(pDb); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 794958f7f0..03b1399ea7 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -294,6 +294,7 @@ static int32_t mnodeChildTableActionRestored() { SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); + mnodeDecDbRef(pDb); continue; } mnodeDecDbRef(pDb); @@ -1259,6 +1260,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -1323,6 +1325,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -2495,6 +2498,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -2548,6 +2552,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -2716,6 +2721,7 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -2768,6 +2774,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index ff253c1935..aa6631ff83 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -89,6 +89,7 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) { if (pDb->status != TSDB_DB_STATUS_READY) { mError("vgId:%d, db:%s status:%d, in dropping", pVgroup->vgId, pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -617,6 +618,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -708,6 +710,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -784,7 +787,10 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 1) { taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); pVgroup->numOfTables++; - mnodeIncVgroupRef(pVgroup); + // The create vgroup message may be received later than the create table message + // and the writing order in sdb is therefore uncertain + // which will cause the reference count of the vgroup to be incorrect when restarting + // mnodeIncVgroupRef(pVgroup); } } @@ -792,7 +798,10 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 1) { taosFreeId(pVgroup->idPool, pTable->sid); pVgroup->numOfTables--; - mnodeDecVgroupRef(pVgroup); + // The create vgroup message may be received later than the create table message + // and the writing order in sdb is therefore uncertain + // which will cause the reference count of the vgroup to be incorrect when restarting + // mnodeDecVgroupRef(pVgroup); } } From aa59367c80a483e6830b78fc339737d1c9e39ab2 Mon Sep 17 00:00:00 2001 From: zyyang Date: Sat, 29 Aug 2020 14:32:33 +0800 Subject: [PATCH 104/111] update jdbc test cases --- src/connector/jdbc/src/test/java/TestPreparedStatement.java | 2 +- .../jdbc/src/test/java/TestTSDBDatabaseMetaData.java | 2 +- src/connector/jdbc/src/test/java/TestTSDBSubscribe.java | 5 +---- .../src/test/java/com/taosdata/jdbc/BatchInsertTest.java | 3 +-- .../src/test/java/com/taosdata/jdbc/ConnectionTest.java | 6 ++---- .../test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java | 3 +-- .../jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java | 3 +-- .../test/java/com/taosdata/jdbc/PreparedStatementTest.java | 3 +-- .../jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java | 3 +-- .../jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java | 3 +-- .../jdbc/src/test/java/com/taosdata/jdbc/StableTest.java | 3 +-- .../jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java | 3 +-- .../jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java | 3 +-- .../src/test/java/com/taosdata/jdbc/TSDBDriverTest.java | 2 +- 14 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java index 2e387206a4..1edb957493 100644 --- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java +++ b/src/connector/jdbc/src/test/java/TestPreparedStatement.java @@ -12,7 +12,7 @@ public class TestPreparedStatement { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties); + connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); String rawSql = "select * from test.log0601"; // String[] params = new String[]{"ts", "c1"}; PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); diff --git a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java index f7e0e78441..39a08f0fe9 100644 --- a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java @@ -13,7 +13,7 @@ public class TestTSDBDatabaseMetaData { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties); + connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); dbMetaData = connection.getMetaData(); resSet = dbMetaData.getCatalogs(); while(resSet.next()) { diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java index df730efa69..47acb20064 100644 --- a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java +++ b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java @@ -3,7 +3,6 @@ import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.TSDBResultSet; import com.taosdata.jdbc.TSDBSubscribe; -import java.sql.Connection; import java.sql.DriverManager; import java.util.Properties; @@ -17,12 +16,10 @@ public class TestTSDBSubscribe { properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - String cs = String.format("jdbc:TAOS://%s:0/%s?user=root&password=taosdata", host, database); + String cs = String.format("jdbc:TAOS://%s:0/%s", host, database); return (TSDBConnection)DriverManager.getConnection(cs, properties); } - - public static void main(String[] args) throws Exception { String usage = "java -Djava.ext.dirs=../ TestTSDBSubscribe [-host host] <-db database> <-topic topic> <-sql sql>"; if (args.length < 2) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java index cb78a5ca0e..c49293c96b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java @@ -40,8 +40,7 @@ public class BatchInsertTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java index 3fc8913ca3..a54ece4ead 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java @@ -29,16 +29,14 @@ public class ConnectionTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); assertTrue(null != connection); statement = connection.createStatement(); assertTrue(null != statement); // try reconnect - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); try { statement.execute("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java index 4c4d56020d..284af3dfe7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java @@ -26,8 +26,7 @@ public class DatabaseMetaDataTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata", - properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); String sql = "drop database if exists " + dbName; statement = (TSDBPreparedStatement) connection.prepareStatement(sql); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java index 016eff1a35..dbe16d9fea 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java @@ -28,8 +28,7 @@ public class ImportTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java index d391771269..2cf88f6630 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java @@ -33,8 +33,7 @@ public class PreparedStatementTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata", - properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/",properties); String sql = "drop database if exists " + dbName; statement = (TSDBPreparedStatement) connection.prepareStatement(sql); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java index a0b9c051c6..8067c547df 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java @@ -35,8 +35,7 @@ public class ResultSetTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java index de21cc6195..1844a92b47 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java @@ -28,8 +28,7 @@ public class SelectTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java index c1321d08fc..6e01fb7c34 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java @@ -31,8 +31,7 @@ public class StableTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index 4fe5c39486..db7b8c8cb1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -30,8 +30,7 @@ public class StatementTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 2dc27adab7..07b43d1227 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -32,8 +32,7 @@ public class SubscribeTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 9ea5a431a5..a0981063a5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -12,7 +12,7 @@ public class TSDBDriverTest { @Test public void urlParserTest() throws SQLException { TSDBDriver driver = new TSDBDriver(); - String url = "jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password"; + String url = "jdbc:TSDB://127.0.0.1:0/db"; Properties properties = new Properties(); driver.parseURL(url, properties); From 09e4432dc2a5ac678959dcabf63f550d96ca33d9 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 29 Aug 2020 14:27:05 +0800 Subject: [PATCH 105/111] TD-1278: add test cases --- tests/pytest/insert/nchar.py | 2 ++ tests/pytest/tag_lite/bigint.py | 14 +++++++++ tests/pytest/tag_lite/binary.py | 14 +++++++++ tests/pytest/tag_lite/float.py | 17 +++++++++++ tests/pytest/tag_lite/int.py | 18 ++++++++++++ tests/pytest/tag_lite/nchar.py | 48 +++++++++++++++++++++++++++++++ tests/pytest/tag_lite/smallint.py | 17 +++++++++++ tests/pytest/tag_lite/tinyint.py | 18 ++++++++++++ 8 files changed, 148 insertions(+) create mode 100644 tests/pytest/tag_lite/nchar.py diff --git a/tests/pytest/insert/nchar.py b/tests/pytest/insert/nchar.py index b8e365f143..3319aa3c56 100644 --- a/tests/pytest/insert/nchar.py +++ b/tests/pytest/insert/nchar.py @@ -35,6 +35,8 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(1, 1, '涛思数据') + tdSql.error("insert into tb values (now, 'taosdata001')") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/bigint.py b/tests/pytest/tag_lite/bigint.py index ecc6fad44d..f83961aaad 100644 --- a/tests/pytest/tag_lite/bigint.py +++ b/tests/pytest/tag_lite/bigint.py @@ -575,6 +575,20 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev bigint)") + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 63) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 63) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/binary.py b/tests/pytest/tag_lite/binary.py index a5757dc5cd..4cbae63bec 100644 --- a/tests/pytest/tag_lite/binary.py +++ b/tests/pytest/tag_lite/binary.py @@ -579,6 +579,20 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev binary(5))") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags("dev_001")') + + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev")') + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/float.py b/tests/pytest/tag_lite/float.py index 7b93cb0ac9..9c4d20e82b 100644 --- a/tests/pytest/tag_lite/float.py +++ b/tests/pytest/tag_lite/float.py @@ -575,6 +575,23 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev float)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % -3.4E38 - 1) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38 + 1) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%f)' % -3.4E38) + + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/int.py b/tests/pytest/tag_lite/int.py index 99d4a69624..d5a6917389 100644 --- a/tests/pytest/tag_lite/int.py +++ b/tests/pytest/tag_lite/int.py @@ -574,6 +574,24 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev int)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 31)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 31))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 31) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 31) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/nchar.py b/tests/pytest/tag_lite/nchar.py new file mode 100644 index 0000000000..851cc32b56 --- /dev/null +++ b/tests/pytest/tag_lite/nchar.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(5))") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags("dev_001")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev")') + + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/smallint.py b/tests/pytest/tag_lite/smallint.py index 089af55a34..c060e3f82b 100644 --- a/tests/pytest/tag_lite/smallint.py +++ b/tests/pytest/tag_lite/smallint.py @@ -574,6 +574,23 @@ class TDTestCase: # TSIM: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev smallint)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 15)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 15))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 15) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 15) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) def stop(self): tdSql.close() diff --git a/tests/pytest/tag_lite/tinyint.py b/tests/pytest/tag_lite/tinyint.py index 55e33d013b..089dd46569 100644 --- a/tests/pytest/tag_lite/tinyint.py +++ b/tests/pytest/tag_lite/tinyint.py @@ -575,6 +575,24 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev tinyint)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 7)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 7))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 7) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 7) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 2c6c3775ed214d9171cf4c15c7f634c1ed59700f Mon Sep 17 00:00:00 2001 From: Steven Li Date: Sun, 30 Aug 2020 02:47:50 +0000 Subject: [PATCH 106/111] Fixed a major problem in crash_gen tool, now it properly releases DB connections --- tests/pytest/crash_gen.py | 65 ++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 3ffe77c6a4..7588e03e17 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -766,27 +766,32 @@ class DbConnRest(DbConn): class MyTDSql: - def __init__(self): + def __init__(self, hostAddr, cfgPath): + # Make the DB connection + self._conn = taos.connect(host=hostAddr, config=cfgPath) + self._cursor = self._conn.cursor() + self.queryRows = 0 self.queryCols = 0 self.affectedRows = 0 - def init(self, cursor, log=True): - self.cursor = cursor + # def init(self, cursor, log=True): + # self.cursor = cursor # if (log): # caller = inspect.getframeinfo(inspect.stack()[1][0]) # self.cursor.log(caller.filename + ".sql") def close(self): - self.cursor.close() + self._conn.close() # TODO: very important, cursor close does NOT close DB connection! + self._cursor.close() def query(self, sql): self.sql = sql try: - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() + self._cursor.execute(sql) + self.queryResult = self._cursor.fetchall() self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) + self.queryCols = len(self._cursor.description) except Exception as e: # caller = inspect.getframeinfo(inspect.stack()[1][0]) # args = (caller.filename, caller.lineno, sql, repr(e)) @@ -797,7 +802,7 @@ class MyTDSql: def execute(self, sql): self.sql = sql try: - self.affectedRows = self.cursor.execute(sql) + self.affectedRows = self._cursor.execute(sql) except Exception as e: # caller = inspect.getframeinfo(inspect.stack()[1][0]) # args = (caller.filename, caller.lineno, sql, repr(e)) @@ -810,12 +815,13 @@ class DbConnNative(DbConn): # Class variables _lock = threading.Lock() _connInfoDisplayed = False + totalConnections = 0 # Not private def __init__(self): super().__init__() self._type = self.TYPE_NATIVE self._conn = None - self._cursor = None + # self._cursor = None def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -832,7 +838,8 @@ class DbConnNative(DbConn): buildPath = root[:len(root) - len("/build/bin")] break if buildPath == None: - raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}".format(selfPath, projPath)) + raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}" + .format(selfPath, projPath)) return buildPath @@ -840,25 +847,34 @@ class DbConnNative(DbConn): cfgPath = self.getBuildPath() + "/test/cfg" hostAddr = "127.0.0.1" - with self._lock: # force single threading for opening DB connections - if not self._connInfoDisplayed: - self.__class__._connInfoDisplayed = True # updating CLASS variable - logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath)) - - self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable - self._cursor = self._conn.cursor() + cls = self.__class__ # Get the class, to access class variables + with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!! + if not cls._connInfoDisplayed: + cls._connInfoDisplayed = True # updating CLASS variable + logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath)) + # Make the connection + # self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable + # self._cursor = self._conn.cursor() + # Record the count in the class + self._tdSql = MyTDSql(hostAddr, cfgPath) # making DB connection + cls.totalConnections += 1 - self._cursor.execute('reset query cache') + self._tdSql.execute('reset query cache') # self._cursor.execute('use db') # do this at the beginning of every # Open connection - self._tdSql = MyTDSql() - self._tdSql.init(self._cursor) - + # self._tdSql = MyTDSql() + # self._tdSql.init(self._cursor) + def close(self): if (not self.isOpen): raise RuntimeError("Cannot clean up database until connection is open") self._tdSql.close() + # Decrement the class wide counter + cls = self.__class__ # Get the class, to access class variables + with cls._lock: + cls.totalConnections -= 1 + logger.debug("[DB] Database connection closed") self.isOpen = False @@ -1694,9 +1710,8 @@ class ExecutionStats: logger.info( "| Total Elapsed Time (from wall clock): {:.3f} seconds".format( self._elapsedTime)) - logger.info( - "| Top numbers written: {}".format( - TaskExecutor.getBoundedList())) + logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList())) + logger.info("| Total Number of Active DB Native Connections: {}".format(DbConnNative.totalConnections)) logger.info( "----------------------------------------------------------------------") @@ -2474,7 +2489,7 @@ class ServiceManagerThread: def svcErrorReader(self, err: IO, queue): for line in iter(err.readline, b''): - print("\nTD Svc STDERR: {}".format(line)) + print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line)) class TdeSubProcess: From 2e7c4d536a42736b1fc906d32328d7364aec5f90 Mon Sep 17 00:00:00 2001 From: "Bison \"goldeagle\" Fan" Date: Mon, 31 Aug 2020 09:48:58 +0800 Subject: [PATCH 107/111] improve the readme document with building steps improve building steps with specific linux distributions --- README.md | 77 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8f29adf89b..42c07ae771 100644 --- a/README.md +++ b/README.md @@ -29,24 +29,73 @@ For user manual, system design and architecture, engineering blogs, refer to [TD # Building At the moment, TDengine only supports building and running on Linux systems. You can choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) or from the source code. This quick guide is for installation from the source only. -To build TDengine, use [CMake](https://cmake.org/) 2.8 or higher versions in the project directory. Install CMake for example on Ubuntu: -``` -sudo apt-get install -y cmake build-essential +To build TDengine, use [CMake](https://cmake.org/) 3.5 or higher versions in the project directory. + +## Install tools + +### Ubuntu & Debian: +```bash +sudo apt-get install -y gcc cmake build-essential git ``` To compile and package the JDBC driver source code, you should have a Java jdk-8 or higher and Apache Maven 2.7 or higher installed. -To install openjdk-8 on Ubuntu: -``` -sudo apt-get install openjdk-8-jdk -``` -To install Apache Maven on Ubuntu: -``` -sudo apt-get install maven +To install openjdk-8: +```bash +sudo apt-get install -y openjdk-8-jdk ``` -Build TDengine: - +To install Apache Maven: +```bash +sudo apt-get install -y maven ``` + +### Centos 7: +```bash +sudo yum install -y gcc gcc-c++ make cmake3 epel-release git +sudo yum remove -y cmake +sudo ln -s /usr/bin/cmake3 /usr/bin/cmake +``` + +To install openjdk-8: +```bash +sudo yum install -y java-1.8.0-openjdk +``` + +To install Apache Maven: +```bash +sudo yum install -y maven +``` + +### Centos 8 & Fedora: +```bash +sudo dnf install -y gcc gcc-c++ make cmake epel-release git +``` + +To install openjdk-8: +```bash +sudo dnf install -y java-1.8.0-openjdk +``` + +To install Apache Maven: +```bash +sudo dnf install -y maven +``` + +## Get the source codes + +- github: +```bash +git clone https://github.com/taosdata/TDengine.git +``` + +- gitee: +```bash +git clone https://gitee.com/mirrors/taosdata-TDengine.git +``` + +## Build TDengine + +```bash mkdir debug && cd debug cmake .. && cmake --build . ``` @@ -54,12 +103,12 @@ cmake .. && cmake --build . To compile on an ARM processor (aarch64 or aarch32), please add option CPUTYPE as below: aarch64: -```cmd +```bash cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` aarch32: -```cmd +```bashapt cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` From 04f023ff0a836bfd1f934f56cd91f34c655ac81f Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Mon, 31 Aug 2020 11:48:29 +0800 Subject: [PATCH 108/111] Update faq-ch.md --- .../webdocs/markdowndocs/faq-ch.md | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index b760fe161a..2b51458f27 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -23,14 +23,18 @@ 客户端遇到链接故障，请按照下面的步骤进行检查： -1. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用 -2. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd* -3. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得） -4. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 -5. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的 -6. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 -7. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) -8. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 +1. 检查网络环境 + * 云服务器：检查云服务器的安全组是否打开TCP/UDP 端口6030-6039的访问权限 + * 本地虚拟机：检查网络能否ping通，尽量避免使用`localhost` 作为hostname + * 公司服务器：如果为NAT网络环境，请务必检查服务器能否将消息返回值客户端 +2. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用 +3. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd* +4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得） +5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 +6. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的 +7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 +8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) +9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` 检查服务器侧TCP端口连接是否工作：`nc -l {port}` 检查客户端侧TCP端口链接是否工作：`nc {hostIP} {port}` From 29d2e3eeadd360248e5d5fa300c9651a01c3cc31 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 31 Aug 2020 13:25:12 +0800 Subject: [PATCH 109/111] TD-1247 crash while release hash obj --- src/util/src/hash.c | 20 ++++++-------------- tests/script/tmp/182.sim | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 tests/script/tmp/182.sim diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 7d10545ce7..9634175db7 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -377,6 +377,12 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe } } + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } @@ -390,22 +396,8 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe if (pRes != NULL) { atomic_sub_fetch_64(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pRes); - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - return 0; } else { - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - return -1; } } diff --git a/tests/script/tmp/182.sim b/tests/script/tmp/182.sim new file mode 100644 index 0000000000..27e064dc9b --- /dev/null +++ b/tests/script/tmp/182.sim @@ -0,0 +1,40 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode3 -c walLevel -v 2 +system sh/cfg.sh -n dnode4 -c walLevel -v 2 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 + +system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000 + +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode2 -c http -v 1 +system sh/cfg.sh -n dnode3 -c http -v 1 +system sh/cfg.sh -n dnode4 -c http -v 1 + + +system sh/cfg.sh -n dnode1 -c firstEp -v 127.0.0.1:6030 +system sh/cfg.sh -n dnode1 -c secondEp -v 127.0.0.1:6030 +system sh/cfg.sh -n dnode1 -c serverPort -v 6030 +system sh/cfg.sh -n dnode1 -c fqdn -v 127.0.0.1 +#system sh/exec.sh -n dnode1 -s start From 03f81764761e1628d978a06e045b6ed47dbce5e2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 31 Aug 2020 14:01:21 +0800 Subject: [PATCH 110/111] TD-1173 TD-1204 --- tests/script/unique/account/usage.sim | 19 +++ tests/test/c/CMakeLists.txt | 7 +- tests/test/c/queryPerformance.c | 167 ++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 tests/test/c/queryPerformance.c diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index 8d611d6fb6..7fde365201 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -129,6 +129,25 @@ print no write auth sql_error insert into d1.t1 values(now + 7s, 2) sql_error insert into d1.t1 values(now + 8s, 2) +print =============== step5 +sql alter account root pass "taosdata" tseries 10 storage 3600 streams 10 dbs 5 users 5 state all +sleep 10000 + +sql insert into d1.t1 values(now + 11s, 1) +sql insert into d1.t1 values(now + 12s, 2) + +sql alter account root pass "taosdata" tseries 10 storage 3600 streams 10 dbs 5 users 5 state no +sleep 10000 +print no write auth +sql_error insert into d1.t1 values(now + 13s, 2) +sql_error insert into d1.t1 values(now + 14s, 2) + +sql alter account root pass "taosdata" tseries 10 storage 3600 streams 10 dbs 5 users 5 state all +sleep 10000 +print has write auth +sql insert into d1.t1 values(now + 15s, 1) +sql insert into d1.t1 values(now + 16s, 2) + print =============== check grant sql_error create database d6 diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 9d11895104..536727100d 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -25,6 +25,9 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - add_executable(createNormalTable createNormalTable.c) - target_link_libraries(createNormalTable taos_static tutil common pthread) + #add_executable(createNormalTable createNormalTable.c) + #target_link_libraries(createNormalTable taos_static tutil common pthread) + + #add_executable(queryPerformance queryPerformance.c) + #target_link_libraries(queryPerformance taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/queryPerformance.c b/tests/test/c/queryPerformance.c new file mode 100644 index 0000000000..5e7a4333de --- /dev/null +++ b/tests/test/c/queryPerformance.c @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tulog.h" +#include "ttimer.h" +#include "tutil.h" +#include "tglobal.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +typedef struct { + int64_t startTimeMs; + int64_t endTimeMs; + int threadIndex; + pthread_t thread; +} SInfo; + +void *syncTest(void *param); +void shellParseArgument(int argc, char *argv[]); +void insertData(); + +int64_t numOfThreads = 100; +char sql[10240] = "show dnodes"; +int32_t loopTimes = 1000; + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + taos_init(); + insertData(); +} + +void insertData() { + struct timeval systemTime; + int64_t st, et; + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + pPrint("%" PRId64 " threads are spawned to query", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads); + + // Start threads to write + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].threadIndex = i; + pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + double mseconds = (et - st) / 1000.0; + + int64_t request = loopTimes * numOfThreads; + float avg = mseconds / request;; + float qps = 1000 / avg * numOfThreads; + + pPrint( + "%sall threads:%ld finished, use %.1lf ms, qps:%f, avg:%f %s", + GREEN, numOfThreads, mseconds, qps, avg, NC); + + pPrint("threads exit"); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void *syncTest(void *param) { + TAOS * con; + SInfo * pInfo = (SInfo *)param; + struct timeval systemTime; + + pPrint("thread:%d, start to run", pInfo->threadIndex); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + taosGetFqdnPortFromEp(tsFirst, fqdn, &port); + + con = taos_connect(fqdn, "root", "taosdata", NULL, port); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + for (int i = 0; i < loopTimes; ++i) { + void *tres = taos_query(con, sql); + + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + taos_free_result(tres); + exit(0); + } + + do { + row = taos_fetch_row(tres); + } while( row != NULL); + + taos_free_result(tres); + } + + gettimeofday(&systemTime, NULL); + + return NULL; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the query performance of TDengine\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The sql to be executed, default is %s", sql); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s%d\n", indent, indent, "Loop Times per thread, default is ", loopTimes); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(sql, argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + loopTimes = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else { + } + } + + pPrint("%ssql:%s%s", GREEN, sql, NC); + pPrint("%sloopTImes:%d%s", GREEN, loopTimes, NC); + pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); + pPrint("%sstart to run%s", GREEN, NC); +} From fc936722c2d74133fdf5c84f56cf8727aca3a666 Mon Sep 17 00:00:00 2001 From: "Bison \"goldeagle\" Fan" Date: Mon, 31 Aug 2020 17:07:54 +0800 Subject: [PATCH 111/111] Update README.md remove gitee relative part. --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 42c07ae771..8c26143a12 100644 --- a/README.md +++ b/README.md @@ -86,11 +86,7 @@ sudo dnf install -y maven - github: ```bash git clone https://github.com/taosdata/TDengine.git -``` - -- gitee: -```bash -git clone https://gitee.com/mirrors/taosdata-TDengine.git +cd TDengine ``` ## Build TDengine @@ -108,7 +104,7 @@ cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` aarch32: -```bashapt +```bash cmake .. -DCPUTYPE=aarch32 && cmake --build . ```