From 420be42269c52f28f64d58d1fc343fee7bc93012 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Sun, 13 Nov 2022 16:45:53 +0800 Subject: [PATCH 01/16] fix: flush pages to get buffer ready for fetching --- source/libs/tdb/src/db/tdbPCache.c | 13 +- source/libs/tdb/src/db/tdbPager.c | 222 ++++++++++++++++++++++++++--- source/libs/tdb/src/inc/tdbInt.h | 5 +- 3 files changed, 214 insertions(+), 26 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index bdbd6c2f3d..e7254c8bc6 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -169,7 +169,7 @@ int tdbPCacheAlter(SPCache *pCache, int32_t nPage) { SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { SPage *pPage; - i32 nRef; + i32 nRef = 0; tdbPCacheLock(pCache); @@ -178,14 +178,17 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { nRef = tdbRefPage(pPage); } - ASSERT(pPage); - tdbPCacheUnlock(pCache); // printf("thread %" PRId64 " fetch page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id, // TDB_PAGE_PGNO(pPage), pPage, nRef); - tdbDebug("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); + if (pPage) { + tdbDebug("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); + } else { + tdbDebug("pcache/fetch page %p", pPage); + } + return pPage; } @@ -266,7 +269,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) } // 4. Try a create new page - if (!pPage) { + if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { // TODO diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index c3ae1dc739..abbad06515 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -27,6 +27,116 @@ typedef struct { TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); +struct hashset_st { + size_t nbits; + size_t mask; + size_t capacity; + size_t *items; + size_t nitems; + double load_factor; +}; + +static const unsigned int prime = 39; +static const unsigned int prime2 = 5009; + +hashset_t hashset_create(void) { + hashset_t set = tdbOsCalloc(1, sizeof(struct hashset_st)); + if (!set) { + return NULL; + } + + set->nbits = 4; + set->capacity = (size_t)(1 << set->nbits); + set->items = tdbOsCalloc(set->capacity, sizeof(size_t)); + if (!set->items) { + tdbOsFree(set); + return NULL; + } + set->mask = set->capacity - 1; + set->nitems = 0; + + set->load_factor = 0.75; + + return set; +} + +void hashset_destroy(hashset_t set) { + if (set) { + tdbOsFree(set->items); + tdbOsFree(set); + } +} + +int hashset_add_member(hashset_t set, void *item) { + size_t value = (size_t) item; + size_t h; + + if (value == 0) { + return -1; + } + + for (h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { + if (set->items[h] == value) { + return 0; + } + } + + set->items[h] = value; + ++set->nitems; + return 1; +} + +int hashset_add(hashset_t set, void *item) { + int ret = hashset_add_member(set, item); + + size_t old_capacity = set->capacity; + if (set->nitems >= (double)old_capacity * set->load_factor) { + size_t *old_items = set->items; + ++set->nbits; + set->capacity = (size_t)(1 << set->nbits); + set->mask = set->capacity - 1; + + set->items = tdbOsCalloc(set->capacity, sizeof(size_t)); + if (!set->items) { + return -1; + } + + set->nitems = 0; + for (size_t i = 0; i < old_capacity; ++i) { + hashset_add_member(set, (void*)old_items[i]); + } + tdbOsFree(old_items); + } + + return ret; +} + +int hashset_remove(hashset_t set, void *item) { + size_t value = (size_t) item; + + for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { + if (set->items[h] == value) { + set->items[h] = 0; + --set->nitems; + return 1; + } + } + + return 0; +} + +int hashset_contains(hashset_t set, void *item) { + size_t value = (size_t) item; + + for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { + if (set->items[h] == value) { + return 1; + } + } + + return 0; +} + #define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL) static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg, @@ -209,12 +319,16 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary - if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) { + if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } + + if (pPager->jPageSet) { + hashset_add(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + } } return 0; @@ -233,6 +347,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { return -1; } + pPager->jPageSet = hashset_create(); // TODO: write the size of the file pPager->inTran = 1; @@ -275,6 +390,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + if (pPager->jPageSet) { + hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + } tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -304,6 +422,9 @@ int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { return -1; } + if (pPager->jPageSet) { + hashset_destroy(pPager->jPageSet); + } pPager->inTran = 0; return 0; @@ -375,36 +496,61 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { return -1; } - tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); - if (jfd == NULL) { - return -1; - } + tdb_fd_t jfd = pPager->jfd; ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { return -1; } - // 1, read pages from jounal file - // 2, write original pages to buffered ones + u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); + if (pageBuf == NULL) { + return -1; + } - /* TODO: reset the buffered pages instead of releasing them - // loop to reset the dirty pages from file - for (pgIdx = 0, pPage = pPager->pDirty; pPage != NULL && pgIndex < journalSize; pPage = pPage->pDirtyNext, ++pgIdx) { + for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { // read pgno & the page from journal SPgno pgno; int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbOsFree(pageBuf); return -1; } ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { + tdbOsFree(pageBuf); + return -1; + } + + i64 offset = pPager->pageSize * (pgno - 1); + if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { + tdbError("failed to lseek fd due to %s. file:%s, offset:%" PRId64, strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); + return -1; + } + + ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); + if (ret < 0) { + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, + pPager->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } } - */ + + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); + return -1; + } + + tdbOsFree(pageBuf); + // 3, release the dirty pages SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); SRBTreeNode *pNode = NULL; @@ -413,17 +559,55 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + + tRBTreeCreate(&pPager->rbt, pageCmpFn); + + // 4, remove the journal file + tdbOsClose(pPager->jfd); + (void)tdbOsRemove(pPager->jFileName); + hashset_destroy(pPager->jPageSet); + + pPager->inTran = 0; + + return 0; +} + +int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { + SPage *pPage; + int ret; + + // loop to write the dirty pages to file + SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); + SRBTreeNode *pNode = NULL; + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; + ret = tdbPagerWritePageToDB(pPager, pPage); + if (ret < 0) { + tdbError("failed to write page to db since %s", tstrerror(terrno)); + return -1; + } + } + + tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize); + pPager->dbOrigSize = pPager->dbFileSize; + + // release the page + iter = tRBTreeIterCreate(&pPager->rbt, 1); + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; + + pPage->isDirty = 0; + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } tRBTreeCreate(&pPager->rbt, pageCmpFn); - // 4, remove the journal file - tdbOsClose(pPager->jfd); - (void)tdbOsRemove(pPager->jFileName); - pPager->inTran = 0; - return 0; } @@ -453,10 +637,8 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // fetch a page container memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); pgid.pgno = pgno; - pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn); - if (pPage == NULL) { - ASSERT(0); - return -1; + while ((pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn)) == NULL) { + tdbPagerFlushPage(pPager, pTxn); } tdbTrace("tdbttl fetch pager:%p", pPage->pPager); diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index e5ece98b28..731b1927e7 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -384,6 +384,8 @@ struct STDB { #endif }; +typedef struct hashset_st *hashset_t; + struct SPager { char *dbFileName; char *jFileName; @@ -394,7 +396,8 @@ struct SPager { SPCache *pCache; SPgno dbFileSize; SPgno dbOrigSize; - SPage *pDirty; + //SPage *pDirty; + hashset_t jPageSet; SRBTree rbt; u8 inTran; SPager *pNext; // used by TDB From a5b1cfc9a25c08e4ae1b87b480ed6cdc8f7a117a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 14 Nov 2022 11:27:38 +0800 Subject: [PATCH 02/16] fix: extend show create table result width --- include/common/tdatablock.h | 1 + include/libs/function/taosudf.h | 2 +- include/util/types.h | 8 +++- source/client/src/clientImpl.c | 4 ++ source/common/src/tdatablock.c | 38 +++++++++++++++++ source/libs/command/src/command.c | 58 +++++++++++++------------- source/libs/parser/src/parTranslater.c | 4 +- 7 files changed, 82 insertions(+), 33 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 502ba10d33..2dc1ca821f 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -186,6 +186,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); +int32_t colDataLenAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t dataLen); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows); diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 3fe3bb7d3b..d79a569944 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -104,7 +104,7 @@ typedef int32_t (*TUdfDestroyFunc)(); } while (0) #define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1) -typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) diff --git a/include/util/types.h b/include/util/types.h index 8dd0947e9c..21a529559a 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -78,7 +78,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { { (*(double *)(x)) = (*(double *)(y)); } // #endif -typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] @@ -87,6 +87,12 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 #define NCHAR_WIDTH_TO_BYTES(n) ((n)*TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE) +typedef uint32_t extVarDataLenT; +#define EXTVARSTR_HEADER_SIZE (VARSTR_HEADER_SIZE + sizeof(extVarDataLenT)) +#define extVarDataLen(v) ((*(VarDataLenT *)(v)) == -1 ? (*(extVarDataLenT*)(((VarDataLenT *)(v)) + 1)) : (*(VarDataLenT *)(v))) +#define extVarDataVal(v) ((char *)(v) + EXTVARSTR_HEADER_SIZE) +#define setExtVarDataLen(v, l) do { *(VarDataLenT *)(v) = -1; *(extVarDataLenT*)(((VarDataLenT *)(v)) + 1) = (l); } while (0) + typedef int32_t VarDataOffsetT; typedef struct tstr { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3140371c4..3b8270b7e5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1482,6 +1482,10 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { pResultInfo->length[i] = varDataLen(pStart); pResultInfo->row[i] = varDataVal(pStart); + if (-1 == pResultInfo->length[i]) { + pResultInfo->length[i] = extVarDataLen(pStart); + pResultInfo->row[i] = extVarDataVal(pStart); + } } else { pResultInfo->row[i] = NULL; pResultInfo->length[i] = 0; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 536cbed33e..ce4fa1f909 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -62,6 +62,44 @@ int32_t getJsonValueLen(const char* data) { return dataLen; } +int32_t colDataLenAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t dataLen) { + ASSERT(pColumnInfoData != NULL); + + int32_t type = pColumnInfoData->info.type; + if (IS_VAR_DATA_TYPE(type)) { + SVarColAttr* pAttr = &pColumnInfoData->varmeta; + if (pAttr->allocLen < pAttr->length + dataLen) { + uint32_t newSize = pAttr->allocLen; + if (newSize <= 1) { + newSize = 8; + } + + while (newSize < pAttr->length + dataLen) { + newSize = newSize * 1.5; + } + + char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = buf; + pAttr->allocLen = newSize; + } + + uint32_t len = pColumnInfoData->varmeta.length; + pColumnInfoData->varmeta.offset[currentRow] = len; + + memcpy(pColumnInfoData->pData + len, pData, dataLen); + pColumnInfoData->varmeta.length += dataLen; + } else { + memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); + } + + return 0; +} + + int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { ASSERT(pColumnInfoData != NULL); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 1c2d7e1f66..0625c23a7a 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -330,7 +330,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } @@ -345,14 +345,14 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); } } @@ -368,7 +368,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (tTagIsJson(pTag)) { char* pJson = parseTagDatatoJson(pTag); if (pJson) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s", pJson); taosMemoryFree(pJson); } @@ -386,11 +386,11 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; if (i > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ", "); } if (j >= valueNum) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "NULL"); continue; } @@ -404,14 +404,14 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { int32_t tlen = 0; if (IS_VAR_DATA_TYPE(type)) { - dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); + dataConverToStr(buf + EXTVARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); } else { - dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); + dataConverToStr(buf + EXTVARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); } *len += tlen; j++; } else { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "NULL"); } /* @@ -450,37 +450,37 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { if (pCfg->commentLen > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT '%s'", pCfg->pComment); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " COMMENT '%s'", pCfg->pComment); } else if (0 == pCfg->commentLen) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT ''"); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " COMMENT ''"); } if (NULL != pDbCfg->pRetensions && pCfg->watermark1 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " WATERMARK %" PRId64 "a", pCfg->watermark1); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " WATERMARK %" PRId64 "a", pCfg->watermark1); if (pCfg->watermark2 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->watermark2); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->watermark2); } } if (NULL != pDbCfg->pRetensions && pCfg->delay1 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " MAX_DELAY %" PRId64 "a", pCfg->delay1); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " MAX_DELAY %" PRId64 "a", pCfg->delay1); if (pCfg->delay2 > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->delay2); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->delay2); } } int32_t funcNum = taosArrayGetSize(pCfg->pFuncs); if (NULL != pDbCfg->pRetensions && funcNum > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP("); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " ROLLUP("); for (int32_t i = 0; i < funcNum; ++i) { char* pFunc = taosArrayGet(pCfg->pFuncs, i); - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); } - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ")"); } if (pCfg->ttl > 0) { - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); + *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); } } @@ -504,33 +504,33 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p int32_t len = 0; if (TSDB_SUPER_TABLE == pCfg->tableType) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ") TAGS ("); appendTagFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } else if (TSDB_CHILD_TABLE == pCfg->tableType) { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE, "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); appendTagNameFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ") TAGS ("); code = appendTagValues(buf2, &len, pCfg); if (code) { taosMemoryFree(buf2); return code; } - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } else { - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); - len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } - varDataLen(buf2) = len; + setExtVarDataLen(buf2, len); - colDataAppend(pCol2, 0, buf2, false); + colDataLenAppend(pCol2, 0, buf2, len + EXTVARSTR_HEADER_SIZE); taosMemoryFree(buf2); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0e5cb14208..e64a370e15 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6060,11 +6060,11 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** } (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; - (*pSchema)[0].bytes = TSDB_TABLE_NAME_LEN; + (*pSchema)[0].bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN; strcpy((*pSchema)[0].name, "Table"); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; - (*pSchema)[1].bytes = TSDB_MAX_BINARY_LEN; + (*pSchema)[1].bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN; strcpy((*pSchema)[1].name, "Create Table"); return TSDB_CODE_SUCCESS; From 2dd1e05cf0271e6d2147fb1aec8dd6b2bd84eabe Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 14 Nov 2022 11:41:11 +0800 Subject: [PATCH 03/16] fix(timezone): taos_options apply new timezone --- source/util/src/tconfig.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index c1fee37610..5250f5dfcd 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -271,8 +271,14 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType cfgStypeStr(stype), value, terrstr()); return -1; } - pItem->stype = stype; + + // apply new timezone + char szTimezone[TD_TIMEZONE_LEN] = {0}; + int8_t dl; + enum TdTimezone tdOffset = TdZeroZone; + taosSetSystemTimezone(value, szTimezone, &dl, &tdOffset); + return 0; } From 51861bb82ba07047939dff773318914b826448cf Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Nov 2022 14:20:13 +0800 Subject: [PATCH 04/16] refactor(sync): delete some code --- source/libs/sync/src/syncAppendEntriesReply.c | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index ea4dee64f1..2e22ac98a5 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -39,43 +39,6 @@ // /\ UNCHANGED <> // -// only start once -static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, SyncTerm lastApplyTerm, - SyncAppendEntriesReply* pMsg) { - if (beginIndex > endIndex) { - sNError(ths, "snapshot param error, start:%" PRId64 ", end:%" PRId64, beginIndex, endIndex); - return; - } - - // get sender - SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); - ASSERT(pSender != NULL); - - if (snapshotSenderIsStart(pSender)) { - sSError(pSender, "snapshot sender already start"); - return; - } - - SSnapshot snapshot = { - .data = NULL, .lastApplyIndex = endIndex, .lastApplyTerm = lastApplyTerm, .lastConfigIndex = SYNC_INDEX_INVALID}; - void* pReader = NULL; - SSnapshotParam readerParam = {.start = beginIndex, .end = endIndex}; - int32_t code = ths->pFsm->FpSnapshotStartRead(ths->pFsm, &readerParam, &pReader); - ASSERT(code == 0); - -#if 0 - if (pMsg->privateTerm < pSender->privateTerm) { - ASSERT(pReader != NULL); - snapshotSenderStart(pSender, readerParam, snapshot, pReader); - - } else { - if (pReader != NULL) { - ths->pFsm->FpSnapshotStopRead(ths->pFsm, pReader); - } - } -#endif -} - int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; From d2468c5548895680d7e3a53c17fd304eeec64a91 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 14 Nov 2022 14:22:13 +0800 Subject: [PATCH 05/16] fix: fix asan issues --- include/common/ttypes.h | 4 ++-- source/common/src/ttypes.c | 26 +++++++++++++++++-------- source/dnode/vnode/src/vnd/vnodeQuery.c | 6 ++++-- source/libs/scalar/src/filter.c | 13 ++++++++----- source/libs/scheduler/src/schRemote.c | 8 +++++--- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index bfd6a75c3a..761ffd0f1c 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -346,8 +346,8 @@ bool isValidDataType(int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); -void *getDataMin(int32_t type); -void *getDataMax(int32_t type); +void *getDataMin(int32_t type, void* value); +void *getDataMax(int32_t type, void* value); #ifdef __cplusplus } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index a4e7a12ce4..8c5d44b8d5 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,26 +61,36 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { static float floatMin = -FLT_MAX, floatMax = FLT_MAX; static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; -FORCE_INLINE void *getDataMin(int32_t type) { +FORCE_INLINE void *getDataMin(int32_t type, void* value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: - return &floatMin; + *(float *)value = floatMin; + break; case TSDB_DATA_TYPE_DOUBLE: - return &doubleMin; + *(double *)value = doubleMin; + break; default: - return &tDataTypes[type].minValue; + *(int64_t *)value = tDataTypes[type].minValue; + break; } + + return value; } -FORCE_INLINE void *getDataMax(int32_t type) { +FORCE_INLINE void *getDataMax(int32_t type, void* value) { switch (type) { case TSDB_DATA_TYPE_FLOAT: - return &floatMax; + *(float *)value = floatMax; + break; case TSDB_DATA_TYPE_DOUBLE: - return &doubleMax; + *(double *)value = doubleMax; + break; default: - return &tDataTypes[type].maxValue; + *(int64_t *)value = tDataTypes[type].maxValue; + break; } + + return value; } bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_MAX; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index ef0ee6ac0b..15769ef4c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -388,8 +388,10 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { offset += sizeof(p->msgLen); *(int32_t *)((char *)pRsp + offset) = htonl(p->rspCode); offset += sizeof(p->rspCode); - memcpy((char *)pRsp + offset, p->msg, p->msgLen); - offset += p->msgLen; + if (p->msg) { + memcpy((char *)pRsp + offset, p->msg, p->msgLen); + offset += p->msgLen; + } taosMemoryFreeClear(p->msg); } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index df9a818fee..be085e6cbd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -512,15 +512,17 @@ int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) } int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull) { + int64_t tmp = 0; + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type)); + int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type, &tmp)); if (sr == 0) { FILTER_SET_FLAG(ra->sflag, RANGE_FLG_NULL); } } if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - int32_t er = cur->pCompareFunc(&ra->e, getDataMax(cur->type)); + int32_t er = cur->pCompareFunc(&ra->e, getDataMax(cur->type, &tmp)); if (er == 0) { FILTER_SET_FLAG(ra->eflag, RANGE_FLG_NULL); } @@ -696,14 +698,15 @@ int32_t filterAddRangeImpl(void *h, SFilterRange *ra, int32_t optr) { int32_t filterAddRange(void *h, SFilterRange *ra, int32_t optr) { SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - + int64_t tmp = 0; + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type)); + SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type, &tmp)); // FILTER_CLR_FLAG(ra->sflag, RA_NULL); } if (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - SIMPLE_COPY_VALUES(&ra->e, getDataMax(ctx->type)); + SIMPLE_COPY_VALUES(&ra->e, getDataMax(ctx->type, &tmp)); // FILTER_CLR_FLAG(ra->eflag, RA_NULL); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 22fb66d92f..a6a2a6c301 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -286,9 +286,11 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (pJob->execRes.res) { SSubmitRsp *sum = pJob->execRes.res; sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + sum->nBlocks += rsp->nBlocks; + if (rsp->nBlocks > 0 && rsp->pBlocks) { + sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); + memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + } taosMemoryFree(rsp->pBlocks); taosMemoryFree(rsp); } else { From c6efc1ed843363952918377f0eb09c64a0316281 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 14 Nov 2022 14:32:34 +0800 Subject: [PATCH 06/16] fix: fix memory leak during crashgen stream and subscribe --- source/common/src/tmsg.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2eb94773e9..bd8e34a395 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2591,7 +2591,10 @@ int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pR for (int32_t i = 0; i < numOfBatch; ++i) { SUseDbRsp usedbRsp = {0}; - if (tDeserializeSUseDbRspImp(&decoder, &usedbRsp) < 0) return -1; + if (tDeserializeSUseDbRspImp(&decoder, &usedbRsp) < 0) { + tDecoderClear(&decoder); + return -1; + } taosArrayPush(pRsp->pArray, &usedbRsp); } tEndDecode(&decoder); From 7bddbb06f0c7c259dac8bdfabe32c11f70c9155f Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 14 Nov 2022 14:38:51 +0800 Subject: [PATCH 07/16] fix: sanitizer error --- source/client/src/clientMain.c | 1 + source/libs/parser/src/parInsertSql.c | 1 + 2 files changed, 2 insertions(+) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index efa7d095c5..0aa88382fe 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -677,6 +677,7 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) { taosArrayDestroy(pCatalogReq->pIndex); taosArrayDestroy(pCatalogReq->pUser); taosArrayDestroy(pCatalogReq->pTableIndex); + taosArrayDestroy(pCatalogReq->pTableCfg); taosMemoryFree(pCatalogReq); } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 7a38f48cb2..411adc680c 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1364,6 +1364,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, break; } } + taosMemoryFree(pLine); if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { From aa9e935854ed1074939fca7490f62de3077d3553 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 14 Nov 2022 14:46:50 +0800 Subject: [PATCH 08/16] enh(stream): set path max length to 1024 --- source/libs/stream/src/streamState.c | 6 +++--- source/libs/wal/src/walWrite.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 88c39c1157..aefe30116b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -114,12 +114,12 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int return NULL; } - char statePath[300]; + char statePath[1024]; if (!specPath) { sprintf(statePath, "%s/%d", path, pTask->taskId); } else { - memset(statePath, 0, 300); - tstrncpy(statePath, path, 300); + memset(statePath, 0, 1024); + tstrncpy(statePath, path, 1024); } if (tdbOpen(statePath, szPage, pages, &pState->db, 0) < 0) { goto _err; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index b683ba1926..7ced5fae39 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -319,7 +319,7 @@ int32_t walEndSnapshot(SWal *pWal) { SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); if (pInfo) { if (ver >= pInfo->lastVer) { - pInfo--; + pInfo++; } if (POINTER_DISTANCE(pInfo, pWal->fileInfoSet->pData) > 0) { wDebug("vgId:%d, begin remove from %" PRId64, pWal->cfg.vgId, pInfo->firstVer); From 7420b0aaafeb17da16436ca4587df0309f75e504 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Nov 2022 15:24:59 +0800 Subject: [PATCH 09/16] fix(sync): fix memory error --- source/libs/sync/src/syncMain.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7ed90fb140..5ceca36aac 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1832,6 +1832,9 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { SElectTimer* pElectTimer = param; SSyncNode* pNode = pElectTimer->pSyncNode; + if (pNode == NULL) return; + if (pNode->syncEqMsg == NULL) return; + SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_ELECTION, pElectTimer->logicClock, pNode->electTimerMS, pNode); From c0463e46c97548ecb106340fbe22ba4d34a6a6bf Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 14 Nov 2022 15:48:01 +0800 Subject: [PATCH 10/16] fix: truncate show create table result --- include/common/tdatablock.h | 1 - include/libs/function/taosudf.h | 2 +- include/util/types.h | 8 +---- source/client/src/clientImpl.c | 4 --- source/common/src/tdatablock.c | 38 -------------------- source/libs/command/src/command.c | 58 +++++++++++++++---------------- 6 files changed, 31 insertions(+), 80 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 2dc1ca821f..502ba10d33 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -186,7 +186,6 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); -int32_t colDataLenAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t dataLen); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows); diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index d79a569944..5f57e203b9 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -104,7 +104,7 @@ typedef int32_t (*TUdfDestroyFunc)(); } while (0) #define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1) -typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t VarDataLenT; // maxVarDataLen: 65535 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) diff --git a/include/util/types.h b/include/util/types.h index 21a529559a..b49670220b 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -78,7 +78,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { { (*(double *)(x)) = (*(double *)(y)); } // #endif -typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t VarDataLenT; // maxVarDataLen: 65535 #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] @@ -87,12 +87,6 @@ typedef int16_t VarDataLenT; // maxVarDataLen: 32767 #define NCHAR_WIDTH_TO_BYTES(n) ((n)*TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE) -typedef uint32_t extVarDataLenT; -#define EXTVARSTR_HEADER_SIZE (VARSTR_HEADER_SIZE + sizeof(extVarDataLenT)) -#define extVarDataLen(v) ((*(VarDataLenT *)(v)) == -1 ? (*(extVarDataLenT*)(((VarDataLenT *)(v)) + 1)) : (*(VarDataLenT *)(v))) -#define extVarDataVal(v) ((char *)(v) + EXTVARSTR_HEADER_SIZE) -#define setExtVarDataLen(v, l) do { *(VarDataLenT *)(v) = -1; *(extVarDataLenT*)(((VarDataLenT *)(v)) + 1) = (l); } while (0) - typedef int32_t VarDataOffsetT; typedef struct tstr { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3b8270b7e5..c3140371c4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1482,10 +1482,6 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { pResultInfo->length[i] = varDataLen(pStart); pResultInfo->row[i] = varDataVal(pStart); - if (-1 == pResultInfo->length[i]) { - pResultInfo->length[i] = extVarDataLen(pStart); - pResultInfo->row[i] = extVarDataVal(pStart); - } } else { pResultInfo->row[i] = NULL; pResultInfo->length[i] = 0; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index ce4fa1f909..536cbed33e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -62,44 +62,6 @@ int32_t getJsonValueLen(const char* data) { return dataLen; } -int32_t colDataLenAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t dataLen) { - ASSERT(pColumnInfoData != NULL); - - int32_t type = pColumnInfoData->info.type; - if (IS_VAR_DATA_TYPE(type)) { - SVarColAttr* pAttr = &pColumnInfoData->varmeta; - if (pAttr->allocLen < pAttr->length + dataLen) { - uint32_t newSize = pAttr->allocLen; - if (newSize <= 1) { - newSize = 8; - } - - while (newSize < pAttr->length + dataLen) { - newSize = newSize * 1.5; - } - - char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->pData = buf; - pAttr->allocLen = newSize; - } - - uint32_t len = pColumnInfoData->varmeta.length; - pColumnInfoData->varmeta.offset[currentRow] = len; - - memcpy(pColumnInfoData->pData + len, pData, dataLen); - pColumnInfoData->varmeta.length += dataLen; - } else { - memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); - } - - return 0; -} - - int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { ASSERT(pColumnInfoData != NULL); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 0625c23a7a..64fec145ef 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -330,7 +330,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } @@ -345,14 +345,14 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); } } @@ -368,7 +368,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (tTagIsJson(pTag)) { char* pJson = parseTagDatatoJson(pTag); if (pJson) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s", pJson); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); taosMemoryFree(pJson); } @@ -386,11 +386,11 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; if (i > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ", "); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); } if (j >= valueNum) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "NULL"); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); continue; } @@ -404,14 +404,14 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { int32_t tlen = 0; if (IS_VAR_DATA_TYPE(type)) { - dataConverToStr(buf + EXTVARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); + dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); } else { - dataConverToStr(buf + EXTVARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); + dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); } *len += tlen; j++; } else { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "NULL"); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); } /* @@ -450,37 +450,37 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { if (pCfg->commentLen > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " COMMENT '%s'", pCfg->pComment); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT '%s'", pCfg->pComment); } else if (0 == pCfg->commentLen) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " COMMENT ''"); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT ''"); } if (NULL != pDbCfg->pRetensions && pCfg->watermark1 > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " WATERMARK %" PRId64 "a", pCfg->watermark1); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " WATERMARK %" PRId64 "a", pCfg->watermark1); if (pCfg->watermark2 > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->watermark2); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->watermark2); } } if (NULL != pDbCfg->pRetensions && pCfg->delay1 > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " MAX_DELAY %" PRId64 "a", pCfg->delay1); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " MAX_DELAY %" PRId64 "a", pCfg->delay1); if (pCfg->delay2 > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->delay2); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->delay2); } } int32_t funcNum = taosArrayGetSize(pCfg->pFuncs); if (NULL != pDbCfg->pRetensions && funcNum > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " ROLLUP("); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP("); for (int32_t i = 0; i < funcNum; ++i) { char* pFunc = taosArrayGet(pCfg->pFuncs, i); - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); } - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, ")"); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); } if (pCfg->ttl > 0) { - *len += sprintf(buf + EXTVARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); } } @@ -504,33 +504,33 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p int32_t len = 0; if (TSDB_SUPER_TABLE == pCfg->tableType) { - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ") TAGS ("); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); appendTagFields(buf2, &len, pCfg); - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ")"); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } else if (TSDB_CHILD_TABLE == pCfg->tableType) { - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE, "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); appendTagNameFields(buf2, &len, pCfg); - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ") TAGS ("); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); code = appendTagValues(buf2, &len, pCfg); if (code) { taosMemoryFree(buf2); return code; } - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ")"); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } else { - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName); + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); - len += sprintf(buf2 + EXTVARSTR_HEADER_SIZE + len, ")"); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pDbCfg, pCfg); } - setExtVarDataLen(buf2, len); + varDataLen(buf2) = (len > 65535) ? 65535 : len; - colDataLenAppend(pCol2, 0, buf2, len + EXTVARSTR_HEADER_SIZE); + colDataAppend(pCol2, 0, buf2, false); taosMemoryFree(buf2); From 9b58176c5872175082470264a27d9e1b7c2cff35 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 14 Nov 2022 15:56:34 +0800 Subject: [PATCH 11/16] fix(sync): fix AddressSanitizer error: TD-20372 --- source/libs/sync/src/syncMain.c | 5 ++++- source/libs/sync/src/syncRaftLog.c | 13 +++++++++++-- source/libs/sync/src/syncReplication.c | 4 +++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7ed90fb140..8bccf6840a 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -385,7 +385,7 @@ bool syncIsReadyForRead(int64_t rid) { if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { SSyncRaftEntry* pEntry = NULL; int32_t code = pSyncNode->pLogStore->syncLogGetEntry( - pSyncNode->pLogStore, pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore), &pEntry); + pSyncNode->pLogStore, pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore), &pEntry); if (code == 0 && pEntry != NULL) { if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { ready = true; @@ -1832,6 +1832,9 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { SElectTimer* pElectTimer = param; SSyncNode* pNode = pElectTimer->pSyncNode; + if (pNode == NULL) return; + if (pNode->syncEqMsg == NULL) return; + SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_ELECTION, pElectTimer->logicClock, pNode->electTimerMS, pNode); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 7e4b18ab88..bc07781e5c 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -197,7 +197,12 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr syncMeta.isWeek = pEntry->isWeak; syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; + + int64_t tsWriteBegin = taosGetTimestampMs(); index = walAppendLog(pWal, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + int64_t tsWriteEnd = taosGetTimestampMs(); + int64_t tsElapsed = tsWriteEnd - tsWriteBegin; + if (index < 0) { int32_t err = terrno; const char* errStr = tstrerror(err); @@ -210,8 +215,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr } pEntry->index = index; - sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s", pEntry->index, - TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType)); + sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index, + TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType), tsElapsed); return 0; } @@ -236,7 +241,11 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR taosThreadMutexLock(&(pData->mutex)); + int64_t tsBegin = taosGetTimestampMs(); code = walReadVer(pWalHandle, index); + int64_t tsEnd = taosGetTimestampMs(); + int64_t tsElapsed = tsEnd - tsBegin; + // code = walReadVerCached(pWalHandle, index); if (code != 0) { int32_t err = terrno; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 25d6474f67..59afe814eb 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -153,14 +153,16 @@ int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftI // save index, otherwise pMsg will be free by rpc SyncIndex saveLastSendIndex = pState->lastSendIndex; + bool update = false; if (pMsg->dataLen > 0) { saveLastSendIndex = pMsg->prevLogIndex + 1; + update = true; } syncLogSendAppendEntries(pSyncNode, pMsg, ""); syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg); - if (pMsg->dataLen > 0) { + if (update) { pState->lastSendIndex = saveLastSendIndex; pState->lastSendTime = taosGetTimestampMs(); } From c0ad5b838be1bd79016f5f7c279a2cb32acffa79 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Nov 2022 15:23:19 +0800 Subject: [PATCH 12/16] test: add asan cases --- source/dnode/mnode/impl/src/mndQuery.c | 6 ++-- tests/parallel_test/cases.task | 44 +++++++++++++------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 2b0edfebc2..3a7c25f7f9 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -178,8 +178,10 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { offset += sizeof(p->msgLen); *(int32_t *)((char *)pRsp + offset) = htonl(p->rspCode); offset += sizeof(p->rspCode); - memcpy((char *)pRsp + offset, p->msg, p->msgLen); - offset += p->msgLen; + if (p->msg != NULL) { + memcpy((char *)pRsp + offset, p->msg, p->msgLen); + offset += p->msgLen; + } rpcFreeCont(p->msg); } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5baa442231..48215b1d47 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -9,7 +9,7 @@ ,,y,script,./test.sh -f tsim/user/basic.sim ,,y,script,./test.sh -f tsim/user/password.sim ,,y,script,./test.sh -f tsim/user/privilege_db.sim -,,,script,./test.sh -f tsim/user/privilege_sysinfo.sim +,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,,script,./test.sh -f tsim/db/alter_option.sim ,,,script,./test.sh -f tsim/db/alter_replica_13.sim ,,,script,./test.sh -f tsim/db/alter_replica_31.sim @@ -23,16 +23,16 @@ ,,,script,./test.sh -f tsim/db/create_all_options.sim ,,y,script,./test.sh -f tsim/db/delete_reuse1.sim ,,y,script,./test.sh -f tsim/db/delete_reuse2.sim -,,,script,./test.sh -f tsim/db/delete_reusevnode.sim +,,y,script,./test.sh -f tsim/db/delete_reusevnode.sim ,,y,script,./test.sh -f tsim/db/delete_reusevnode2.sim -,,,script,./test.sh -f tsim/db/delete_writing1.sim -,,,script,./test.sh -f tsim/db/delete_writing2.sim +,,y,script,./test.sh -f tsim/db/delete_writing1.sim +,,y,script,./test.sh -f tsim/db/delete_writing2.sim ,,y,script,./test.sh -f tsim/db/error1.sim ,,y,script,./test.sh -f tsim/db/keep.sim ,,y,script,./test.sh -f tsim/db/len.sim ,,y,script,./test.sh -f tsim/db/repeat.sim ,,y,script,./test.sh -f tsim/db/show_create_db.sim -,,,script,./test.sh -f tsim/db/show_create_table.sim +,,y,script,./test.sh -f tsim/db/show_create_table.sim ,,y,script,./test.sh -f tsim/db/tables.sim ,,y,script,./test.sh -f tsim/db/taosdlog.sim ,,,script,./test.sh -f tsim/dnode/balance_replica1.sim @@ -80,8 +80,8 @@ ,,y,script,./test.sh -f tsim/insert/query_multi_file.sim ,,y,script,./test.sh -f tsim/insert/tcp.sim ,,y,script,./test.sh -f tsim/insert/update0.sim -,,,script,./test.sh -f tsim/insert/update1_sort_merge.sim -,,,script,./test.sh -f tsim/parser/alter__for_community_version.sim +,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim ,,y,script,./test.sh -f tsim/parser/alter_column.sim ,,y,script,./test.sh -f tsim/parser/alter_stable.sim ,,y,script,./test.sh -f tsim/parser/alter.sim @@ -92,21 +92,21 @@ ,,y,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim ,,,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim ,,y,script,./test.sh -f tsim/parser/columnValue_bigint.sim -,,,script,./test.sh -f tsim/parser/columnValue_bool.sim +,,y,script,./test.sh -f tsim/parser/columnValue_bool.sim ,,y,script,./test.sh -f tsim/parser/columnValue_double.sim ,,y,script,./test.sh -f tsim/parser/columnValue_float.sim ,,y,script,./test.sh -f tsim/parser/columnValue_int.sim ,,y,script,./test.sh -f tsim/parser/columnValue_smallint.sim ,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim ,,,script,./test.sh -f tsim/parser/columnValue_unsign.sim -,,,script,./test.sh -f tsim/parser/commit.sim -,,,script,./test.sh -f tsim/parser/condition.sim +,,y,script,./test.sh -f tsim/parser/commit.sim +,,y,script,./test.sh -f tsim/parser/condition.sim ,,y,script,./test.sh -f tsim/parser/constCol.sim -,,,script,./test.sh -f tsim/parser/create_db.sim -,,,script,./test.sh -f tsim/parser/create_mt.sim +,,y,script,./test.sh -f tsim/parser/create_db.sim +,,y,script,./test.sh -f tsim/parser/create_mt.sim ,,y,script,./test.sh -f tsim/parser/create_tb_with_tag_name.sim ,,y,script,./test.sh -f tsim/parser/create_tb.sim -,,,script,./test.sh -f tsim/parser/dbtbnameValidate.sim +,,y,script,./test.sh -f tsim/parser/dbtbnameValidate.sim ,,y,script,./test.sh -f tsim/parser/distinct.sim ,,y,script,./test.sh -f tsim/parser/fill_us.sim ,,,script,./test.sh -f tsim/parser/fill.sim @@ -120,9 +120,9 @@ ,,y,script,./test.sh -f tsim/parser/import_commit1.sim ,,y,script,./test.sh -f tsim/parser/import_commit2.sim ,,y,script,./test.sh -f tsim/parser/import_commit3.sim -,,,script,./test.sh -f tsim/parser/import_file.sim +,,y,script,./test.sh -f tsim/parser/import_file.sim ,,y,script,./test.sh -f tsim/parser/import.sim -,,,script,./test.sh -f tsim/parser/insert_multiTbl.sim +,,y,script,./test.sh -f tsim/parser/insert_multiTbl.sim ,,y,script,./test.sh -f tsim/parser/insert_tb.sim ,,,script,./test.sh -f tsim/parser/join_manyblocks.sim ,,y,script,./test.sh -f tsim/parser/join_multitables.sim @@ -131,12 +131,12 @@ ,,y,script,./test.sh -f tsim/parser/last_cache.sim ,,y,script,./test.sh -f tsim/parser/last_groupby.sim ,,y,script,./test.sh -f tsim/parser/lastrow.sim -,,,script,./test.sh -f tsim/parser/lastrow2.sim +,,y,script,./test.sh -f tsim/parser/lastrow2.sim ,,y,script,./test.sh -f tsim/parser/like.sim ,,,script,./test.sh -f tsim/parser/limit.sim ,,,script,./test.sh -f tsim/parser/limit1.sim ,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim -,,,script,./test.sh -f tsim/parser/nchar.sim +,,y,script,./test.sh -f tsim/parser/nchar.sim ,,,script,./test.sh -f tsim/parser/nestquery.sim ,,,script,./test.sh -f tsim/parser/null_char.sim ,,y,script,./test.sh -f tsim/parser/precision_ns.sim @@ -146,7 +146,7 @@ ,,y,script,./test.sh -f tsim/parser/select_distinct_tag.sim ,,y,script,./test.sh -f tsim/parser/select_from_cache_disk.sim ,,y,script,./test.sh -f tsim/parser/select_with_tags.sim -,,,script,./test.sh -f tsim/parser/selectResNum.sim +,,y,script,./test.sh -f tsim/parser/selectResNum.sim ,,,script,./test.sh -f tsim/parser/set_tag_vals.sim ,,y,script,./test.sh -f tsim/parser/single_row_in_tb.sim ,,,script,./test.sh -f tsim/parser/sliding.sim @@ -271,7 +271,7 @@ ,,y,script,./test.sh -f tsim/stable/tag_filter.sim ,,y,script,./test.sh -f tsim/stable/tag_modify.sim ,,y,script,./test.sh -f tsim/stable/tag_rename.sim -,,,script,./test.sh -f tsim/stable/values.sim +,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim ,,,script,./test.sh -f tsim/sma/drop_sma.sim @@ -325,7 +325,7 @@ ,,y,script,./test.sh -f tsim/compute/bottom.sim ,,y,script,./test.sh -f tsim/compute/count.sim ,,y,script,./test.sh -f tsim/compute/diff.sim -,,,script,./test.sh -f tsim/compute/diff2.sim +,,y,script,./test.sh -f tsim/compute/diff2.sim ,,y,script,./test.sh -f tsim/compute/first.sim ,,y,script,./test.sh -f tsim/compute/interval.sim ,,y,script,./test.sh -f tsim/compute/last_row.sim @@ -358,7 +358,7 @@ ,,y,script,./test.sh -f tsim/vector/metrics_query.sim ,,y,script,./test.sh -f tsim/vector/metrics_tag.sim ,,y,script,./test.sh -f tsim/vector/metrics_time.sim -,,,script,./test.sh -f tsim/vector/multi.sim +,,y,script,./test.sh -f tsim/vector/multi.sim ,,y,script,./test.sh -f tsim/vector/single.sim ,,y,script,./test.sh -f tsim/vector/table_field.sim ,,y,script,./test.sh -f tsim/vector/table_mix.sim @@ -380,7 +380,7 @@ ,,y,script,./test.sh -f tsim/tag/column.sim ,,y,script,./test.sh -f tsim/tag/commit.sim ,,y,script,./test.sh -f tsim/tag/create.sim -,,,script,./test.sh -f tsim/tag/delete.sim +,,y,script,./test.sh -f tsim/tag/delete.sim ,,y,script,./test.sh -f tsim/tag/double.sim ,,y,script,./test.sh -f tsim/tag/filter.sim ,,y,script,./test.sh -f tsim/tag/float.sim From 1d795f5c62e4102c45b6263a234b74c5a08f7283 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 14 Nov 2022 16:27:02 +0800 Subject: [PATCH 13/16] fix: null pointer when merge col data of varchar --- source/common/src/tdatablock.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e8d5989e4d..f7b1196248 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -277,7 +277,9 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->varmeta.allocLen = len + oldLen; } - memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); + if (pColumnInfoData->pData && pSource->pData) { // TD-20382 + memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); + } pColumnInfoData->varmeta.length = len + oldLen; } else { if (finalNumOfRows > (*capacity)) { From 349189a696e0b31fad95eba5840d9a4fdf120139 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 14 Nov 2022 16:39:25 +0800 Subject: [PATCH 14/16] fix: ingore tree.c error found by asan --- tests/parallel_test/cases.task | 2 +- tests/script/sh/checkAsan.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5baa442231..31b21e3465 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -392,7 +392,7 @@ ,,y,script,./test.sh -f tsim/tag/tinyint.sim ,,y,script,./test.sh -f tsim/tag/drop_tag.sim ,,y,script,./test.sh -f tsim/tag/tbNameIn.sim -,,,script,./test.sh -f tmp/monitor.sim +,,y,script,./test.sh -f tmp/monitor.sim #system test diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 4d1b0a3d6b..184dc9a88f 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -20,7 +20,7 @@ LOG_DIR=$TAOS_DIR/sim/tsim/asan error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | wc -l` +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" From 901c9b8f77cfa2c64074edd7795cf3c8f9311ddf Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 14 Nov 2022 16:53:01 +0800 Subject: [PATCH 15/16] fix: some problems of parser --- include/libs/nodes/querynodes.h | 5 ++--- source/libs/function/src/builtins.c | 14 +++++++------- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parTranslater.c | 18 ++++++++++-------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index a1dad1806d..d0971b013f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -74,9 +74,8 @@ typedef struct SColumnNode { char tableName[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; - // SNode* pProjectRef; - int16_t dataBlockId; - int16_t slotId; + int16_t dataBlockId; + int16_t slotId; } SColumnNode; typedef struct SColumnRefNode { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index d3f03e8e9c..8249a8f6b1 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -475,7 +475,7 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1506,7 +1506,7 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1519,7 +1519,7 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); uint8_t dbPrec = pFunc->node.resType.precision; - //if (1 != numOfParams && 3 != numOfParams && 4 != numOfParams) { + // if (1 != numOfParams && 3 != numOfParams && 4 != numOfParams) { if (1 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1835,7 +1835,7 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1894,7 +1894,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2474,7 +2474,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "first", .type = FUNCTION_TYPE_FIRST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateFirstLast, .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2512,7 +2512,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last", .type = FUNCTION_TYPE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateFirstLast, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 39d17153d0..cc1bae6a3c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -622,7 +622,7 @@ void nodesDestroyNode(SNode* pNode) { } switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: // pProjectRef is weak reference, no need to release + case QUERY_NODE_COLUMN: destroyExprNode((SExprNode*)pNode); break; case QUERY_NODE_VALUE: { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0e5cb14208..260626f9c0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -744,7 +744,8 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { return (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId); } else if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { SFunctionNode* pFunc = (SFunctionNode*)pExpr; - if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType) { + if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_FIRST == pFunc->funcType || + FUNCTION_TYPE_LAST == pFunc->funcType) { return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType) { return true; @@ -787,7 +788,6 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColumnNode** pColRef) { SColumnNode* pCol = *pColRef; - // pCol->pProjectRef = (SNode*)pExpr; if (NULL == pExpr->pAssociation) { pExpr->pAssociation = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); } @@ -2932,8 +2932,8 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* return TSDB_CODE_SUCCESS; } - if (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER) || - TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER)) { + if (!pCxt->createStream && (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER) || + TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); } @@ -5268,9 +5268,7 @@ static int32_t checkTopicQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { } static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SCMCreateTopicReq* pReq) { - SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); - tNameGetFullDbName(&name, pReq->name); + snprintf(pReq->name, sizeof(pReq->name), "%d.%s", pCxt->pParseCxt->acctId, pStmt->topicName); pReq->igExists = pStmt->ignoreExists; pReq->withMeta = pStmt->withMeta; @@ -5280,7 +5278,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS } int32_t code = TSDB_CODE_SUCCESS; - + SName name; if ('\0' != pStmt->subSTbName[0]) { pReq->subType = TOPIC_SUB_TYPE__TABLE; toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); @@ -5548,6 +5546,10 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { crossTableWithUdaf(pSelect)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } + if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "SUBTABLE expression must be of VARCHAR type"); + } return TSDB_CODE_SUCCESS; } From fae4fe5a755ca90685052357970e724b8a8fa907 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 14 Nov 2022 17:13:44 +0800 Subject: [PATCH 16/16] fix(timezone): call right apply function --- source/util/src/tconfig.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 5250f5dfcd..76a312cd91 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -274,10 +274,7 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType pItem->stype = stype; // apply new timezone - char szTimezone[TD_TIMEZONE_LEN] = {0}; - int8_t dl; - enum TdTimezone tdOffset = TdZeroZone; - taosSetSystemTimezone(value, szTimezone, &dl, &tdOffset); + osSetTimezone(value); return 0; }