xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: review comments

This commit is contained in:
slzhou 2024-03-12 16:53:56 +08:00
parent 3db2c1edf5
commit 5c691685ba
6 changed files with 58 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/en/12-taos-sql/06-select.md
View File

@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
hint:
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT | PARTITION_FIRST | SMALLDATA_TS_SORT
select_list:
select_expr [, select_expr] ...
@ -94,6 +94,7 @@ The list of currently supported Hints is as follows:
| SORT_FOR_GROUP| None | Use sort for partition, conflict with PARTITION_FIRST | With normal column in partition by list |
| PARTITION_FIRST| None | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list |
| PARA_TABLES_SORT| None | When sorting the supertable rows by timestamp, No temporary disk space is used. When there are numerous tables, each with long rows, the corresponding algorithm associated with this prompt may consume a substantial amount of memory, potentially leading to an Out Of Memory (OOM) situation. | Sorting the supertable rows by timestamp |
| SMALLDATA_TS_SORT| None | When sorting the supertable rows by timestamp, if the length of query columns >= 256, and there are relatively few rows, this hint can improve performance. | Sorting the supertable rows by timestamp |
For example:
@ -102,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0
SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts;
```
## Lists

5
docs/zh/12-taos-sql/06-select.md
View File

@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
hint:
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARTITION_FIRST | PARA_TABLES_SORT | SMALLDATA_TS_SORT
select_list:
select_expr [, select_expr] ...
@ -94,6 +94,8 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适
| SORT_FOR_GROUP| 无 | 采用sort方式进行分组, 与PARTITION_FIRST冲突 | partition by 列表有普通列时 |
| PARTITION_FIRST| 无 | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时 |
| PARA_TABLES_SORT| 无 | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存。当子表数量多, 行长比较大时候, 会使用大量内存, 可能发生OOM | 超级表的数据按时间戳排序时 |
| SMALLDATA_TS_SORT| 无 | 超级表的数据按时间戳排序时, 查询列长度大于等于256, 但是行数不多, 使用这个提示, 可以提高性能 | 超级表的数据按时间戳排序时 |
举例：
```sql
@ -101,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0
SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts;
```
## 列表

7
include/os/osFile.h
View File

@ -119,6 +119,13 @@ int32_t taosSetFileHandlesLimit();
int32_t taosLinkFile(char *src, char *dst);
FILE* taosOpenCFile(const char* filename, const char* mode);
int taosSeekCFile(FILE* file, int64_t offset, int whence);
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream );
size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream);
int taosCloseCFile(FILE *);
int taosSetAutoDelFile(char* path);
bool lastErrorIsFileNotExist();
#ifdef __cplusplus

5
source/libs/executor/src/scanoperator.c
View File

@ -4011,17 +4011,14 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
if (pInfo->bSortRowId && numOfTable != 1) {
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
if (pInfo->bSortRowId && numOfTable != 1) {
int32_t memSize = 512 * 1024 * 1024;
code = tsortSetSortByRowId(pInfo->pSortHandle, memSize);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} else {
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
}
tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit);
tsortSetMergeLimitReachedFp(pInfo->pSortHandle, tableMergeScanDoSkipTable, pInfo);

49
source/libs/executor/src/tsort.c
View File

@ -13,8 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "query.h"
#include "tcommon.h"
@ -34,17 +32,6 @@ struct STupleHandle {
int32_t rowIndex;
};
typedef struct SSortMemPageEntry {
int32_t pageId;
bool active;
void* data;
struct SSortMemPageEntry* next;
struct SSortMemPageEntry* prev;
} SSortMemPageEntry;
typedef struct SSortMemFileRegion {
int64_t fileOffset;
int32_t regionSize;
@ -68,7 +55,6 @@ typedef struct SSortMemFile {
int32_t blockSize;
FILE* pTdFile;
// TdFilePtr pTdFile;
char memFilePath[PATH_MAX];
} SSortMemFile;
@ -240,21 +226,6 @@ void destroyTuple(void* t) {
}
}
int tsortSeekFile(FILE* file, int64_t offset, int whence) {
#ifdef WINDOWS
return _fseeki64(file, offset, whence);
#else
return fseeko(file, offset, whence);
#endif
}
int tsortSetAutoDelFile(char* path) {
#ifdef WINDOWS
return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
#else
return unlink(path);
#endif
}
/**
*
@ -1038,9 +1009,9 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
if (pRegion->buf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize);
int ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
int ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
if (ret != 1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
@ -1058,9 +1029,9 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
}
int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset, szThisBlock);
tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
int ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
int ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
if (ret != 1) {
taosMemoryFreeClear(*ppRow);
terrno = TAOS_SYSTEM_ERROR(errno);
@ -1085,13 +1056,13 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
}
if (code == TSDB_CODE_SUCCESS) {
taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+");
if (pMemFile->pTdFile == NULL) {
code = terrno = TAOS_SYSTEM_ERROR(errno);
}
}
if (code == TSDB_CODE_SUCCESS) {
tsortSetAutoDelFile(pMemFile->memFilePath);
taosSetAutoDelFile(pMemFile->memFilePath);
pMemFile->currRegionId = -1;
pMemFile->currRegionOffset = -1;
@ -1119,7 +1090,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
if (pMemFile->aFileRegions) taosMemoryFreeClear(pMemFile->aFileRegions);
if (pMemFile->writeBuf) taosMemoryFreeClear(pMemFile->writeBuf);
if (pMemFile->pTdFile) {
fclose(pMemFile->pTdFile);
taosCloseCFile(pMemFile->pTdFile);
pMemFile->pTdFile = NULL;
}
taosMemoryFreeClear(pMemFile);
@ -1142,7 +1113,7 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
taosMemoryFree(pMemFile->writeBuf);
pMemFile->writeBuf = NULL;
fclose(pMemFile->pTdFile);
taosCloseCFile(pMemFile->pTdFile);
pMemFile->pTdFile = NULL;
taosRemoveFile(pMemFile->memFilePath);
taosMemoryFree(pMemFile);
@ -1272,8 +1243,8 @@ static void initRowIdSort(SSortHandle* pHandle) {
int32_t rowSize = blockDataGetRowSize(pHandle->pDataBlock);
size_t nCols = taosArrayGetSize(pHandle->pDataBlock->pDataBlock);
pHandle->pageSize = getProperSortPageSize(rowSize, nCols);
pHandle->numOfPages = 2048;
pHandle->pageSize = 256 * 1024; // 256k
pHandle->numOfPages = 256;
SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
SBlockOrderInfo bi = {0};

32
source/os/src/osFile.c
View File

@ -1404,3 +1404,35 @@ int32_t taosLinkFile(char *src, char *dst) {
#endif
return 0;
}
FILE* taosOpenCFile(const char* filename, const char* mode) {
return fopen(filename, mode);
}
int taosSeekCFile(FILE* file, int64_t offset, int whence) {
#ifdef WINDOWS
return _fseeki64(file, offset, whence);
#else
return fseeko(file, offset, whence);
#endif
}
size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) {
return fread(buffer, size, count, stream);
}
size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) {
return fwrite(ptr, size, nitems, stream);
}
int taosCloseCFile(FILE *f) {
return fclose(f);
}
int taosSetAutoDelFile(char* path) {
#ifdef WINDOWS
return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
#else
return unlink(path);
#endif
}