xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/TS-2891-M

This commit is contained in:
kailixu 2023-03-17 16:49:06 +08:00
parent b61a5ece5a 9e6cad752c
commit 5f2ced466b
20 changed files with 130 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG ea02029
GIT_TAG d11f210
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
include/libs/qworker/qworker.h
View File

@ -106,6 +106,8 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64
int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
void **pRsp, SArray *explainRes);
int32_t qWorkerDbgEnableDebug(char *option);
#ifdef __cplusplus
}
#endif

4
include/util/tdef.h
View File

@ -316,10 +316,10 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_MAX_MINROWS_FBLOCK 1000000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100
#define TSDB_MIN_MAXROWS_FBLOCK 200
#define TSDB_MAX_MAXROWS_FBLOCK 10000
#define TSDB_MAX_MAXROWS_FBLOCK 10000000
#define TSDB_DEFAULT_MAXROWS_FBLOCK 4096
#define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond

4
source/client/src/clientJniConnector.c
View File

@ -581,8 +581,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
return JNI_RESULT_SET_NULL;
}
void *data;
int32_t numOfRows;
void *data = NULL;
int32_t numOfRows = 0;
int error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
if (numOfRows == 0) {
if (error_code == JNI_SUCCESS) {

3
source/client/src/clientMain.c
View File

@ -611,6 +611,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
}
int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
*numOfRows = 0;
*pData = NULL;
if (res == NULL || TD_RES_TMQ_META(res)) {
return 0;
}

4
source/client/src/clientTmqConnector.c
View File

@ -361,8 +361,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(
TAOS_RES *tres = (TAOS_RES *)res;
void *data;
int32_t numOfRows;
void *data = NULL;
int32_t numOfRows = 0;
int error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
if (numOfRows == 0) {
if (error_code == JNI_SUCCESS) {

82
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -600,6 +600,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
int nCols) {
SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
int32_t code = 0;
bool checkRemainingRow = true;
switch (state->state) {
case SFSLASTNEXTROW_FS:
@ -632,6 +633,8 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
if (code) goto _err;
}
state->pLoadInfo->colIds = aCols;
state->pLoadInfo->numOfCols = nCols;
tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
&(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
&(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
@ -647,22 +650,71 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
goto _next_fileset;
}
state->state = SFSLASTNEXTROW_BLOCKROW;
checkRemainingRow = false;
}
case SFSLASTNEXTROW_BLOCKROW: {
bool hasVal = false;
state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row;
hasVal = tMergeTreeNext(&state->mergeTree);
if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true;
*ppRow = NULL;
return code;
}
bool skipRow = false;
do {
bool hasVal = false;
state->row = tMergeTreeGetRow(&state->mergeTree);
*ppRow = &state->row;
hasVal = tMergeTreeNext(&state->mergeTree);
if (TSDBROW_TS(&state->row) <= state->lastTs) {
*pIgnoreEarlierTs = true;
*ppRow = NULL;
return code;
}
*pIgnoreEarlierTs = false;
if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET;
}
*pIgnoreEarlierTs = false;
if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET;
break;
}
if (checkRemainingRow) {
bool skipBlock = true;
SBlockData *pBlockData = state->row.pBlockData;
for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
SColData *pColData = &pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
}
}
}
/*
for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
SColData *pColData = &pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}
++inputColIndex;
}
}
*/
if (skipBlock) {
skipRow = true;
}
}
} while (skipRow);
return code;
}
@ -908,10 +960,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
int16_t cid = pColData->cid;
if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
if (isLast && pColData->numOfValue != 0) {
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
} else if (pColData->numOfNone != pColData->nVal) {
} else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
}

1
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -118,7 +118,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id
if (suid != 0) {
p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1);
if (p->pSchema == NULL) {
taosMemoryFree(p);
tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
}

4
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -390,8 +390,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) {
taosHashCleanup(pTableMap);
return NULL;
}
pUidList->currentIndex = 0;
for (int32_t j = 0; j < numOfTables; ++j) {
@ -4763,7 +4765,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
pTableBlockInfo->defMinRows = pc->minRows;
pTableBlockInfo->defMaxRows = pc->maxRows;
int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / numOfBucket);
int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBucket);
pTableBlockInfo->numOfFiles += 1;

3
source/libs/catalog/src/ctgDbg.c
View File

@ -21,6 +21,8 @@
extern SCatalogMgmt gCtgMgmt;
SCtgDebug gCTGDebug = {0};
#if 0
void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) {
taosMemoryFree(param);
@ -224,6 +226,7 @@ _return:
CTG_RET(code);
}
#endif
int32_t ctgdEnableDebug(char *option, bool enable) {
if (0 == strcasecmp(option, "lock")) {

3
source/libs/catalog/src/ctgUtil.c
View File

@ -1330,6 +1330,7 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)
static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
#if 0
static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) {
if (NULL == pSrc) {
return TSDB_CODE_SUCCESS;
@ -1421,3 +1422,5 @@ void catalogFreeMetaData(SMetaData* pData) {
taosMemoryFreeClear(pData->pSvrVer);
taosMemoryFree(pData);
}
#endif

1
source/libs/executor/src/executil.c
View File

@ -820,6 +820,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
taosMemoryFree(pResBlock);
return NULL;
}

1
source/libs/executor/src/executorimpl.c
View File

@ -2344,6 +2344,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo);
} else {
terrno = TSDB_CODE_INVALID_PARA;
taosMemoryFree(ops);
return NULL;
}

7
source/libs/executor/src/groupoperator.c
View File

@ -173,9 +173,14 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
size_t numOfGroupCols = taosArrayGetSize(pGroupCols);
for (int32_t i = 0; i < numOfGroupCols; ++i) {
SColumn* pCol = taosArrayGet(pGroupCols, i);
SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId);
// valid range check. todo: return error code.
if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) {
continue;
}
if (pBlock->pBlockAgg != NULL) {
pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched?
}

1
source/libs/executor/src/tsort.c
View File

@ -214,7 +214,6 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
if (pPage == NULL) {
taosArrayDestroy(pPageIdList);
blockDataDestroy(p);
taosArrayDestroy(pPageIdList);
return terrno;
}

2
source/libs/qworker/src/qwDbg.c
View File

@ -271,7 +271,7 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) {
}
}
int32_t qwDbgEnableDebug(char *option) {
int32_t qWorkerDbgEnableDebug(char *option) {
if (0 == strcasecmp(option, "lock")) {
gQWDebug.lockEnable = true;
qError("qw lock debug enabled");

8
tests/script/tsim/db/alter_option.sim
View File

@ -285,12 +285,12 @@ sql_error alter database db keep -1
print ============== modify minrows
sql_error alter database db minrows 8
sql_error alter database db minrows 8000
sql_error alter database db minrows 8001
sql_error alter database db minrows 8000000
sql_error alter database db minrows 8001000
print ============== modify maxrows
sql_error alter database db maxrows 1000
sql_error alter database db maxrows 2000
sql_error alter database db maxrows 10000001
sql_error alter database db maxrows 20000000
sql_error alter database db maxrows 11 # equal minrows
sql_error alter database db maxrows 10 # little than minrows

4
tests/script/tsim/db/create_all_options.sim
View File

@ -298,11 +298,11 @@ sql drop database db
sql_error create database db MAXROWS -1
sql_error create database db MAXROWS 0
sql_error create database db MAXROWS 199
sql_error create database db MAXROWS 10001
sql_error create database db MAXROWS 10000001
sql_error create database db MINROWS -1
sql_error create database db MINROWS 0
sql_error create database db MINROWS 9
sql_error create database db MINROWS 1001
sql_error create database db MINROWS 1000001
sql_error create database db MAXROWS 500 MINROWS 1000
print ====> PRECISION ['ms' | 'us' | 'ns', default: ms]

18
tests/script/tsim/query/explain.sim
View File

@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sql connect
print ======== step1
sql create database db1 vgroups 3;
sql create database db1 vgroups 3 cachesize 10 cachemodel 'both';
sql use db1;
sql select * from information_schema.ins_databases;
sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int);
@ -60,6 +60,15 @@ sql explain verbose true select * from information_schema.ins_stables where db_n
sql explain verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain verbose true insert into t1(ts) select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain verbose true insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s);
sql explain verbose true select distinct tbname, table_name from information_schema.ins_tables;
sql explain verbose true select diff(f1) as f11 from tb1 order by f11;
sql explain verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
sql explain verbose true select count(*) from st1 partition by tbname;
sql explain verbose true select count(*) from information_schema.ins_tables group by stable_name;
sql explain verbose true select last(*) from st1;
sql explain verbose true select last_row(*) from st1;
sql explain verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
sql explain verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
print ======== step4
sql explain analyze select ts from st1 where -2;
@ -96,6 +105,13 @@ sql explain analyze verbose true select * from information_schema.ins_stables wh
sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0;
sql explain analyze verbose true select count(f1) from st1 group by tbname;
sql explain analyze verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
sql explain analyze verbose true select diff(f1) as f11 from tb1 order by f11;
sql explain analyze verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
sql explain analyze verbose true select count(*) from information_schema.ins_tables group by stable_name;
sql explain analyze verbose true select last(*) from st1;
sql explain analyze verbose true select last_row(*) from st1;
sql explain analyze verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
sql explain analyze verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
#not pass case
#sql explain verbose true select count(*),sum(f1) as aa from tb1 where (f1 > 0 or f1 < -1) and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00' order by aa;

10
tests/script/tsim/show/basic.sim
View File

@ -244,5 +244,15 @@ if $rows <= 0 then
return -1
endi
sql show cluster alive;
if $rows <= 0 then
return -1
endi
sql show db.alive;
if $rows <= 0 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT