xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #14323 from taosdata/enh/first_last_select 

enh(query): first/last function support selectivity
This commit is contained in:
Ganlin Zhao 2022-06-29 13:31:40 +08:00 committed by GitHub
parent 5c8d734cb3 e9fba684de
commit 4524b826df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 78 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
source/libs/function/src/builtins.c
View File

@ -1904,7 +1904,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "_first_partial", .name = "_first_partial",
.type = FUNCTION_TYPE_FIRST_PARTIAL, .type = FUNCTION_TYPE_FIRST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastPartial, .translateFunc = translateFirstLastPartial,
.getEnvFunc = getFirstLastFuncEnv, .getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup, .initFunc = functionSetup,
@ -1915,7 +1915,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "_first_merge", .name = "_first_merge",
.type = FUNCTION_TYPE_FIRST_MERGE, .type = FUNCTION_TYPE_FIRST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastMerge, .translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv, .getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup, .initFunc = functionSetup,
@ -1939,7 +1939,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "_last_partial", .name = "_last_partial",
.type = FUNCTION_TYPE_LAST_PARTIAL, .type = FUNCTION_TYPE_LAST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastPartial, .translateFunc = translateFirstLastPartial,
.getEnvFunc = getFirstLastFuncEnv, .getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup, .initFunc = functionSetup,
@ -1950,7 +1950,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "_last_merge", .name = "_last_merge",
.type = FUNCTION_TYPE_LAST_MERGE, .type = FUNCTION_TYPE_LAST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
.translateFunc = translateFirstLastMerge, .translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv, .getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup, .initFunc = functionSetup,

81
source/libs/function/src/builtinsimpl.c
View File

@ -2409,11 +2409,11 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t); } int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos); }
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
pEnv->calcMemSize = sizeof(SFirstLastRes) + pNode->node.resType.bytes + sizeof(int64_t); pEnv->calcMemSize = getFirstLastInfoSize(pNode->node.resType.bytes);
return true; return true;
} }
@ -2491,9 +2491,17 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
} }
memcpy(pInfo->buf, data, bytes); memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts; *(TSKEY*)(pInfo->buf + bytes) = cts;
//handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true; pInfo->hasResult = true;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts); //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
pResInfo->numOfRes = 1; pResInfo->numOfRes = 1;
break; break;
} }
@ -2525,8 +2533,17 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
} }
memcpy(pInfo->buf, data, bytes); memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts; *(TSKEY*)(pInfo->buf + bytes) = cts;
//handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true; pInfo->hasResult = true;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts); //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
pResInfo->numOfRes = 1; pResInfo->numOfRes = 1;
break; break;
} }
@ -2580,8 +2597,17 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
} }
memcpy(pInfo->buf, data, bytes); memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts; *(TSKEY*)(pInfo->buf + bytes) = cts;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts); //handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true; pInfo->hasResult = true;
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
pResInfo->numOfRes = 1; pResInfo->numOfRes = 1;
} }
break; break;
@ -2603,9 +2629,18 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
} }
memcpy(pInfo->buf, data, bytes); memcpy(pInfo->buf, data, bytes);
*(TSKEY*)(pInfo->buf + bytes) = cts; *(TSKEY*)(pInfo->buf + bytes) = cts;
//handle selectivity
if (pCtx->subsidiaries.num > 0) {
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
if (!pInfo->hasResult) {
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
}
}
pInfo->hasResult = true; pInfo->hasResult = true;
pResInfo->numOfRes = 1; pResInfo->numOfRes = 1;
// DO_UPDATE_TAG_COLUMNS(pCtx, ts); //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
} }
break; break;
} }
@ -2615,7 +2650,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) {
SInputColumnInfoData* pColInfo = &pCtx->input;
int32_t start = pColInfo->startRowIndex;
pOutput->bytes = pInput->bytes; pOutput->bytes = pInput->bytes;
TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes); TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes);
TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes);
@ -2632,7 +2670,17 @@ static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput,
} }
*tsOut = *tsIn; *tsOut = *tsIn;
memcpy(pOutput->buf, pInput->buf, pOutput->bytes); memcpy(pOutput->buf, pInput->buf, pOutput->bytes);
//handle selectivity
STuplePos* pTuplePos = (STuplePos*)(pOutput->buf + pOutput->bytes + sizeof(TSKEY));
if (pCtx->subsidiaries.num > 0) {
if (!pOutput->hasResult) {
saveTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos);
} else {
copyTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos);
}
}
pOutput->hasResult = true; pOutput->hasResult = true;
return; return;
} }
@ -2647,7 +2695,7 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer
char* data = colDataGetData(pCol, start); char* data = colDataGetData(pCol, start);
SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data); SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data);
firstLastTransferInfo(pInputInfo, pInfo, isFirstQuery); firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery);
int32_t numOfElems = pInputInfo->hasResult ? 1 : 0; int32_t numOfElems = pInputInfo->hasResult ? 1 : 0;
@ -2669,6 +2717,9 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes); colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes);
//handle selectivity
STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
return pResInfo->numOfRes; return pResInfo->numOfRes;
} }
@ -2687,6 +2738,9 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
colDataAppend(pCol, pBlock->info.rows, res, false); colDataAppend(pCol, pBlock->info.rows, res, false);
//handle selectivity
STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
taosMemoryFree(res); taosMemoryFree(res);
return 1; return 1;
@ -3043,7 +3097,9 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
pItem->uid = uid; pItem->uid = uid;
// save the data of this tuple // save the data of this tuple
saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); if (pCtx->subsidiaries.num > 0) {
saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
}
// allocate the buffer and keep the data of this row into the new allocated buffer // allocate the buffer and keep the data of this row into the new allocated buffer
pEntryInfo->numOfRes++; pEntryInfo->numOfRes++;
@ -3062,7 +3118,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
pItem->uid = uid; pItem->uid = uid;
// save the data of this tuple by over writing the old data // save the data of this tuple by over writing the old data
copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); if (pCtx->subsidiaries.num > 0) {
copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
}
taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type, taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type,
topBotResComparFn, NULL, !isTopQuery); topBotResComparFn, NULL, !isTopQuery);
} }

8
tests/system-test/2-query/json_tag.py
View File

@ -562,7 +562,7 @@ class TDTestCase:
tdSql.checkRows(3) tdSql.checkRows(3)
tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1") tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1")
tdSql.checkRows(3) tdSql.checkRows(3)
#math function #math function
tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;") tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;")
tdSql.checkRows(3) tdSql.checkRows(3)
@ -606,7 +606,7 @@ class TDTestCase:
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;") tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;")
tdSql.checkRows(1) tdSql.checkRows(1)
# function not ready # function not ready
# tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;") # tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;")
# tdSql.checkRows(3) # tdSql.checkRows(3)
@ -616,7 +616,7 @@ class TDTestCase:
# tdSql.checkRows(3) # tdSql.checkRows(3)
# tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;") # tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;")
# tdSql.checkRows(1) # tdSql.checkRows(1)
#str function #str function
tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;") tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;")
tdSql.checkRows(3) tdSql.checkRows(3)
@ -658,7 +658,7 @@ class TDTestCase:
tdSql.checkRows(3) tdSql.checkRows(3)
tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;") tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;")
tdSql.checkRows(1) tdSql.checkRows(1)
# #
# #test TD-12077 # #test TD-12077
tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')")