xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[td-225] fix bugs for leastsquares query.

This commit is contained in:
hjLiao 2020-05-13 11:52:33 +08:00
parent 11c0f8ba32
commit 5f0a05163d
9 changed files with 50 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/client/inc/tscUtil.h
View File

@ -183,7 +183,7 @@ void tscSqlExprInfoDestroy(SArray* pExprInfo);
SColumn* tscColumnClone(const SColumn* src); SColumn* tscColumnClone(const SColumn* src);
SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex);
void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex); SArray* tscColumnListClone(const SArray* src, int16_t tableIndex);
void tscColumnListDestroy(SArray* pColList); void tscColumnListDestroy(SArray* pColList);
SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);

6
src/client/src/tscFunctionImpl.c
View File

@ -2904,7 +2904,11 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
param[1][2] /= param[1][1]; param[1][2] /= param[1][1];
sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]); int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE;
size_t n = snprintf(varDataVal(pCtx->aOutputBuf), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}",
param[0][2], param[1][2]);
varDataSetLen(pCtx->aOutputBuf, n);
doFinalizer(pCtx); doFinalizer(pCtx);
} }

2
src/client/src/tscSQLParser.c
View File

@ -3792,6 +3792,8 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr,
tSQLExprDestroy(p1); tSQLExprDestroy(p1);
tExprTreeDestroy(&p, NULL); tExprTreeDestroy(&p, NULL);
taosArrayDestroy(colList);
} }
pCondExpr->pTagCond = NULL; pCondExpr->pTagCond = NULL;

27
src/client/src/tscUtil.c
View File

@ -1209,18 +1209,18 @@ void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) {
} }
} }
void tscColumnListDestroy(SArray* pColumnBaseInfo) { void tscColumnListDestroy(SArray* pColumnList) {
if (pColumnBaseInfo == NULL) { if (pColumnList == NULL) {
return; return;
} }
size_t num = taosArrayGetSize(pColumnBaseInfo); size_t num = taosArrayGetSize(pColumnList);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i); SColumn* pCol = taosArrayGetP(pColumnList, i);
tscColumnDestroy(pCol); tscColumnDestroy(pCol);
} }
taosArrayDestroy(pColumnBaseInfo); taosArrayDestroy(pColumnList);
} }
/* /*
@ -1572,7 +1572,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
assert(pQueryInfo->exprList == NULL); assert(pQueryInfo->exprList == NULL);
pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
} }
int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
@ -1666,8 +1666,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
assert(pTableMetaInfo != NULL); assert(pTableMetaInfo != NULL);
if (name != NULL) { if (name != NULL) {
assert(strlen(name) <= TSDB_TABLE_ID_LEN); strncpy(pTableMetaInfo->name, name, TSDB_TABLE_ID_LEN);
strcpy(pTableMetaInfo->name, name);
} }
pTableMetaInfo->pTableMeta = pTableMeta; pTableMetaInfo->pTableMeta = pTableMeta;
@ -1678,10 +1677,9 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST
memcpy(pTableMetaInfo->vgroupList, vgroupList, size); memcpy(pTableMetaInfo->vgroupList, vgroupList, size);
} }
if (pTagCols == NULL) { pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); if (pTagCols != NULL) {
} else { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1);
pTableMetaInfo->tagColList = taosArrayClone(pTagCols);
} }
pQueryInfo->numOfTables += 1; pQueryInfo->numOfTables += 1;
@ -1794,7 +1792,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
tscFreeSqlObj(pNew); tscFreeSqlObj(pNew);
return NULL; return NULL;
} }
tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex);
// set the correct query type // set the correct query type
@ -1853,7 +1851,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
if (pPrevSql == NULL) { if (pPrevSql == NULL) {
STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name);
// todo handle error
assert(pTableMeta != NULL);
pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
} else { // transfer the ownership of pTableMeta to the newly create sql object. } else { // transfer the ownership of pTableMeta to the newly create sql object.
STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);

1
src/common/inc/tname.h
View File

@ -23,4 +23,5 @@ void extractTableName(const char *tableId, char *name);
char* extractDBName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name);
#endif // TDENGINE_NAME_H #endif // TDENGINE_NAME_H

5
src/inc/taosdef.h
View File

@ -32,6 +32,9 @@ extern "C" {
#define TSKEY int64_t #define TSKEY int64_t
#endif #endif
#define TSWINDOW_INITIALIZER {INT64_MIN, INT64_MAX};
#define TSKEY_INITIAL_VAL INT64_MIN
// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
typedef int32_t VarDataOffsetT; typedef int32_t VarDataOffsetT;
typedef int16_t VarDataLenT; typedef int16_t VarDataLenT;
@ -341,8 +344,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_MAX_DBS 100 #define TSDB_MAX_DBS 100
#define TSDB_MAX_VGROUPS 1000 #define TSDB_MAX_VGROUPS 1000
#define TSDB_MAX_SUPER_TABLES 100 #define TSDB_MAX_SUPER_TABLES 100
#define TSDB_MAX_NORMAL_TABLES 1000
#define TSDB_MAX_CHILD_TABLES 100000
#define TSDB_PORT_DNODESHELL 0 #define TSDB_PORT_DNODESHELL 0
#define TSDB_PORT_DNODEDNODE 5 #define TSDB_PORT_DNODEDNODE 5

31
src/query/src/queryExecutor.c
View File

@ -507,7 +507,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
w.ekey = pQuery->window.ekey; w.ekey = pQuery->window.ekey;
} }
assert(ts >= w.skey && ts <= w.ekey && w.skey != 0); assert(ts >= w.skey && ts <= w.ekey);
return w; return w;
} }
@ -624,7 +624,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL);
} else { // set the current index to be the last unclosed window } else { // set the current index to be the last unclosed window
int32_t i = 0; int32_t i = 0;
int64_t skey = 0; int64_t skey = TSKEY_INITIAL_VAL;
for (i = 0; i < pWindowResInfo->size; ++i) { for (i = 0; i < pWindowResInfo->size; ++i) {
SWindowResult *pResult = &pWindowResInfo->pResult[i]; SWindowResult *pResult = &pWindowResInfo->pResult[i];
@ -642,7 +642,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
} }
// all windows are closed, set the last one to be the skey // all windows are closed, set the last one to be the skey
if (skey == 0) { if (skey == TSKEY_INITIAL_VAL) {
assert(i == pWindowResInfo->size); assert(i == pWindowResInfo->size);
pWindowResInfo->curIndex = pWindowResInfo->size - 1; pWindowResInfo->curIndex = pWindowResInfo->size - 1;
} else { } else {
@ -660,7 +660,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, n); qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, n);
} }
assert(pWindowResInfo->prevSKey != 0); assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL);
} }
static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn,
@ -3080,6 +3080,9 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
int32_t functId = pQuery->pSelectExpr[j].base.functionId; int32_t functId = pQuery->pSelectExpr[j].base.functionId;
SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j];
if (pCtx->resultInfo == NULL) {
continue; // resultInfo is NULL, means no data checked in previous scan
}
if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) ||
((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) {
@ -3590,7 +3593,6 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
if (pTableQueryInfo->queryRangeSet) { if (pTableQueryInfo->queryRangeSet) {
pTableQueryInfo->lastKey = key; pTableQueryInfo->lastKey = key;
} else { } else {
// pQuery->window.skey = key;
pTableQueryInfo->win.skey = key; pTableQueryInfo->win.skey = key;
STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey};
@ -3613,18 +3615,16 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w);
pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp
if (pWindowResInfo->prevSKey == 0) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
if (QUERY_IS_ASC_QUERY(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) {
pWindowResInfo->prevSKey = w.skey;
} else {
assert(win.ekey == pQuery->window.skey); assert(win.ekey == pQuery->window.skey);
pWindowResInfo->prevSKey = w.skey;
} }
pWindowResInfo->prevSKey = w.skey;
} }
pTableQueryInfo->queryRangeSet = 1; pTableQueryInfo->queryRangeSet = 1;
pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
pTableQueryInfo->win.skey = pTableQueryInfo->win.skey;
} }
} }
@ -4057,10 +4057,11 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) {
* pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is
* not valid. otherwise, we only forward pQuery->limit.offset number of points * not valid. otherwise, we only forward pQuery->limit.offset number of points
*/ */
assert(pRuntimeEnv->windowResInfo.prevSKey == 0); assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL);
TSKEY skey1, ekey1; TSKEY skey1, ekey1;
STimeWindow w = {0}; STimeWindow w = TSWINDOW_INITIALIZER;
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
STableQueryInfo *pTableQueryInfo = pQuery->current; STableQueryInfo *pTableQueryInfo = pQuery->current;
@ -4730,7 +4731,7 @@ static void doRestoreContext(SQInfo *pQInfo) {
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->pTSBuf != NULL) {
pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order);
} }
switchCtxOrder(pRuntimeEnv); switchCtxOrder(pRuntimeEnv);

5
src/tsdb/src/tsdbRead.c
View File

@ -1346,7 +1346,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
int32_t type = 0; int32_t type = 0;
int32_t bytes = 0; int32_t bytes = 0;
if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor extract method , to queryExecutor to generate tags values
f1 = (char*) pTable1->name; f1 = (char*) pTable1->name;
f2 = (char*) pTable2->name; f2 = (char*) pTable2->name;
type = TSDB_DATA_TYPE_BINARY; type = TSDB_DATA_TYPE_BINARY;
@ -1354,7 +1354,8 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
} else { } else {
STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex);
bytes = pCol->bytes; bytes = pCol->bytes;
type = pCol->type;
f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
} }

12
tests/script/general/compute/leastsquare.sim
View File

@ -48,41 +48,41 @@ $tb = $tbPrefix . $i
sql select leastsquares(tbcol, 1, 1) from $tb sql select leastsquares(tbcol, 1, 1) from $tb
print ===> $data00 print ===> $data00
if $data00 != @(1.000000, 1.000000)@ then if $data00 != @{slop:1.000000, intercept:1.000000}@ then
return -1 return -1
endi endi
print =============== step3 print =============== step3
sql select leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m sql select leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m
print ===> $data00 print ===> $data00
if $data00 != @(1.000000, 1.000000)@ then if $data00 != @{slop:1.000000, intercept:1.000000}@ then
return -1 return -1
endi endi
print =============== step4 print =============== step4
sql select leastsquares(tbcol, 1, 1) as b from $tb sql select leastsquares(tbcol, 1, 1) as b from $tb
print ===> $data00 print ===> $data00
if $data00 != @(1.000000, 1.000000)@ then if $data00 != @{slop:1.000000, intercept:1.000000}@ then
return -1 return -1
endi endi
print =============== step5 print =============== step5
sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1m) sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1m)
print ===> $data01 print ===> $data01
if $data01 != @(1.000000, 1.000000)@ then if $data01 != @{slop:1.000000, intercept:1.000000}@ then
return -1 return -1
endi endi
sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1d) sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1d)
print ===> $data01 print ===> $data01
if $data01 != @(1.000000, 1.000000)@ then if $data01 != @{slop:1.000000, intercept:1.000000}@ then
return -1 return -1
endi endi
print =============== step6 print =============== step6
sql select leastsquares(tbcol, 1, 1) as b from $tb where ts < now + 4m interval(1m) sql select leastsquares(tbcol, 1, 1) as b from $tb where ts < now + 4m interval(1m)
print ===> $data01 print ===> $data01
if $data01 != @(1.000000, 1.000000)@ then if $data01 != @{slop:1.000000, intercept:1.000000}@ then
return -1 return -1
endi endi
print ===> $rows print ===> $rows