Merge branch 'develop' into jdbcfixes

This commit is contained in:
Ping Xiao 2020-07-09 16:50:48 +08:00
commit f1aa756d0e
130 changed files with 7262 additions and 3051 deletions

View File

@ -18,7 +18,7 @@ import (
"sync" "sync"
"time" "time"
_ "github.com/taosdata/TDengine/src/connector/go/taosSql" _ "github.com/taosdata/driver-go/taosSql"
) )
const ( const (

View File

@ -110,8 +110,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
//todo tags value as well as the table id structure needs refactor //todo tags value as well as the table id structure needs refactor
char *tsGetTagsValue(STableMeta *pMeta); char *tsGetTagsValue(STableMeta *pMeta);
void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -320,6 +320,8 @@ typedef struct SSqlStream {
SSqlObj *pSql; SSqlObj *pSql;
uint32_t streamId; uint32_t streamId;
char listed; char listed;
bool isProject;
int16_t precision;
int64_t num; // number of computing count int64_t num; // number of computing count
/* /*
@ -334,7 +336,6 @@ typedef struct SSqlStream {
int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed
int64_t interval; int64_t interval;
int64_t slidingTime; int64_t slidingTime;
int16_t precision;
void * pTimer; void * pTimer;
void (*fp)(); void (*fp)();

View File

@ -14,7 +14,6 @@
*/ */
#include "os.h" #include "os.h"
#include "qast.h"
#include "qextbuffer.h" #include "qextbuffer.h"
#include "qfill.h" #include "qfill.h"
#include "qhistogram.h" #include "qhistogram.h"
@ -23,6 +22,7 @@
#include "qtsbuf.h" #include "qtsbuf.h"
#include "taosdef.h" #include "taosdef.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "qast.h"
#include "tscLog.h" #include "tscLog.h"
#include "tscSubquery.h" #include "tscSubquery.h"
#include "tscompression.h" #include "tscompression.h"

View File

@ -406,7 +406,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
pSql->res.qhandle = 0x1; pSql->res.qhandle = 0x1;
pSql->res.numOfRows = 0; pSql->res.numOfRows = 0;
} else if (pCmd->command == TSDB_SQL_RESET_CACHE) { } else if (pCmd->command == TSDB_SQL_RESET_CACHE) {
taosCacheEmpty(tscCacheHandle,false); taosCacheEmpty(tscCacheHandle);
} else if (pCmd->command == TSDB_SQL_SERV_VERSION) { } else if (pCmd->command == TSDB_SQL_SERV_VERSION) {
tscProcessServerVer(pSql); tscProcessServerVer(pSql);
} else if (pCmd->command == TSDB_SQL_CLI_VERSION) { } else if (pCmd->command == TSDB_SQL_CLI_VERSION) {

View File

@ -18,19 +18,19 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "os.h" #include "os.h"
#include "qast.h"
#include "taos.h" #include "taos.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tstoken.h" #include "qast.h"
#include "tstrbuild.h" #include "tcompare.h"
#include "ttime.h" #include "tname.h"
#include "tscLog.h" #include "tscLog.h"
#include "tscUtil.h" #include "tscUtil.h"
#include "tschemautil.h" #include "tschemautil.h"
#include "tsclient.h" #include "tsclient.h"
#include "tstoken.h"
#include "tstrbuild.h"
#include "ttime.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tname.h"
#include "tcompare.h"
#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@ -90,6 +90,7 @@ static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryI
static int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString); static int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString);
static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type);
static int32_t validateEp(char* ep);
static int32_t validateDNodeConfig(tDCLSQL* pOptions); static int32_t validateDNodeConfig(tDCLSQL* pOptions);
static int32_t validateLocalConfig(tDCLSQL* pOptions); static int32_t validateLocalConfig(tDCLSQL* pOptions);
static int32_t validateColumnName(char* name); static int32_t validateColumnName(char* name);
@ -359,6 +360,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
case TSDB_SQL_CFG_DNODE: { case TSDB_SQL_CFG_DNODE: {
const char* msg2 = "invalid configure options or values"; const char* msg2 = "invalid configure options or values";
const char* msg3 = "invalid dnode ep";
/* validate the ip address */ /* validate the ip address */
tDCLSQL* pDCL = pInfo->pDCLInfo; tDCLSQL* pDCL = pInfo->pDCLInfo;
@ -375,6 +377,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n); strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n);
if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n); strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n);
if (pDCL->nTokens == 3) { if (pDCL->nTokens == 3) {
@ -654,11 +660,14 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
const char* msg0 = "sliding value too small"; const char* msg0 = "sliding value too small";
const char* msg1 = "sliding value no larger than the interval value"; const char* msg1 = "sliding value no larger than the interval value";
const char* msg2 = "sliding value can not less than 1% of interval value";
const static int32_t INTERVAL_SLIDING_FACTOR = 100;
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
SSQLToken* pSliding = &pQuerySql->sliding;
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
SSQLToken* pSliding = &pQuerySql->sliding;
if (pSliding->n != 0) { if (pSliding->n != 0) {
getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime);
if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) {
@ -676,6 +685,10 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
pQueryInfo->slidingTime = pQueryInfo->intervalTime; pQueryInfo->slidingTime = pQueryInfo->intervalTime;
} }
if ((pQueryInfo->intervalTime != 0) && (pQueryInfo->intervalTime/pQueryInfo->slidingTime > INTERVAL_SLIDING_FACTOR)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -4629,6 +4642,24 @@ typedef struct SDNodeDynConfOption {
int32_t len; // name string length int32_t len; // name string length
} SDNodeDynConfOption; } SDNodeDynConfOption;
int32_t validateEp(char* ep) {
char buf[TSDB_EP_LEN + 1] = {0};
tstrncpy(buf, ep, TSDB_EP_LEN);
char *pos = strchr(buf, ':');
if (NULL == pos) {
return TSDB_CODE_TSC_INVALID_SQL;
}
uint16_t port = atoi(pos+1);
if (0 == port) {
return TSDB_CODE_TSC_INVALID_SQL;
}
return TSDB_CODE_SUCCESS;
}
int32_t validateDNodeConfig(tDCLSQL* pOptions) { int32_t validateDNodeConfig(tDCLSQL* pOptions) {
if (pOptions->nTokens < 2 || pOptions->nTokens > 3) { if (pOptions->nTokens < 2 || pOptions->nTokens > 3) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
@ -6096,16 +6127,12 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
} }
} }
// NOTE: binary|nchar data allows the >|< type filter
if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
if (pRight->nodeType == TSQL_NODE_VALUE) { if (pRight->nodeType == TSQL_NODE_VALUE) {
if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) { if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
if ((pRight->pVal->nType == TSDB_DATA_TYPE_BINARY || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR)
&& (*pExpr)->_node.optr != TSDB_RELATION_LIKE) {
return TSDB_CODE_TSC_INVALID_SQL;
}
} }
} }
} }

View File

@ -215,25 +215,3 @@ __attribute__ ((unused)) static FORCE_INLINE size_t copy(char* dst, const char*
return len; return len;
} }
/*
* tablePrefix.columnName
* extract table name and save it in pTable, with only column name in pToken
*/
void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) {
const char sep = TS_PATH_DELIMITER[0];
if (pToken == pTable || pToken == NULL || pTable == NULL) {
return;
}
char* r = strnchr(pToken->z, sep, pToken->n, false);
if (r != NULL) { // record the table name token
pTable->n = r - pToken->z;
pTable->z = pToken->z;
r += 1;
pToken->n -= (r - pToken->z);
pToken->z = r;
}
}

View File

@ -438,8 +438,9 @@ void tscKillSTableQuery(SSqlObj *pSql) {
* here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause
* sub-queries not correctly released and master sql object of super table query reaches an abnormal state. * sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
*/ */
pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; rpcCancelRequest(pSub->pRpcCtx);
rpcCancelRequest(pSql->pSubs[i]->pRpcCtx); pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
tscQueueAsyncRes(pSub);
} }
/* /*
@ -1955,7 +1956,7 @@ int tscProcessUseDbRsp(SSqlObj *pSql) {
} }
int tscProcessDropDbRsp(SSqlObj *UNUSED_PARAM(pSql)) { int tscProcessDropDbRsp(SSqlObj *UNUSED_PARAM(pSql)) {
taosCacheEmpty(tscCacheHandle, false); taosCacheEmpty(tscCacheHandle);
return 0; return 0;
} }
@ -2001,7 +2002,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) {
if (isSuperTable) { // if it is a super table, reset whole query cache if (isSuperTable) { // if it is a super table, reset whole query cache
tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name);
taosCacheEmpty(tscCacheHandle, false); taosCacheEmpty(tscCacheHandle);
} }
} }

View File

@ -617,19 +617,18 @@ void taos_stop_query(TAOS_RES *res) {
if (pSql->signature != pSql) return; if (pSql->signature != pSql) return;
tscDebug("%p start to cancel query", res); tscDebug("%p start to cancel query", res);
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
tscKillSTableQuery(pSql); tscKillSTableQuery(pSql);
return;
} }
if (pSql->cmd.command >= TSDB_SQL_LOCAL) { if (pSql->cmd.command < TSDB_SQL_LOCAL) {
return; rpcCancelRequest(pSql->pRpcCtx);
} }
pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
tscQueueAsyncRes(pSql);
rpcCancelRequest(pSql->pRpcCtx);
tscDebug("%p query is cancelled", res); tscDebug("%p query is cancelled", res);
} }

View File

@ -71,6 +71,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
pSql->fp = tscProcessStreamQueryCallback; pSql->fp = tscProcessStreamQueryCallback;
pSql->param = pStream; pSql->param = pStream;
pSql->res.completed = false;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
@ -86,7 +87,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
// failed to get meter/metric meta, retry in 10sec. // failed to get meter/metric meta, retry in 10sec.
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision);
tscError("%p stream:%p,get metermeta failed, retry in %" PRId64 "ms", pStream->pSql, pStream, retryDelayTime); tscDebug("%p stream:%p,get metermeta failed, retry in %" PRId64 "ms", pStream->pSql, pStream, retryDelayTime);
tscSetRetryTimer(pStream, pSql, retryDelayTime); tscSetRetryTimer(pStream, pSql, retryDelayTime);
} else { } else {
@ -108,7 +109,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
tscDebug("%p add into timer", pSql); tscDebug("%p add into timer", pSql);
if (isProjectStream(pQueryInfo)) { if (pStream->isProject) {
/* /*
* pQueryInfo->window.ekey, which is the start time, does not change in case of * pQueryInfo->window.ekey, which is the start time, does not change in case of
* repeat first execution, once the first execution failed. * repeat first execution, once the first execution failed.
@ -121,7 +122,19 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
} }
} else { } else {
pQueryInfo->window.skey = pStream->stime - pStream->interval; pQueryInfo->window.skey = pStream->stime - pStream->interval;
pQueryInfo->window.ekey = pStream->stime - 1; int64_t etime = taosGetTimestamp(pStream->precision);
// delay to wait all data in last time window
if (pStream->precision == TSDB_TIME_PRECISION_MICRO) {
etime -= tsMaxStreamComputDelay * 1000l;
} else {
etime -= tsMaxStreamComputDelay;
}
if (etime > pStream->etime) {
etime = pStream->etime;
} else {
etime = pStream->stime + (etime - pStream->stime) / pStream->interval * pStream->interval;
}
pQueryInfo->window.ekey = etime;
} }
// launch stream computing in a new thread // launch stream computing in a new thread
@ -137,7 +150,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf
SSqlStream *pStream = (SSqlStream *)param; SSqlStream *pStream = (SSqlStream *)param;
if (tres == NULL || numOfRows < 0) { if (tres == NULL || numOfRows < 0) {
int64_t retryDelay = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); int64_t retryDelay = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision);
tscError("%p stream:%p, query data failed, code:%d, retry in %" PRId64 "ms", pStream->pSql, pStream, numOfRows, tscError("%p stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql, pStream, numOfRows,
retryDelay); retryDelay);
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0);
@ -151,17 +164,45 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf
taos_fetch_rows_a(tres, tscProcessStreamRetrieveResult, param); taos_fetch_rows_a(tres, tscProcessStreamRetrieveResult, param);
} }
static void tscSetTimestampForRes(SSqlStream *pStream, SSqlObj *pSql) { // no need to be called as this is alreay done in the query
SSqlRes *pRes = &pSql->res; static void tscStreamFillTimeGap(SSqlStream* pStream, TSKEY ts) {
#if 0
int64_t timestamp = *(int64_t *)pRes->data; SSqlObj * pSql = pStream->pSql;
int64_t actualTimestamp = pStream->stime - pStream->interval; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (timestamp != actualTimestamp) { if (pQueryInfo->fillType != TSDB_FILL_SET_VALUE && pQueryInfo->fillType != TSDB_FILL_NULL) {
// reset the timestamp of each agg point by using start time of each interval return;
*((int64_t *)pRes->data) = actualTimestamp;
tscWarn("%p stream:%p, timestamp of points is:%" PRId64 ", reset to %" PRId64, pSql, pStream, timestamp, actualTimestamp);
} }
SSqlRes *pRes = &pSql->res;
/* failed to retrieve any result in this retrieve */
pSql->res.numOfRows = 1;
void *row[TSDB_MAX_COLUMNS] = {0};
char tmpRes[TSDB_MAX_BYTES_PER_ROW] = {0};
void *oldPtr = pSql->res.data;
pSql->res.data = tmpRes;
int32_t rowNum = 0;
while (pStream->stime + pStream->slidingTime < ts) {
pStream->stime += pStream->slidingTime;
*(TSKEY*)row[0] = pStream->stime;
for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i);
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->fillVal[i]), pField->bytes, pField->type);
row[i] = pSql->res.data + offset;
}
(*pStream->fp)(pStream->param, pSql, row);
++rowNum;
}
if (rowNum > 0) {
tscDebug("%p stream:%p %d rows padded", pSql, pStream, rowNum);
}
pRes->numOfRows = 0;
pRes->data = oldPtr;
#endif
} }
static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows) { static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows) {
@ -170,7 +211,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
if (pSql == NULL || numOfRows < 0) { if (pSql == NULL || numOfRows < 0) {
int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision);
tscError("%p stream:%p, retrieve data failed, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime); tscError("%p stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime);
tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime);
return; return;
@ -180,16 +221,11 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
if (numOfRows > 0) { // when reaching here the first execution of stream computing is successful. if (numOfRows > 0) { // when reaching here the first execution of stream computing is successful.
pStream->numOfRes += numOfRows; pStream->numOfRes += numOfRows;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
for(int32_t i = 0; i < numOfRows; ++i) { for(int32_t i = 0; i < numOfRows; ++i) {
TAOS_ROW row = taos_fetch_row(res); TAOS_ROW row = taos_fetch_row(res);
tscDebug("%p stream:%p fetch result", pSql, pStream); tscDebug("%p stream:%p fetch result", pSql, pStream);
if (isProjectStream(pQueryInfo)) { tscStreamFillTimeGap(pStream, *(TSKEY*)row[0]);
pStream->stime = *(TSKEY *)row[0]; pStream->stime = *(TSKEY *)row[0];
} else {
tscSetTimestampForRes(pStream, pSql);
}
// user callback function // user callback function
(*pStream->fp)(pStream->param, res, row); (*pStream->fp)(pStream->param, res, row);
@ -199,55 +235,18 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream); taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream);
} else { // numOfRows == 0, all data has been retrieved } else { // numOfRows == 0, all data has been retrieved
pStream->useconds += pSql->res.useconds; pStream->useconds += pSql->res.useconds;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (pStream->numOfRes == 0) { if (pStream->numOfRes == 0) {
if (pQueryInfo->fillType == TSDB_FILL_SET_VALUE || pQueryInfo->fillType == TSDB_FILL_NULL) { if (pStream->isProject) {
SSqlRes *pRes = &pSql->res;
/* failed to retrieve any result in this retrieve */
pSql->res.numOfRows = 1;
void *row[TSDB_MAX_COLUMNS] = {0};
char tmpRes[TSDB_MAX_BYTES_PER_ROW] = {0};
void *oldPtr = pSql->res.data;
pSql->res.data = tmpRes;
for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i);
TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->fillVal[i]), pField->bytes, pField->type);
row[i] = pSql->res.data + offset;
}
tscSetTimestampForRes(pStream, pSql);
row[0] = pRes->data;
// char result[512] = {0};
// taos_print_row(result, row, pQueryInfo->fieldsInfo.pFields, pQueryInfo->fieldsInfo.numOfOutput);
// tscInfo("%p stream:%p query result: %s", pSql, pStream, result);
tscDebug("%p stream:%p fetch result", pSql, pStream);
// user callback function
(*pStream->fp)(pStream->param, res, row);
pRes->numOfRows = 0;
pRes->data = oldPtr;
} else if (isProjectStream(pQueryInfo)) {
/* no resuls in the query range, retry */ /* no resuls in the query range, retry */
// todo set retry dynamic time // todo set retry dynamic time
int32_t retry = tsProjectExecInterval; int32_t retry = tsProjectExecInterval;
tscError("%p stream:%p, retrieve no data, code:%d, retry in %" PRId32 "ms", pSql, pStream, numOfRows, retry); tscError("%p stream:%p, retrieve no data, code:0x%08x, retry in %" PRId32 "ms", pSql, pStream, numOfRows, retry);
tscSetRetryTimer(pStream, pStream->pSql, retry); tscSetRetryTimer(pStream, pStream->pSql, retry);
return; return;
} }
} else { } else if (pStream->isProject) {
if (isProjectStream(pQueryInfo)) { pStream->stime += 1;
pStream->stime += 1;
}
} }
tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name, tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name,
@ -262,10 +261,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
} }
static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer) { static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
int64_t delay = getDelayValueAfterTimewindowClosed(pStream, timer); int64_t delay = getDelayValueAfterTimewindowClosed(pStream, timer);
if (isProjectStream(pQueryInfo)) { if (pStream->isProject) {
int64_t now = taosGetTimestamp(pStream->precision); int64_t now = taosGetTimestamp(pStream->precision);
int64_t etime = now > pStream->etime ? pStream->etime : now; int64_t etime = now > pStream->etime ? pStream->etime : now;
@ -323,8 +321,7 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) {
static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) {
int64_t timer = 0; int64_t timer = 0;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); if (pStream->isProject) {
if (isProjectStream(pQueryInfo)) {
/* /*
* for project query, no mater fetch data successfully or not, next launch will issue * for project query, no mater fetch data successfully or not, next launch will issue
* more than the sliding time window * more than the sliding time window
@ -342,7 +339,6 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) {
return; return;
} }
} else { } else {
pStream->stime += pStream->slidingTime;
if ((pStream->stime - pStream->interval) >= pStream->etime) { if ((pStream->stime - pStream->interval) >= pStream->etime) {
tscDebug("%p stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql, pStream, tscDebug("%p stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql, pStream,
pStream->stime, pStream->etime); pStream->stime, pStream->etime);
@ -409,14 +405,16 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
pStream->slidingTime = pQueryInfo->slidingTime; pStream->slidingTime = pQueryInfo->slidingTime;
pQueryInfo->intervalTime = 0; // clear the interval value to avoid the force time window split by query processor if (pStream->isProject) {
pQueryInfo->slidingTime = 0; pQueryInfo->intervalTime = 0; // clear the interval value to avoid the force time window split by query processor
pQueryInfo->slidingTime = 0;
}
} }
static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) { static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (isProjectStream(pQueryInfo)) { if (pStream->isProject) {
// no data in table, flush all data till now to destination meter, 10sec delay // no data in table, flush all data till now to destination meter, 10sec delay
pStream->interval = tsProjectExecInterval; pStream->interval = tsProjectExecInterval;
pStream->slidingTime = tsProjectExecInterval; pStream->slidingTime = tsProjectExecInterval;
@ -489,7 +487,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream)); SSqlStream *pStream = (SSqlStream *)calloc(1, sizeof(SSqlStream));
if (pStream == NULL) { if (pStream == NULL) {
tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code); tscError("%p open stream failed, sql:%s, reason:%s, code:0x%08x", pSql, sqlstr, pCmd->payload, pRes->code);
tscFreeSqlObj(pSql); tscFreeSqlObj(pSql);
return NULL; return NULL;
} }
@ -514,7 +512,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
if (pRes->code != TSDB_CODE_SUCCESS) { if (pRes->code != TSDB_CODE_SUCCESS) {
setErrorInfo(pSql, pRes->code, pCmd->payload); setErrorInfo(pSql, pRes->code, pCmd->payload);
tscError("%p open stream failed, sql:%s, reason:%s, code:%d", pSql, sqlstr, pCmd->payload, pRes->code); tscError("%p open stream failed, sql:%s, reason:%s, code:0x%08x", pSql, sqlstr, pCmd->payload, pRes->code);
tscFreeSqlObj(pSql); tscFreeSqlObj(pSql);
return NULL; return NULL;
} }
@ -523,6 +521,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
pStream->isProject = isProjectStream(pQueryInfo);
pStream->fp = fp; pStream->fp = fp;
pStream->callback = callback; pStream->callback = callback;
pStream->param = param; pStream->param = param;
@ -565,6 +564,8 @@ void taos_close_stream(TAOS_STREAM *handle) {
taosTmrStopA(&(pStream->pTimer)); taosTmrStopA(&(pStream->pTimer));
tscDebug("%p stream:%p is closed", pSql, pStream); tscDebug("%p stream:%p is closed", pSql, pStream);
// notify CQ to release the pStream object
pStream->fp(pStream->param, NULL, NULL);
tscFreeSqlObj(pSql); tscFreeSqlObj(pSql);
pStream->pSql = NULL; pStream->pSql = NULL;

View File

@ -14,12 +14,12 @@
*/ */
#include "os.h" #include "os.h"
#include "tscSubquery.h" #include "qtsbuf.h"
#include "qast.h" #include "qast.h"
#include "tcompare.h" #include "tcompare.h"
#include "tschemautil.h"
#include "qtsbuf.h"
#include "tscLog.h" #include "tscLog.h"
#include "tscSubquery.h"
#include "tschemautil.h"
#include "tsclient.h" #include "tsclient.h"
typedef struct SInsertSupporter { typedef struct SInsertSupporter {
@ -57,10 +57,15 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
pSubQueryInfo1->tsBuf = output1; pSubQueryInfo1->tsBuf = output1;
pSubQueryInfo2->tsBuf = output2; pSubQueryInfo2->tsBuf = output2;
// no result generated, return directly
if (pSupporter1->pTSBuf == NULL || pSupporter2->pTSBuf == NULL) {
tscDebug("%p at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql);
return 0;
}
tsBufResetPos(pSupporter1->pTSBuf); tsBufResetPos(pSupporter1->pTSBuf);
tsBufResetPos(pSupporter2->pTSBuf); tsBufResetPos(pSupporter2->pTSBuf);
// TODO add more details information
if (!tsBufNextPos(pSupporter1->pTSBuf)) { if (!tsBufNextPos(pSupporter1->pTSBuf)) {
tsBufFlush(output1); tsBufFlush(output1);
tsBufFlush(output2); tsBufFlush(output2);
@ -210,6 +215,7 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
pSupporter->f = NULL; pSupporter->f = NULL;
} }
tfree(pSupporter->pIdTagList);
tscTagCondRelease(&pSupporter->tagCond); tscTagCondRelease(&pSupporter->tagCond);
free(pSupporter); free(pSupporter);
} }
@ -420,43 +426,6 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {
pQueryInfo->window = *win; pQueryInfo->window = *win;
} }
static UNUSED_FUNC void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj* pSql) {
SSqlObj* pParentSql = pSupporter->pObj;
SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex);
// if (tscNonOrderedProjectionQueryOnSTable(pParentQueryInfo, 0)) {
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
// assert(pQueryInfo->numOfTables == 1);
//
// // for projection query, need to try next vnode
//// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes;
// int32_t totalVnode = 0;
// if ((++pTableMetaInfo->vgroupIndex) < totalVnode) {
// tscDebug("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql,
// pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotal);
//
// pSql->cmd.command = TSDB_SQL_SELECT;
// pSql->fp = tscJoinQueryCallback;
// tscProcessSql(pSql);
//
// return;
// }
// }
SJoinSupporter* p1 = pParentSql->pSubs[0]->param;
SJoinSupporter* p2 = pParentSql->pSubs[1]->param;
STimeWindow win = TSWINDOW_INITIALIZER;
int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &win);
if (num <= 0) { // no result during ts intersect
tscDebug("%p free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql);
} else {
updateQueryTimeRange(pParentQueryInfo, &win);
tscLaunchRealSubqueries(pParentSql);
}
}
int32_t tscCompareTidTags(const void* p1, const void* p2) { int32_t tscCompareTidTags(const void* p1, const void* p2) {
const STidTags* t1 = (const STidTags*) varDataVal(p1); const STidTags* t1 = (const STidTags*) varDataVal(p1);
const STidTags* t2 = (const STidTags*) varDataVal(p2); const STidTags* t2 = (const STidTags*) varDataVal(p2);
@ -713,9 +682,12 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
SArray *s1 = NULL, *s2 = NULL; SArray *s1 = NULL, *s2 = NULL;
getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2); getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2);
if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return. if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return.
tscDebug("%p free all sub SqlObj and quit", pParentSql); tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql); freeJoinSubqueryObj(pParentSql);
// set no result command
pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else { } else {
// proceed to for ts_comp query // proceed to for ts_comp query
SSqlCmd* pSubCmd1 = &pParentSql->pSubs[0]->cmd; SSqlCmd* pSubCmd1 = &pParentSql->pSubs[0]->cmd;
@ -846,7 +818,10 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
if (num <= 0) { // no result during ts intersect if (num <= 0) { // no result during ts intersect
tscDebug("%p no results generated in ts intersection, free all sub SqlObj and quit", pParentSql); tscDebug("%p no results generated in ts intersection, free all sub SqlObj and quit", pParentSql);
freeJoinSubqueryObj(pParentSql); freeJoinSubqueryObj(pParentSql);
// set no result command
pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
return; return;
} }

View File

@ -14,21 +14,21 @@
*/ */
#include "os.h" #include "os.h"
#include "qast.h" #include "hash.h"
#include "tscUtil.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "qast.h"
#include "tcache.h" #include "tcache.h"
#include "tkey.h" #include "tkey.h"
#include "tmd5.h" #include "tmd5.h"
#include "tscProfile.h"
#include "tscLocalMerge.h" #include "tscLocalMerge.h"
#include "tscLog.h"
#include "tscProfile.h"
#include "tscSubquery.h" #include "tscSubquery.h"
#include "tschemautil.h" #include "tschemautil.h"
#include "tsclient.h" #include "tsclient.h"
#include "ttimer.h" #include "ttimer.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tscLog.h"
#include "tscUtil.h"
#include "hash.h"
static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache);

View File

@ -3,6 +3,7 @@
#include "os.h" #include "os.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tstoken.h"
typedef struct SDataStatis { typedef struct SDataStatis {
int16_t colId; int16_t colId;
@ -23,6 +24,8 @@ void extractTableName(const char *tableId, char *name);
char* extractDBName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name);
void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable);
SSchema tGetTableNameColumnSchema(); SSchema tGetTableNameColumnSchema();
bool tscValidateTableNameLength(size_t len); bool tscValidateTableNameLength(size_t len);

View File

@ -105,3 +105,26 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
} }
return start; return start;
} }
/*
* tablePrefix.columnName
* extract table name and save it in pTable, with only column name in pToken
*/
void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) {
const char sep = TS_PATH_DELIMITER[0];
if (pToken == pTable || pToken == NULL || pTable == NULL) {
return;
}
char* r = strnchr(pToken->z, sep, pToken->n, false);
if (r != NULL) { // record the table name token
pTable->n = r - pToken->z;
pTable->z = pToken->z;
r += 1;
pToken->n -= (r - pToken->z);
pToken->z = r;
}
}

View File

@ -1,44 +0,0 @@
#!/bin/bash
ulimit -c unlimited
function buildTDengine {
cd /root/TDengine
git remote update
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
LOCAL_COMMIT=`git rev-parse --short @`
echo " LOCAL: $LOCAL_COMMIT"
echo "REMOTE: $REMOTE_COMMIT"
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
echo "repo up-to-date"
else
echo "repo need to pull"
git pull
LOCAL_COMMIT=`git rev-parse --short @`
cd /root/TDengine/debug
rm -rf /root/TDengine/debug/*
cmake ..
make > /dev/null
make install
fi
}
function restartTaosd {
systemctl stop taosd
pkill -KILL -x taosd
sleep 10
logDir=`grep 'logDir' /etc/taos/taos.cfg|awk 'END{print $2}'`
dataDir=`grep 'dataDir' /etc/taos/taos.cfg|awk '{print $2}'`
rm -rf $logDir/*
rm -rf $dataDir/*
taosd 2>&1 > /dev/null &
sleep 10
}
buildTDengine
restartTaosd

View File

@ -9,8 +9,7 @@ public class TDNode {
private int index; private int index;
private int running; private int running;
private int deployed; private int deployed;
private boolean testCluster; private boolean testCluster;
private int valgrind;
private String path; private String path;
private String cfgDir; private String cfgDir;
private String dataDir; private String dataDir;
@ -21,18 +20,13 @@ public class TDNode {
this.index = index; this.index = index;
running = 0; running = 0;
deployed = 0; deployed = 0;
testCluster = false; testCluster = false;
valgrind = 0;
} }
public void setPath(String path) { public void setPath(String path) {
this.path = path; this.path = path;
} }
public void setValgrind(int valgrind) {
this.valgrind = valgrind;
}
public void setTestCluster(boolean testCluster) { public void setTestCluster(boolean testCluster) {
this.testCluster = testCluster; this.testCluster = testCluster;
} }
@ -58,7 +52,7 @@ public class TDNode {
public void start() { public void start() {
String selfPath = System.getProperty("user.dir"); String selfPath = System.getProperty("user.dir");
String binPath = ""; String binPath = "";
String projDir = selfPath + "../../../../"; String projDir = selfPath + "/../../../../";
try { try {
ArrayList<String> taosdPath = new ArrayList<>(); ArrayList<String> taosdPath = new ArrayList<>();
@ -95,15 +89,9 @@ public class TDNode {
return; return;
} }
String cmd = ""; String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & ";
if(this.valgrind == 0) { System.out.println("start taosd cmd: " + cmd);
cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & ";
System.out.println("start taosd cmd: " + cmd);
} else {
String valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reac∏hable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes";
cmd = "nohup " + valgrindCmdline + " " + binPath + " -c " + this.cfgDir + " 2>&1 & ";
}
try{ try{
Runtime.getRuntime().exec(cmd); Runtime.getRuntime().exec(cmd);
TimeUnit.SECONDS.sleep(5); TimeUnit.SECONDS.sleep(5);
@ -115,12 +103,7 @@ public class TDNode {
} }
public void stop() { public void stop() {
String toBeKilled = ""; String toBeKilled = "taosd";
if (this.valgrind == 0) {
toBeKilled = "taosd";
} else {
toBeKilled = "valgrind.bin";
}
if (this.running != 0) { if (this.running != 0) {
String psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print " + toBeKilled + "}'"; String psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print " + toBeKilled + "}'";
@ -136,10 +119,6 @@ public class TDNode {
String fuserCmd = "fuser -k -n tcp " + port; String fuserCmd = "fuser -k -n tcp " + port;
Runtime.getRuntime().exec(fuserCmd).waitFor(); Runtime.getRuntime().exec(fuserCmd).waitFor();
} }
if (this.valgrind == 1) {
TimeUnit.SECONDS.sleep(2);
}
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }

View File

@ -5,8 +5,7 @@ import java.util.*;
public class TDNodes { public class TDNodes {
private ArrayList<TDNode> tdNodes; private ArrayList<TDNode> tdNodes;
private boolean testCluster; private boolean testCluster;
private int valgrind;
public TDNodes () { public TDNodes () {
tdNodes = new ArrayList<>(); tdNodes = new ArrayList<>();
@ -21,13 +20,7 @@ public class TDNodes {
Process ps = Runtime.getRuntime().exec(psCmd); Process ps = Runtime.getRuntime().exec(psCmd);
ps.waitFor(); ps.waitFor();
String killCmd = "kill -9 " + ps.pid(); String killCmd = "kill -9 " + ps.pid();
Runtime.getRuntime().exec(killCmd).waitFor(); Runtime.getRuntime().exec(killCmd).waitFor();
psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'";
ps = Runtime.getRuntime().exec(psCmd);
ps.waitFor();
killCmd = "kill -9 " + ps.pid();
Runtime.getRuntime().exec(killCmd).waitFor();
String binPath = System.getProperty("user.dir"); String binPath = System.getProperty("user.dir");
binPath += "/../../../debug"; binPath += "/../../../debug";
@ -54,10 +47,6 @@ public class TDNodes {
this.testCluster = testCluster; this.testCluster = testCluster;
} }
public void setValgrid(int valgrind) {
this.valgrind = valgrind;
}
public void check(int index) { public void check(int index) {
if(index < 1 || index > 10) { if(index < 1 || index > 10) {
System.out.println("index: " + index + " should on a scale of [1, 10]"); System.out.println("index: " + index + " should on a scale of [1, 10]");
@ -70,8 +59,7 @@ public class TDNodes {
File file = new File(System.getProperty("user.dir") + "/../../../"); File file = new File(System.getProperty("user.dir") + "/../../../");
String projectRealPath = file.getCanonicalPath(); String projectRealPath = file.getCanonicalPath();
check(index); check(index);
tdNodes.get(index - 1).setTestCluster(this.testCluster); tdNodes.get(index - 1).setTestCluster(this.testCluster);
tdNodes.get(index - 1).setValgrind(valgrind);
tdNodes.get(index - 1).setPath(projectRealPath); tdNodes.get(index - 1).setPath(projectRealPath);
tdNodes.get(index - 1).deploy(); tdNodes.get(index - 1).deploy();
} catch (Exception e) { } catch (Exception e) {

View File

@ -9,20 +9,18 @@ import org.junit.BeforeClass;
public class BaseTest { public class BaseTest {
private static boolean testCluster = false; private static boolean testCluster = false;
private static String deployPath = System.getProperty("user.dir"); private static String deployPath = System.getProperty("user.dir");
private static int valgrind = 0;
private static TDNodes tdNodes = new TDNodes(); private static TDNodes tdNodes = new TDNodes();
@BeforeClass @BeforeClass
public static void setUpEvn() { public static void setupEnv() {
try{ try{
File file = new File(deployPath + "/../../../"); File file = new File(deployPath + "/../../../");
String rootPath = file.getCanonicalPath(); String rootPath = file.getCanonicalPath();
tdNodes.setPath(rootPath); tdNodes.setPath(rootPath);
tdNodes.setTestCluster(testCluster); tdNodes.setTestCluster(testCluster);
tdNodes.setValgrid(valgrind);
tdNodes.deploy(1); tdNodes.deploy(1);
tdNodes.start(1); tdNodes.start(1);

View File

@ -1,6 +1,7 @@
from .cinterface import CTaosInterface from .cinterface import CTaosInterface
from .error import * from .error import *
from .constants import FieldType from .constants import FieldType
import threading
# querySeqNum = 0 # querySeqNum = 0
@ -37,6 +38,7 @@ class TDengineCursor(object):
self._block_iter = 0 self._block_iter = 0
self._affected_rows = 0 self._affected_rows = 0
self._logfile = "" self._logfile = ""
self._threadId = threading.get_ident()
if connection is not None: if connection is not None:
self._connection = connection self._connection = connection
@ -103,6 +105,12 @@ class TDengineCursor(object):
def execute(self, operation, params=None): def execute(self, operation, params=None):
"""Prepare and execute a database operation (query or command). """Prepare and execute a database operation (query or command).
""" """
# if threading.get_ident() != self._threadId:
# info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
# raise OperationalError(info)
# print(info)
# return None
if not operation: if not operation:
return None return None
@ -188,6 +196,11 @@ class TDengineCursor(object):
def fetchall(self): def fetchall(self):
"""Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.
""" """
# if threading.get_ident() != self._threadId:
# info ="[WARNING] Cursor fetchall:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
# raise OperationalError(info)
# print(info)
# return None
if self._result is None or self._fields is None: if self._result is None or self._fields is None:
raise OperationalError("Invalid use of fetchall") raise OperationalError("Invalid use of fetchall")
@ -232,6 +245,12 @@ class TDengineCursor(object):
def _handle_result(self): def _handle_result(self):
"""Handle the return result from query. """Handle the return result from query.
""" """
# if threading.get_ident() != self._threadId:
# info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
# raise OperationalError(info)
# print(info)
# return None
self._description = [] self._description = []
for ele in self._fields: for ele in self._fields:
self._description.append( self._description.append(

View File

@ -109,6 +109,8 @@ void cqClose(void *handle) {
while (pObj) { while (pObj) {
SCqObj *pTemp = pObj; SCqObj *pTemp = pObj;
pObj = pObj->next; pObj = pObj->next;
tdFreeSchema(pTemp->pSchema);
tfree(pTemp->sqlStr);
free(pTemp); free(pTemp);
} }
@ -242,6 +244,10 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
SCqObj *pObj = (SCqObj *)param; SCqObj *pObj = (SCqObj *)param;
if (tres == NULL && row == NULL) {
pObj->pStream = NULL;
return;
}
SCqContext *pContext = pObj->pContext; SCqContext *pContext = pObj->pContext;
STSchema *pSchema = pObj->pSchema; STSchema *pSchema = pObj->pSchema;
if (pObj->pStream == NULL) return; if (pObj->pStream == NULL) return;
@ -263,8 +269,14 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
void* val = row[i]; void* val = row[i];
if (val == NULL) { if (val == NULL) {
val = getNullValue(c->type); val = getNullValue(c->type);
} else if (IS_VAR_DATA_TYPE(c->type)) { } else if (c->type == TSDB_DATA_TYPE_BINARY) {
val = ((char*)val) - sizeof(VarDataLenT); val = ((char*)val) - sizeof(VarDataLenT);
} else if (c->type == TSDB_DATA_TYPE_NCHAR) {
char buf[TSDB_MAX_NCHAR_LEN];
size_t len = taos_fetch_lengths(tres)[i];
taosMbsToUcs4(val, len, buf, sizeof(buf), &len);
memcpy(val + sizeof(VarDataLenT), buf, len);
varDataLen(val) = len;
} }
tdAppendColVal(trow, val, c->type, c->bytes, c->offset); tdAppendColVal(trow, val, c->type, c->bytes, c->offset);
} }

View File

@ -131,8 +131,8 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
taosFreeQitem(pWrite); taosFreeQitem(pWrite);
} }
void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) { void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
SMnodeMsg *pWrite = pRaw; SMnodeMsg *pWrite = pMsg;
if (pWrite == NULL) return; if (pWrite == NULL) return;
if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {

View File

@ -402,6 +402,7 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
void *pVnode = vnodeAcquireVnode(pCreate->cfg.vgId); void *pVnode = vnodeAcquireVnode(pCreate->cfg.vgId);
if (pVnode != NULL) { if (pVnode != NULL) {
dDebug("vgId:%d, already exist, processed as alter msg", pCreate->cfg.vgId);
int32_t code = vnodeAlter(pVnode, pCreate); int32_t code = vnodeAlter(pVnode, pCreate);
vnodeRelease(pVnode); vnodeRelease(pVnode);
return code; return code;

View File

@ -87,8 +87,8 @@ int32_t qKillQuery(qinfo_t qinfo);
void* qOpenQueryMgmt(int32_t vgId); void* qOpenQueryMgmt(int32_t vgId);
void qSetQueryMgmtClosed(void* pExecutor); void qSetQueryMgmtClosed(void* pExecutor);
void qCleanupQueryMgmt(void* pExecutor); void qCleanupQueryMgmt(void* pExecutor);
void** qRegisterQInfo(void* pMgmt, void* qInfo); void** qRegisterQInfo(void* pMgmt, uint64_t qInfo);
void** qAcquireQInfo(void* pMgmt, void** key); void** qAcquireQInfo(void* pMgmt, uint64_t key);
void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree); void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree);
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -129,9 +129,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "mnode clus
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "mnode accounts already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "mnode accounts already exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "mnode invalid account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "mnode invalid account")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_PARA, 0, 0x0342, "mnode invalid account parameter") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, 0, 0x0342, "mnode invalid acct option")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, 0, 0x0343, "mnode invalid acct option")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_ACCTS, 0, 0x0344, "mnode too many accounts")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, 0, 0x0350, "mnode user already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, 0, 0x0350, "mnode user already exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, 0, 0x0351, "mnode invalid user") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, 0, 0x0351, "mnode invalid user")
@ -145,7 +143,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, 0, 0x0361, "mnode inva
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, 0, 0x0362, "mnode invalid table name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, 0, 0x0362, "mnode invalid table name")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, 0, 0x0363, "mnode invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, 0, 0x0363, "mnode invalid table type")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, 0, 0x0364, "mnode too many tags") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, 0, 0x0364, "mnode too many tags")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TABLES, 0, 0x0365, "mnode too many tables")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, 0, 0x0366, "mnode not enough time series") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, 0, 0x0366, "mnode not enough time series")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, 0, 0x0367, "mnode no super table") // operation only available for super table TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, 0, 0x0367, "mnode no super table") // operation only available for super table
TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, 0, 0x0368, "mnode column name too long") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, 0, 0x0368, "mnode column name too long")
@ -161,13 +158,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, 0, 0x0382, "mnode inva
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "mnode invalid database") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "mnode invalid database")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "mnode monitor db forbidden") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "mnode monitor db forbidden")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "mnode too many databases") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "mnode too many databases")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "mnode db in dropping")
// dnode // dnode
TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "dnode message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "dnode message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "dnode out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "dnode out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "dnode no disk write access") TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "dnode no disk write access")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "dnode invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "dnode invalid message length")
TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_FILE_FORMAT, 0, 0x0404, "dnode invalid file format")
// vnode // vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "vnode action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "vnode action in progress")

View File

@ -473,7 +473,7 @@ typedef struct {
typedef struct { typedef struct {
int32_t code; int32_t code;
uint64_t qhandle; uint64_t qhandle; // query handle
} SQueryTableRsp; } SQueryTableRsp;
typedef struct { typedef struct {
@ -486,7 +486,7 @@ typedef struct SRetrieveTableRsp {
int32_t numOfRows; int32_t numOfRows;
int8_t completed; // all results are returned to client int8_t completed; // all results are returned to client
int16_t precision; int16_t precision;
int64_t offset; // updated offset value for multi-vnode projection query int64_t offset; // updated offset value for multi-vnode projection query
int64_t useconds; int64_t useconds;
char data[]; char data[];
} SRetrieveTableRsp; } SRetrieveTableRsp;

View File

@ -235,9 +235,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle);
* Get current data block information * Get current data block information
* *
* @param pQueryHandle * @param pQueryHandle
* @param pBlockInfo
* @return * @return
*/ */
SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle); void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo);
/** /**
* *

View File

@ -206,9 +206,10 @@ static void shellSourceFile(TAOS *con, char *fptr) {
if (code != 0) { if (code != 0) {
fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(con), fname, lineNo); fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(con), fname, lineNo);
/* free local resouce: allocated memory/metric-meta refcnt */
taos_free_result(pSql);
} }
/* free local resouce: allocated memory/metric-meta refcnt */
taos_free_result(pSql);
memset(cmd, 0, MAX_COMMAND_SIZE); memset(cmd, 0, MAX_COMMAND_SIZE);
cmd_len = 0; cmd_len = 0;

View File

@ -520,9 +520,8 @@ int main(int argc, char *argv[]) {
snprintf(command, BUFFER_SIZE, "create table if not exists %s.meters (ts timestamp%s tags (areaid int, loc binary(10))", db_name, cols); snprintf(command, BUFFER_SIZE, "create table if not exists %s.meters (ts timestamp%s tags (areaid int, loc binary(10))", db_name, cols);
queryDB(taos, command); queryDB(taos, command);
printf("meters created!\n"); printf("meters created!\n");
taos_close(taos);
} }
taos_close(taos);
/* Wait for table to create */ /* Wait for table to create */
multiThreadCreateTable(cols, use_metric, threads, ntables, db_name, tb_prefix, ip_addr, port, user, pass); multiThreadCreateTable(cols, use_metric, threads, ntables, db_name, tb_prefix, ip_addr, port, user, pass);
@ -792,9 +791,6 @@ void * createTable(void *sarg)
snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols); snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols);
queryDB(winfo->taos, command); queryDB(winfo->taos, command);
} }
taos_close(winfo->taos);
} else { } else {
/* Create all the tables; */ /* Create all the tables; */
printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id);
@ -812,7 +808,6 @@ void * createTable(void *sarg)
} }
queryDB(winfo->taos, command); queryDB(winfo->taos, command);
} }
taos_close(winfo->taos);
} }
return NULL; return NULL;
@ -857,7 +852,6 @@ void multiThreadCreateTable(char* cols, bool use_metric, int threads, int ntable
for (int i = 0; i < threads; i++) { for (int i = 0; i < threads; i++) {
info *t_info = infos + i; info *t_info = infos + i;
taos_close(t_info->taos);
sem_destroy(&(t_info->mutex_sem)); sem_destroy(&(t_info->mutex_sem));
sem_destroy(&(t_info->lock_sem)); sem_destroy(&(t_info->lock_sem));
} }
@ -875,6 +869,11 @@ void *readTable(void *sarg) {
int64_t sTime = rinfo->start_time; int64_t sTime = rinfo->start_time;
char *tb_prefix = rinfo->tb_prefix; char *tb_prefix = rinfo->tb_prefix;
FILE *fp = fopen(rinfo->fp, "a"); FILE *fp = fopen(rinfo->fp, "a");
if (NULL == fp) {
printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
return NULL;
}
int num_of_DPT = rinfo->nrecords_per_table; int num_of_DPT = rinfo->nrecords_per_table;
int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1;
int totalData = num_of_DPT * num_of_tables; int totalData = num_of_DPT * num_of_tables;
@ -930,6 +929,11 @@ void *readMetric(void *sarg) {
TAOS *taos = rinfo->taos; TAOS *taos = rinfo->taos;
char command[BUFFER_SIZE] = "\0"; char command[BUFFER_SIZE] = "\0";
FILE *fp = fopen(rinfo->fp, "a"); FILE *fp = fopen(rinfo->fp, "a");
if (NULL == fp) {
printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
return NULL;
}
int num_of_DPT = rinfo->nrecords_per_table; int num_of_DPT = rinfo->nrecords_per_table;
int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1;
int totalData = num_of_DPT * num_of_tables; int totalData = num_of_DPT * num_of_tables;

File diff suppressed because it is too large Load Diff

View File

@ -40,7 +40,7 @@ struct arguments {
static error_t parse_opt(int key, char *arg, struct argp_state *state) { static error_t parse_opt(int key, char *arg, struct argp_state *state) {
struct arguments *arguments = state->input; struct arguments *arguments = state->input;
switch (key) { switch (key) {
case 'w': case 'r':
arguments->dataDir = arg; arguments->dataDir = arg;
break; break;
case 'd': case 'd':
@ -51,6 +51,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
break; break;
case 'f': case 'f':
arguments->fqdn = arg; arguments->fqdn = arg;
break;
case 'g': case 'g':
arguments->dnodeGroups = arg; arguments->dnodeGroups = arg;
break; break;

View File

@ -96,6 +96,7 @@ void walModWalFile(char* walfile) {
if (wfd < 0) { if (wfd < 0) {
printf("wal:%s, failed to open(%s)\n", newWalFile, strerror(errno)); printf("wal:%s, failed to open(%s)\n", newWalFile, strerror(errno));
free(buffer); free(buffer);
close(rfd);
return ; return ;
} }
@ -116,6 +117,11 @@ void walModWalFile(char* walfile) {
break; break;
} }
if (pHead->len >= 1024000 - sizeof(SWalHead)) {
printf("wal:%s, SWalHead.len(%d) overflow, skip the rest of file\n", walfile, pHead->len);
break;
}
ret = read(rfd, pHead->cont, pHead->len); ret = read(rfd, pHead->cont, pHead->len);
if ( ret != pHead->len) { if ( ret != pHead->len) {
printf("wal:%s, failed to read body, skip, len:%d ret:%d\n", walfile, pHead->len, ret); printf("wal:%s, failed to read body, skip, len:%d ret:%d\n", walfile, pHead->len, ret);

View File

@ -99,6 +99,8 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile)
goto PARSE_OVER; goto PARSE_OVER;
} }
content[maxLen] = (char)0;
root = cJSON_Parse(content); root = cJSON_Parse(content);
if (root == NULL) { if (root == NULL) {
printf("failed to json parse %s, invalid json format\n", cfgFile); printf("failed to json parse %s, invalid json format\n", cfgFile);

View File

@ -44,6 +44,7 @@ void mnodeDecMnodeRef(struct SMnodeObj *pMnode);
char * mnodeGetMnodeRoleStr(); char * mnodeGetMnodeRoleStr();
void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet); void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet);
void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet); void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet);
char* mnodeGetMnodeMasterEp();
void mnodeGetMnodeInfos(void *mnodes); void mnodeGetMnodeInfos(void *mnodes);
void mnodeUpdateMnodeIpSet(); void mnodeUpdateMnodeIpSet();

View File

@ -53,6 +53,7 @@ typedef struct {
void * rowData; void * rowData;
int32_t rowSize; int32_t rowSize;
int32_t retCode; // for callback in sdb queue int32_t retCode; // for callback in sdb queue
int32_t processedCount; // for sync fwd callback
int32_t (*cb)(struct SMnodeMsg *pMsg, int32_t code); int32_t (*cb)(struct SMnodeMsg *pMsg, int32_t code);
struct SMnodeMsg *pMsg; struct SMnodeMsg *pMsg;
} SSdbOper; } SSdbOper;

View File

@ -128,6 +128,7 @@ int32_t mnodeInitAccts() {
void mnodeCleanupAccts() { void mnodeCleanupAccts() {
acctCleanUp(); acctCleanUp();
sdbCloseTable(tsAcctSdb); sdbCloseTable(tsAcctSdb);
tsAcctSdb = NULL;
} }
void *mnodeGetAcct(char *name) { void *mnodeGetAcct(char *name) {

View File

@ -459,6 +459,7 @@ void mnodeMoveVgroupToHead(SVgObj *pVgroup) {
void mnodeCleanupDbs() { void mnodeCleanupDbs() {
sdbCloseTable(tsDbSdb); sdbCloseTable(tsDbSdb);
tsDbSdb = NULL;
} }
static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
@ -964,6 +965,11 @@ static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
mError("db:%s, failed to alter, invalid db", pAlter->db); mError("db:%s, failed to alter, invalid db", pAlter->db);
return TSDB_CODE_MND_INVALID_DB; return TSDB_CODE_MND_INVALID_DB;
} }
if (pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pAlter->db, pMsg->pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
return mnodeAlterDb(pMsg->pDb, pAlter, pMsg); return mnodeAlterDb(pMsg->pDb, pAlter, pMsg);
} }

View File

@ -88,13 +88,13 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
} }
static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) { static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
SDnodeObj *pDnode = pOper->pObj; SDnodeObj *pNew = pOper->pObj;
SDnodeObj *pSaved = mnodeGetDnode(pDnode->dnodeId); SDnodeObj *pDnode = mnodeGetDnode(pNew->dnodeId);
if (pSaved != NULL && pDnode != pSaved) { if (pDnode != NULL && pNew != pDnode) {
memcpy(pSaved, pDnode, pOper->rowSize); memcpy(pDnode, pNew, pOper->rowSize);
free(pDnode); free(pNew);
mnodeDecDnodeRef(pSaved);
} }
mnodeDecDnodeRef(pDnode);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -176,6 +176,7 @@ int32_t mnodeInitDnodes() {
void mnodeCleanupDnodes() { void mnodeCleanupDnodes() {
sdbCloseTable(tsDnodeSdb); sdbCloseTable(tsDnodeSdb);
tsDnodeSdb = NULL;
} }
void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) { void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) {
@ -334,7 +335,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
if (pStatus->dnodeId == 0) { if (pStatus->dnodeId == 0) {
mDebug("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp); mDebug("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp);
} else { } else {
//mDebug("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess);
} }
int32_t openVnodes = htons(pStatus->openVnodes); int32_t openVnodes = htons(pStatus->openVnodes);
@ -468,7 +469,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
} }
mnodeDecDnodeRef(pDnode); mnodeDecDnodeRef(pDnode);
if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) { if (strcmp(pDnode->dnodeEp, mnodeGetMnodeMasterEp()) == 0) {
mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep); mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep);
return TSDB_CODE_MND_NO_REMOVE_MASTER; return TSDB_CODE_MND_NO_REMOVE_MASTER;
} }

View File

@ -121,9 +121,9 @@ void mnodeCleanupSystem() {
dnodeFreeMnodeWqueue(); dnodeFreeMnodeWqueue();
dnodeFreeMnodeRqueue(); dnodeFreeMnodeRqueue();
dnodeFreeMnodePqueue(); dnodeFreeMnodePqueue();
mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
mnodeCleanupTimer(); mnodeCleanupTimer();
mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
mInfo("mnode is cleaned up"); mInfo("mnode is cleaned up");
} }

View File

@ -165,6 +165,7 @@ int32_t mnodeInitMnodes() {
void mnodeCleanupMnodes() { void mnodeCleanupMnodes() {
sdbCloseTable(tsMnodeSdb); sdbCloseTable(tsMnodeSdb);
tsMnodeSdb = NULL;
mnodeMnodeDestroyLock(); mnodeMnodeDestroyLock();
} }
@ -267,6 +268,10 @@ void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet) {
mnodeMnodeUnLock(); mnodeMnodeUnLock();
} }
char* mnodeGetMnodeMasterEp() {
return tsMnodeInfos.nodeInfos[tsMnodeInfos.inUse].nodeEp;
}
void mnodeGetMnodeInfos(void *mnodeInfos) { void mnodeGetMnodeInfos(void *mnodeInfos) {
mnodeMnodeRdLock(); mnodeMnodeRdLock();
*(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos; *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;

View File

@ -72,8 +72,6 @@ typedef struct {
void * sync; void * sync;
void * wal; void * wal;
SSyncCfg cfg; SSyncCfg cfg;
sem_t sem;
int32_t code;
int32_t numOfTables; int32_t numOfTables;
SSdbTable *tableList[SDB_TABLE_MAX]; SSdbTable *tableList[SDB_TABLE_MAX];
pthread_mutex_t mutex; pthread_mutex_t mutex;
@ -201,7 +199,7 @@ static void sdbRestoreTables() {
sdbDebug("table:%s, is restored, numOfRows:%" PRId64, pTable->tableName, pTable->numOfRows); sdbDebug("table:%s, is restored, numOfRows:%" PRId64, pTable->tableName, pTable->numOfRows);
} }
sdbInfo("sdb is restored, version:%" PRId64 " totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables); sdbInfo("sdb is restored, ver:%" PRId64 " totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
} }
void sdbUpdateMnodeRoles() { void sdbUpdateMnodeRoles() {
@ -244,27 +242,36 @@ static void sdbNotifyRole(void *ahandle, int8_t role) {
sdbUpdateMnodeRoles(); sdbUpdateMnodeRoles();
} }
FORCE_INLINE
static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
tsSdbObj.code = code; assert(param);
sem_post(&tsSdbObj.sem); SSdbOper * pOper = param;
sdbDebug("forward request confirmed, version:%" PRIu64 ", result:%s", (int64_t)param, tstrerror(code)); SMnodeMsg *pMsg = pOper->pMsg;
} if (code <= 0) pOper->retCode = code;
static int32_t sdbForwardToPeer(SWalHead *pHead) { int32_t processedCount = atomic_add_fetch_32(&pOper->processedCount, 1);
if (tsSdbObj.sync == NULL) return TSDB_CODE_SUCCESS; if (processedCount <= 1) {
if (pMsg != NULL) {
sdbDebug("app:%p:%p, waiting for confirm this operation, count:%d", pMsg->rpcMsg.ahandle, pMsg, processedCount);
}
return;
}
int32_t code = syncForwardToPeer(tsSdbObj.sync, pHead, (void*)pHead->version, TAOS_QTYPE_RPC); if (pMsg != NULL) {
if (code > 0) { sdbDebug("app:%p:%p, is confirmed and will do callback func", pMsg->rpcMsg.ahandle, pMsg);
sdbDebug("forward request is sent, version:%" PRIu64 ", code:%d", pHead->version, code); }
sem_wait(&tsSdbObj.sem);
return tsSdbObj.code; if (pOper->cb != NULL) {
} pOper->retCode = (*pOper->cb)(pMsg, pOper->retCode);
return code; }
dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode);
taosFreeQitem(pOper);
} }
void sdbUpdateSync() { void sdbUpdateSync() {
SSyncCfg syncCfg = {0}; SSyncCfg syncCfg = {0};
int32_t index = 0; int32_t index = 0;
SDMMnodeInfos *mnodes = dnodeGetMnodeInfos(); SDMMnodeInfos *mnodes = dnodeGetMnodeInfos();
for (int32_t i = 0; i < mnodes->nodeNum; ++i) { for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
@ -298,7 +305,7 @@ void sdbUpdateSync() {
} }
syncCfg.replica = index; syncCfg.replica = index;
syncCfg.quorum = (syncCfg.replica == 1) ? 1:2; syncCfg.quorum = (syncCfg.replica == 1) ? 1 : 2;
bool hasThisDnode = false; bool hasThisDnode = false;
for (int32_t i = 0; i < syncCfg.replica; ++i) { for (int32_t i = 0; i < syncCfg.replica; ++i) {
@ -325,10 +332,10 @@ void sdbUpdateSync() {
syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getWalInfo = sdbGetWalInfo;
syncInfo.getFileInfo = sdbGetFileInfo; syncInfo.getFileInfo = sdbGetFileInfo;
syncInfo.writeToCache = sdbWriteToQueue; syncInfo.writeToCache = sdbWriteToQueue;
syncInfo.confirmForward = sdbConfirmForward; syncInfo.confirmForward = sdbConfirmForward;
syncInfo.notifyRole = sdbNotifyRole; syncInfo.notifyRole = sdbNotifyRole;
tsSdbObj.cfg = syncCfg; tsSdbObj.cfg = syncCfg;
if (tsSdbObj.sync) { if (tsSdbObj.sync) {
syncReconfig(tsSdbObj.sync, &syncCfg); syncReconfig(tsSdbObj.sync, &syncCfg);
} else { } else {
@ -339,7 +346,6 @@ void sdbUpdateSync() {
int32_t sdbInit() { int32_t sdbInit() {
pthread_mutex_init(&tsSdbObj.mutex, NULL); pthread_mutex_init(&tsSdbObj.mutex, NULL);
sem_init(&tsSdbObj.sem, 0, 0);
if (sdbInitWriteWorker() != 0) { if (sdbInitWriteWorker() != 0) {
return -1; return -1;
@ -367,7 +373,7 @@ void sdbCleanUp() {
tsSdbObj.status = SDB_STATUS_CLOSING; tsSdbObj.status = SDB_STATUS_CLOSING;
sdbCleanupWriteWorker(); sdbCleanupWriteWorker();
sdbDebug("sdb will be closed, version:%" PRId64, tsSdbObj.version); sdbDebug("sdb will be closed, ver:%" PRId64, tsSdbObj.version);
if (tsSdbObj.sync) { if (tsSdbObj.sync) {
syncStop(tsSdbObj.sync); syncStop(tsSdbObj.sync);
@ -379,7 +385,6 @@ void sdbCleanUp() {
tsSdbObj.wal = NULL; tsSdbObj.wal = NULL;
} }
sem_destroy(&tsSdbObj.sem);
pthread_mutex_destroy(&tsSdbObj.mutex); pthread_mutex_destroy(&tsSdbObj.mutex);
} }
@ -466,8 +471,8 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
atomic_add_fetch_32(&pTable->autoIndex, 1); atomic_add_fetch_32(&pTable->autoIndex, 1);
} }
sdbDebug("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName, sdbDebug("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 " ver:%" PRIu64 ", msg:%p", pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion()); sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, sdbGetVersion(), pOper->pMsg);
(*pTable->insertFp)(pOper); (*pTable->insertFp)(pOper);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -485,8 +490,8 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
taosHashRemove(pTable->iHandle, key, keySize); taosHashRemove(pTable->iHandle, key, keySize);
atomic_sub_fetch_32(&pTable->numOfRows, 1); atomic_sub_fetch_32(&pTable->numOfRows, 1);
sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName, sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 " ver:%" PRIu64 ", msg:%p", pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion(), pOper->pMsg);
int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1;
*updateEnd = 1; *updateEnd = 1;
@ -496,8 +501,8 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
} }
static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
sdbDebug("table:%s, update record:%s in hash, numOfRows:%" PRId64 " version:%" PRIu64, pTable->tableName, sdbDebug("table:%s, update record:%s in hash, numOfRows:%" PRId64 " ver:%" PRIu64 ", msg:%p", pTable->tableName,
sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion(), pOper->pMsg);
(*pTable->updateFp)(pOper); (*pTable->updateFp)(pOper);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -513,24 +518,22 @@ static int sdbWrite(void *param, void *data, int type) {
assert(pTable != NULL); assert(pTable != NULL);
pthread_mutex_lock(&tsSdbObj.mutex); pthread_mutex_lock(&tsSdbObj.mutex);
if (pHead->version == 0) { if (pHead->version == 0) {
// assign version // assign version
tsSdbObj.version++; tsSdbObj.version++;
pHead->version = tsSdbObj.version; pHead->version = tsSdbObj.version;
} else { } else {
// for data from WAL or forward, version may be smaller // for data from WAL or forward, version may be smaller
if (pHead->version <= tsSdbObj.version) { if (pHead->version <= tsSdbObj.version) {
pthread_mutex_unlock(&tsSdbObj.mutex); pthread_mutex_unlock(&tsSdbObj.mutex);
if (type == TAOS_QTYPE_FWD && tsSdbObj.sync != NULL) { sdbDebug("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64,
sdbDebug("forward request is received, version:%" PRIu64 " confirm it", pHead->version); pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version);
syncConfirmForward(tsSdbObj.sync, pHead->version, TSDB_CODE_SUCCESS);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (pHead->version != tsSdbObj.version + 1) { } else if (pHead->version != tsSdbObj.version + 1) {
pthread_mutex_unlock(&tsSdbObj.mutex); pthread_mutex_unlock(&tsSdbObj.mutex);
sdbError("table:%s, failed to restore %s record:%s from wal, version:%" PRId64 " too large, sdb version:%" PRId64, sdbError("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64,
pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version);
tsSdbObj.version);
return TSDB_CODE_MND_APP_ERROR; return TSDB_CODE_MND_APP_ERROR;
} else { } else {
tsSdbObj.version = pHead->version; tsSdbObj.version = pHead->version;
@ -542,28 +545,36 @@ static int sdbWrite(void *param, void *data, int type) {
pthread_mutex_unlock(&tsSdbObj.mutex); pthread_mutex_unlock(&tsSdbObj.mutex);
return code; return code;
} }
code = sdbForwardToPeer(pHead);
pthread_mutex_unlock(&tsSdbObj.mutex); pthread_mutex_unlock(&tsSdbObj.mutex);
// from app, oper is created // from app, oper is created
if (pOper != NULL) { if (pOper != NULL) {
sdbTrace("record from app is disposed, table:%s action:%s record:%s version:%" PRIu64 " result:%s", // forward to peers
pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->processedCount = 0;
tstrerror(code)); int32_t syncCode = syncForwardToPeer(tsSdbObj.sync, pHead, pOper, TAOS_QTYPE_RPC);
return code; if (syncCode <= 0) pOper->processedCount = 1;
if (syncCode < 0) {
sdbError("table:%s, failed to forward request, result:%s action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName,
tstrerror(syncCode), sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg);
} else if (syncCode > 0) {
sdbDebug("table:%s, forward request is sent, action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName,
sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg);
} else {
sdbTrace("table:%s, no need to send fwd request, action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName,
sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg);
}
return syncCode;
} }
sdbDebug("table:%s, record from wal/fwd is disposed, action:%s record:%s ver:%" PRId64, pTable->tableName,
sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version);
// even it is WAL/FWD, it shall be called to update version in sync
syncForwardToPeer(tsSdbObj.sync, pHead, pOper, TAOS_QTYPE_RPC);
// from wal or forward msg, oper not created, should add into hash // from wal or forward msg, oper not created, should add into hash
if (tsSdbObj.sync != NULL) {
sdbTrace("record from wal forward is disposed, table:%s action:%s record:%s version:%" PRIu64 " confirm it",
pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version);
syncConfirmForward(tsSdbObj.sync, pHead->version, code);
} else {
sdbTrace("record from wal restore is disposed, table:%s action:%s record:%s version:%" PRIu64, pTable->tableName,
sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version);
}
if (action == SDB_ACTION_INSERT) { if (action == SDB_ACTION_INSERT) {
SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable}; SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
code = (*pTable->decodeFp)(&oper); code = (*pTable->decodeFp)(&oper);
@ -627,7 +638,7 @@ int32_t sdbInsertRow(SSdbOper *pOper) {
memcpy(pNewOper, pOper, sizeof(SSdbOper)); memcpy(pNewOper, pOper, sizeof(SSdbOper));
if (pNewOper->pMsg != NULL) { if (pNewOper->pMsg != NULL) {
sdbDebug("app:%p:%p, table:%s record:%p:%s, insert action is add to sdb queue, ", pNewOper->pMsg->rpcMsg.ahandle, sdbDebug("app:%p:%p, table:%s record:%p:%s, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle,
pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj));
} }
@ -677,7 +688,7 @@ int32_t sdbDeleteRow(SSdbOper *pOper) {
memcpy(pNewOper, pOper, sizeof(SSdbOper)); memcpy(pNewOper, pOper, sizeof(SSdbOper));
if (pNewOper->pMsg != NULL) { if (pNewOper->pMsg != NULL) {
sdbDebug("app:%p:%p, table:%s record:%p:%s, delete action is add to sdb queue, ", pNewOper->pMsg->rpcMsg.ahandle, sdbDebug("app:%p:%p, table:%s record:%p:%s, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle,
pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj));
} }
@ -727,7 +738,7 @@ int32_t sdbUpdateRow(SSdbOper *pOper) {
memcpy(pNewOper, pOper, sizeof(SSdbOper)); memcpy(pNewOper, pOper, sizeof(SSdbOper));
if (pNewOper->pMsg != NULL) { if (pNewOper->pMsg != NULL) {
sdbDebug("app:%p:%p, table:%s record:%p:%s, update action is add to sdb queue, ", pNewOper->pMsg->rpcMsg.ahandle, sdbDebug("app:%p:%p, table:%s record:%p:%s, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle,
pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj)); pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj));
} }
@ -943,20 +954,20 @@ static void *sdbWorkerFp(void *param) {
taosGetQitem(tsSdbWriteQall, &type, &item); taosGetQitem(tsSdbWriteQall, &type, &item);
if (type == TAOS_QTYPE_RPC) { if (type == TAOS_QTYPE_RPC) {
pOper = (SSdbOper *)item; pOper = (SSdbOper *)item;
pOper->processedCount = 1;
pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK; pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
if (pOper->pMsg != NULL) {
sdbDebug("app:%p:%p, table:%s record:%p:%s ver:%" PRIu64 ", will be processed in sdb queue",
pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, ((SSdbTable *)pOper->table)->tableName, pOper->pObj,
sdbGetKeyStr(pOper->table, pHead->cont), pHead->version);
}
} else { } else {
pHead = (SWalHead *)item; pHead = (SWalHead *)item;
pOper = NULL; pOper = NULL;
} }
if (pOper != NULL && pOper->pMsg != NULL) {
sdbDebug("app:%p:%p, table:%s record:%p:%s version:%" PRIu64 ", will be processed in sdb queue",
pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, ((SSdbTable *)pOper->table)->tableName, pOper->pObj,
sdbGetKeyStr(pOper->table, pHead->cont), pHead->version);
}
int32_t code = sdbWrite(pOper, pHead, type); int32_t code = sdbWrite(pOper, pHead, type);
if (pOper) pOper->retCode = code; if (pOper && code <= 0) pOper->retCode = code;
} }
walFsync(tsSdbObj.wal); walFsync(tsSdbObj.wal);
@ -965,25 +976,17 @@ static void *sdbWorkerFp(void *param) {
taosResetQitems(tsSdbWriteQall); taosResetQitems(tsSdbWriteQall);
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(tsSdbWriteQall, &type, &item); taosGetQitem(tsSdbWriteQall, &type, &item);
if (type == TAOS_QTYPE_RPC) { if (type == TAOS_QTYPE_RPC) {
pOper = (SSdbOper *)item; pOper = (SSdbOper *)item;
if (pOper != NULL && pOper->cb != NULL) { sdbDecRef(pOper->table, pOper->pObj);
sdbTrace("app:%p:%p, will do callback func, index:%d", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, i); sdbConfirmForward(NULL, pOper, pOper->retCode);
pOper->retCode = (*pOper->cb)(pOper->pMsg, pOper->retCode); } else if (type == TAOS_QTYPE_FWD) {
} syncConfirmForward(tsSdbObj.sync, pHead->version, TSDB_CODE_SUCCESS);
taosFreeQitem(item);
if (pOper != NULL && pOper->pMsg != NULL) { } else {
sdbTrace("app:%p:%p, msg is processed, result:%s", pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, taosFreeQitem(item);
tstrerror(pOper->retCode));
}
if (pOper != NULL) {
sdbDecRef(pOper->table, pOper->pObj);
}
dnodeSendRpcMnodeWriteRsp(pOper->pMsg, pOper->retCode);
} }
taosFreeQitem(item);
} }
} }

View File

@ -307,6 +307,11 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
code = TSDB_CODE_MND_INVALID_DB; code = TSDB_CODE_MND_INVALID_DB;
goto connect_over; goto connect_over;
} }
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
mnodeDecDbRef(pDb); mnodeDecDbRef(pDb);
} }
@ -352,6 +357,11 @@ static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) {
if (pMsg->pDb == NULL) { if (pMsg->pDb == NULL) {
code = TSDB_CODE_MND_INVALID_DB; code = TSDB_CODE_MND_INVALID_DB;
} }
if (pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
return code; return code;
} }
@ -403,4 +413,4 @@ void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capa
memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows);
} }
} }
} }

View File

@ -116,6 +116,11 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
mError("ctable:%s, vgId:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); mError("ctable:%s, vgId:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
return TSDB_CODE_MND_INVALID_DB; return TSDB_CODE_MND_INVALID_DB;
} }
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
mnodeDecDbRef(pDb); mnodeDecDbRef(pDb);
SAcctObj *pAcct = mnodeGetAcct(pDb->acct); SAcctObj *pAcct = mnodeGetAcct(pDb->acct);
@ -284,8 +289,8 @@ static int32_t mnodeChildTableActionRestored() {
if (pTable == NULL) break; if (pTable == NULL) break;
SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId); SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId);
if (pDb == NULL) { if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) {
mError("ctable:%s, failed to get db, discard it", pTable->info.tableId); mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId);
SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc); sdbDeleteRow(&desc);
mnodeDecTableRef(pTable); mnodeDecTableRef(pTable);
@ -376,6 +381,7 @@ static int32_t mnodeInitChildTables() {
static void mnodeCleanupChildTables() { static void mnodeCleanupChildTables() {
sdbCloseTable(tsChildTableSdb); sdbCloseTable(tsChildTableSdb);
tsChildTableSdb = NULL;
} }
static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
@ -422,7 +428,7 @@ static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) {
static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) { static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) {
SSuperTableObj *pStable = pOper->pObj; SSuperTableObj *pStable = pOper->pObj;
SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId);
if (pDb != NULL) { if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) {
mnodeAddSuperTableIntoDb(pDb); mnodeAddSuperTableIntoDb(pDb);
} }
mnodeDecDbRef(pDb); mnodeDecDbRef(pDb);
@ -554,6 +560,7 @@ static int32_t mnodeInitSuperTables() {
static void mnodeCleanupSuperTables() { static void mnodeCleanupSuperTables() {
sdbCloseTable(tsSuperTableSdb); sdbCloseTable(tsSuperTableSdb);
tsSuperTableSdb = NULL;
} }
int32_t mnodeInitTables() { int32_t mnodeInitTables() {
@ -683,10 +690,15 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) {
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { if (pMsg->pDb == NULL) {
mError("app:%p:%p, table:%s, failed to create, db not selected", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); mError("app:%p:%p, table:%s, failed to create, db not selected", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED; return TSDB_CODE_MND_DB_NOT_SELECTED;
} }
if (pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId);
if (pMsg->pTable != NULL && pMsg->retry == 0) { if (pMsg->pTable != NULL && pMsg->retry == 0) {
@ -717,10 +729,15 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) {
static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont; SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { if (pMsg->pDb == NULL) {
mError("app:%p:%p, table:%s, failed to drop table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); mError("app:%p:%p, table:%s, failed to drop table, db not selected or db in dropping", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED; return TSDB_CODE_MND_DB_NOT_SELECTED;
} }
if (pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
mError("app:%p:%p, table:%s, failed to drop table, in monitor database", pMsg->rpcMsg.ahandle, pMsg, mError("app:%p:%p, table:%s, failed to drop table, in monitor database", pMsg->rpcMsg.ahandle, pMsg,
@ -755,11 +772,16 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { if (pMsg->pDb == NULL) {
mError("app:%p:%p, table:%s, failed to get table meta, db not selected", pMsg->rpcMsg.ahandle, pMsg, mError("app:%p:%p, table:%s, failed to get table meta, db not selected", pMsg->rpcMsg.ahandle, pMsg,
pInfo->tableId); pInfo->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED; return TSDB_CODE_MND_DB_NOT_SELECTED;
} }
if (pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId); if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId);
if (pMsg->pTable == NULL) { if (pMsg->pTable == NULL) {
@ -783,9 +805,15 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
if (pTable != NULL) { assert(pTable);
mLInfo("app:%p:%p, stable:%s, is created in sdb, result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
tstrerror(code)); if (code == TSDB_CODE_SUCCESS) {
mLInfo("stable:%s, is created in sdb", pTable->info.tableId);
} else {
mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
tstrerror(code));
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsSuperTableSdb};
sdbDeleteRow(&desc);
} }
return code; return code;
@ -1201,6 +1229,11 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED;
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
int32_t cols = 0; int32_t cols = 0;
SSchema *pSchema = pMeta->schema; SSchema *pSchema = pMeta->schema;
@ -1260,6 +1293,11 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return 0; if (pDb == NULL) return 0;
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return 0;
}
tstrncpy(prefix, pDb->name, 64); tstrncpy(prefix, pDb->name, 64);
strcat(prefix, TS_PATH_DELIMITER); strcat(prefix, TS_PATH_DELIMITER);
@ -1561,10 +1599,16 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable;
assert(pTable); assert(pTable);
mDebug("app:%p:%p, table:%s, create table in id:%d, uid:%" PRIu64 ", result:%s", pMsg->rpcMsg.ahandle, pMsg, if (code == TSDB_CODE_SUCCESS) {
pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); mDebug("app:%p:%p, table:%s, create table in sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
pTable->sid, pTable->uid);
if (code != TSDB_CODE_SUCCESS) return code; } else {
mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg,
pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code));
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb};
sdbDeleteRow(&desc);
return code;
}
SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, pTable); SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, pTable);
@ -2285,7 +2329,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
if (pTable == NULL) continue; if (pTable == NULL) continue;
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId);
if (pMsg->pDb == NULL) { if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mnodeDecTableRef(pTable); mnodeDecTableRef(pTable);
continue; continue;
} }
@ -2323,6 +2367,11 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED;
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
int32_t cols = 0; int32_t cols = 0;
SSchema *pSchema = pMeta->schema; SSchema *pSchema = pMeta->schema;
@ -2371,6 +2420,11 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return 0; if (pDb == NULL) return 0;
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return 0;
}
int32_t numOfRows = 0; int32_t numOfRows = 0;
SChildTableObj *pTable = NULL; SChildTableObj *pTable = NULL;
@ -2462,10 +2516,15 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
pAlter->tableId, pMsg->rpcMsg.handle); pAlter->tableId, pMsg->rpcMsg.handle);
if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId);
if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { if (pMsg->pDb == NULL) {
mError("app:%p:%p, table:%s, failed to alter table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); mError("app:%p:%p, table:%s, failed to alter table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
return TSDB_CODE_MND_DB_NOT_SELECTED; return TSDB_CODE_MND_DB_NOT_SELECTED;
} }
if (pMsg->pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
mError("app:%p:%p, table:%s, failed to alter table, its log db", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId); mError("app:%p:%p, table:%s, failed to alter table, its log db", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
@ -2525,6 +2584,11 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED;
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
int32_t cols = 0; int32_t cols = 0;
SSchema *pSchema = pMeta->schema; SSchema *pSchema = pMeta->schema;
@ -2572,7 +2636,11 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo
static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return 0; if (pDb == NULL) return 0;
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return 0;
}
int32_t numOfRows = 0; int32_t numOfRows = 0;
SChildTableObj *pTable = NULL; SChildTableObj *pTable = NULL;

View File

@ -154,6 +154,7 @@ int32_t mnodeInitUsers() {
void mnodeCleanupUsers() { void mnodeCleanupUsers() {
sdbCloseTable(tsUserSdb); sdbCloseTable(tsUserSdb);
tsUserSdb = NULL;
} }
SUserObj *mnodeGetUser(char *name) { SUserObj *mnodeGetUser(char *name) {

View File

@ -75,6 +75,11 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) {
if (pDb == NULL) { if (pDb == NULL) {
return TSDB_CODE_MND_INVALID_DB; return TSDB_CODE_MND_INVALID_DB;
} }
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
pVgroup->pDb = pDb; pVgroup->pDb = pDb;
pVgroup->prev = NULL; pVgroup->prev = NULL;
@ -171,6 +176,12 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) {
mnodeVgroupUpdateIdPool(pVgroup); mnodeVgroupUpdateIdPool(pVgroup);
// reset vgid status on vgroup changed
mDebug("vgId:%d, reset sync status to unsynced", pVgroup->vgId);
for (int32_t v = 0; v < pVgroup->numOfVnodes; ++v) {
pVgroup->vnodeGid[v].role = TAOS_SYNC_ROLE_UNSYNCED;
}
mnodeDecVgroupRef(pVgroup); mnodeDecVgroupRef(pVgroup);
mDebug("vgId:%d, is updated, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); mDebug("vgId:%d, is updated, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes);
@ -302,6 +313,7 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
if (pVgid->pDnode == pDnode) { if (pVgid->pDnode == pDnode) {
mTrace("dnode:%d, receive status from dnode, vgId:%d status is %d", pVgroup->vgId, pDnode->dnodeId, pVgid->role);
pVgid->role = pVload->role; pVgid->role = pVload->role;
if (pVload->role == TAOS_SYNC_ROLE_MASTER) { if (pVload->role == TAOS_SYNC_ROLE_MASTER) {
pVgroup->inUse = i; pVgroup->inUse = i;
@ -341,17 +353,23 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
} }
static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
SVgObj *pVgroup = pMsg->pVgroup;
SDbObj *pDb = pMsg->pDb;
assert(pVgroup);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pMsg->pVgroup = NULL; mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
tstrerror(code));
SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
sdbDeleteRow(&desc);
return code; return code;
} }
SVgObj *pVgroup = pMsg->pVgroup; mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
SDbObj *pDb = pMsg->pDb; pDb->name, pVgroup->numOfVnodes);
mInfo("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
mInfo("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId); mInfo("app:%p:%p, vgId:%d, index:%d, dnode:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, i,
pVgroup->vnodeGid[i].dnodeId);
} }
mnodeIncVgroupRef(pVgroup); mnodeIncVgroupRef(pVgroup);
@ -414,6 +432,7 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) {
void mnodeCleanupVgroups() { void mnodeCleanupVgroups() {
sdbCloseTable(tsVgroupSdb); sdbCloseTable(tsVgroupSdb);
tsVgroupSdb = NULL;
} }
int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
@ -421,6 +440,11 @@ int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
if (pDb == NULL) { if (pDb == NULL) {
return TSDB_CODE_MND_DB_NOT_SELECTED; return TSDB_CODE_MND_DB_NOT_SELECTED;
} }
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return TSDB_CODE_MND_DB_IN_DROPPING;
}
int32_t cols = 0; int32_t cols = 0;
SSchema *pSchema = pMeta->schema; SSchema *pSchema = pMeta->schema;
@ -509,6 +533,11 @@ int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pC
SDbObj *pDb = mnodeGetDb(pShow->db); SDbObj *pDb = mnodeGetDb(pShow->db);
if (pDb == NULL) return 0; if (pDb == NULL) return 0;
if (pDb->status != TSDB_DB_STATUS_READY) {
mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
return 0;
}
pVgroup = pDb->pHead; pVgroup = pDb->pHead;
while (pVgroup != NULL) { while (pVgroup != NULL) {
@ -683,9 +712,9 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
if (rpcMsg->ahandle == NULL) return; if (rpcMsg->ahandle == NULL) return;
SMnodeMsg *mnodeMsg = rpcMsg->ahandle; SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
mnodeMsg->received++; atomic_add_fetch_8(&mnodeMsg->received, 1);
if (rpcMsg->code == TSDB_CODE_SUCCESS) { if (rpcMsg->code == TSDB_CODE_SUCCESS) {
mnodeMsg->successed++; atomic_add_fetch_8(&mnodeMsg->successed, 1);
} else { } else {
mnodeMsg->code = rpcMsg->code; mnodeMsg->code = rpcMsg->code;
} }

View File

@ -16,16 +16,16 @@
#ifndef TDENGINE_TAST_H #ifndef TDENGINE_TAST_H
#define TDENGINE_TAST_H #define TDENGINE_TAST_H
#include <tbuffer.h>
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include <tskiplist.h>
#include "os.h" #include "os.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "taosdef.h" #include "taosdef.h"
#include "tskiplist.h"
#include "tbuffer.h"
#include "tvariant.h" #include "tvariant.h"
struct tExprNode; struct tExprNode;
@ -75,10 +75,6 @@ typedef struct tExprNode {
}; };
} tExprNode; } tExprNode;
void tSQLBinaryExprFromString(tExprNode **pExpr, SSchema *pSchema, int32_t numOfCols, char *src, int32_t len);
void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len);
void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param);
@ -86,12 +82,9 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*cb)(void *, const char*, int32_t)); char *(*cb)(void *, const char*, int32_t));
// todo refactor: remove it
void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res);
uint8_t getBinaryExprOptr(SSQLToken *pToken); uint8_t getBinaryExprOptr(SSQLToken *pToken);
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
tExprNode* exprTreeFromBinary(const void* data, size_t size); tExprNode* exprTreeFromBinary(const void* data, size_t size);

View File

@ -18,18 +18,18 @@
#include "qfill.h" #include "qfill.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "exception.h"
#include "hash.h" #include "hash.h"
#include "qExecutor.h" #include "qExecutor.h"
#include "qUtil.h" #include "qUtil.h"
#include "qast.h"
#include "qresultBuf.h" #include "qresultBuf.h"
#include "query.h" #include "query.h"
#include "queryLog.h" #include "queryLog.h"
#include "qast.h"
#include "tfile.h"
#include "tlosertree.h" #include "tlosertree.h"
#include "exception.h"
#include "tscompression.h" #include "tscompression.h"
#include "ttime.h" #include "ttime.h"
#include "tfile.h"
/** /**
* check if the primary column is load by default, otherwise, the program will * check if the primary column is load by default, otherwise, the program will
@ -49,6 +49,8 @@
#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step)) #define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step))
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
#define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0}
/* get the qinfo struct address from the query struct address */ /* get the qinfo struct address from the query struct address */
#define GET_COLUMN_BYTES(query, colidx) \ #define GET_COLUMN_BYTES(query, colidx) \
((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].bytes) ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].bytes)
@ -120,7 +122,6 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) {
static void setQueryStatus(SQuery *pQuery, int8_t status); static void setQueryStatus(SQuery *pQuery, int8_t status);
#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->intervalTime > 0) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->intervalTime > 0)
static bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; }
// todo move to utility // todo move to utility
static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group);
@ -397,34 +398,38 @@ static bool hasNullValue(SQuery *pQuery, int32_t col, int32_t numOfCols, SDataSt
} }
static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData,
int16_t bytes) { int16_t bytes, bool masterscan) {
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes); int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes);
if (p1 != NULL) { if (p1 != NULL) {
pWindowResInfo->curIndex = *p1; pWindowResInfo->curIndex = *p1;
} else { // more than the capacity, reallocate the resources } else {
if (pWindowResInfo->size >= pWindowResInfo->capacity) { if (masterscan) { // more than the capacity, reallocate the resources
int64_t newCap = pWindowResInfo->capacity * 2; if (pWindowResInfo->size >= pWindowResInfo->capacity) {
int64_t newCap = pWindowResInfo->capacity * 2;
char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult));
if (t != NULL) { if (t != NULL) {
pWindowResInfo->pResult = (SWindowResult *)t; pWindowResInfo->pResult = (SWindowResult *)t;
memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * pWindowResInfo->capacity); memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * pWindowResInfo->capacity);
} else { } else {
// todo // todo
}
for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) {
SPosInfo pos = {-1, -1};
createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos);
}
pWindowResInfo->capacity = newCap;
} }
for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { // add a new result set for a new group
SPosInfo pos = {-1, -1}; pWindowResInfo->curIndex = pWindowResInfo->size++;
createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &pos); taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t));
} } else {
pWindowResInfo->capacity = newCap; return NULL;
} }
// add a new result set for a new group
pWindowResInfo->curIndex = pWindowResInfo->size++;
taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t));
} }
return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex);
@ -511,15 +516,19 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
} }
static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid,
STimeWindow *win) { STimeWindow *win, bool masterscan, bool* newWind) {
assert(win->skey <= win->ekey); assert(win->skey <= win->ekey);
SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE); SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey,
TSDB_KEYSIZE, masterscan);
if (pWindowRes == NULL) { if (pWindowRes == NULL) {
return -1; *newWind = false;
return masterscan? -1:0;
} }
*newWind = true;
// not assign result buffer yet, add new result buffer // not assign result buffer yet, add new result buffer
if (pWindowRes->pos.pageId == -1) { if (pWindowRes->pos.pageId == -1) {
int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage);
@ -563,7 +572,7 @@ static int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t sea
*/ */
static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) { static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) {
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
if (pRuntimeEnv->scanFlag != MASTER_SCAN || (!isIntervalQuery(pQuery))) { if (pRuntimeEnv->scanFlag != MASTER_SCAN || (!QUERY_IS_INTERVAL_QUERY(pQuery))) {
return pWindowResInfo->size; return pWindowResInfo->size;
} }
@ -686,24 +695,26 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat
SQuery * pQuery = pRuntimeEnv->pQuery; SQuery * pQuery = pRuntimeEnv->pQuery;
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) {
int32_t functionId = pQuery->pSelectExpr[k].base.functionId; for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
pCtx[k].nStartQueryTimestamp = pWin->skey; pCtx[k].nStartQueryTimestamp = pWin->skey;
pCtx[k].size = forwardStep; pCtx[k].size = forwardStep;
pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
pCtx[k].ptsList = &tsBuf[offset]; pCtx[k].ptsList = &tsBuf[offset];
} }
// not a whole block involved in query processing, statistics data can not be used // not a whole block involved in query processing, statistics data can not be used
if (forwardStep != numOfTotal) { if (forwardStep != numOfTotal) {
pCtx[k].preAggVals.isSet = false; pCtx[k].preAggVals.isSet = false;
} }
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
aAggs[functionId].xFunction(&pCtx[k]); aAggs[functionId].xFunction(&pCtx[k]);
}
} }
} }
} }
@ -713,12 +724,14 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus
SQuery * pQuery = pRuntimeEnv->pQuery; SQuery * pQuery = pRuntimeEnv->pQuery;
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) {
pCtx[k].nStartQueryTimestamp = pWin->skey; for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
pCtx[k].nStartQueryTimestamp = pWin->skey;
int32_t functionId = pQuery->pSelectExpr[k].base.functionId; int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
aAggs[functionId].xFunctionF(&pCtx[k], offset); aAggs[functionId].xFunctionF(&pCtx[k], offset);
}
} }
} }
} }
@ -880,7 +893,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo,
__block_search_fn_t searchFn, SArray *pDataBlock) { __block_search_fn_t searchFn, SArray *pDataBlock) {
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
bool masterScan = IS_MASTER_SCAN(pRuntimeEnv);
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
TSKEY *tsCols = NULL; TSKEY *tsCols = NULL;
if (pDataBlock != NULL) { if (pDataBlock != NULL) {
@ -903,18 +917,21 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); int32_t offset = GET_COL_DATA_POS(pQuery, 0, step);
TSKEY ts = tsCols[offset]; TSKEY ts = tsCols[offset];
bool hasTimeWindow = false;
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win) != TSDB_CODE_SUCCESS) { if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
tfree(sasArray); tfree(sasArray);
return; return;
} }
TSKEY ekey = reviseWindowEkey(pQuery, &win); if (hasTimeWindow) {
int32_t forwardStep = TSKEY ekey = reviseWindowEkey(pQuery, &win);
getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); int32_t forwardStep =
getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true);
SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, tsCols, pDataBlockInfo->rows); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, tsCols, pDataBlockInfo->rows);
}
int32_t index = pWindowResInfo->curIndex; int32_t index = pWindowResInfo->curIndex;
STimeWindow nextWin = win; STimeWindow nextWin = win;
@ -926,14 +943,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
} }
// null data, failed to allocate more memory buffer // null data, failed to allocate more memory buffer
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin) != TSDB_CODE_SUCCESS) { hasTimeWindow = false;
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
break; break;
} }
ekey = reviseWindowEkey(pQuery, &nextWin); if (!hasTimeWindow) {
forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); continue;
}
pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); TSKEY ekey = reviseWindowEkey(pQuery, &nextWin);
int32_t forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true);
SWindowStatus* pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows);
} }
@ -983,7 +1005,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
} }
// assert(pRuntimeEnv->windowResInfo.hashList->size <= 2); // assert(pRuntimeEnv->windowResInfo.hashList->size <= 2);
SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes); SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true);
if (pWindowRes == NULL) { if (pWindowRes == NULL) {
return -1; return -1;
} }
@ -1113,6 +1135,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo,
SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SWindowResInfo *pWindowResInfo, SArray *pDataBlock) {
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
bool masterScan = IS_MASTER_SCAN(pRuntimeEnv);
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
STableQueryInfo* item = pQuery->current; STableQueryInfo* item = pQuery->current;
@ -1179,16 +1202,21 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
} }
// interval window query // interval window query
if (isIntervalQuery(pQuery)) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
// decide the time window according to the primary timestamp // decide the time window according to the primary timestamp
int64_t ts = tsCols[offset]; int64_t ts = tsCols[offset];
STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win); bool hasTimeWindow = false;
int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow);
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
continue; continue;
} }
if (!hasTimeWindow) {
continue;
}
SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset);
@ -1208,12 +1236,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
} }
// null data, failed to allocate more memory buffer // null data, failed to allocate more memory buffer
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin) != TSDB_CODE_SUCCESS) { bool hasTimeWindow = false;
if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) {
break; break;
} }
pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); if (hasTimeWindow) {
doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset);
}
} }
pWindowResInfo->curIndex = index; pWindowResInfo->curIndex = index;
@ -1283,7 +1314,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
// interval query with limit applied // interval query with limit applied
int32_t numOfRes = 0; int32_t numOfRes = 0;
if (isIntervalQuery(pQuery)) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo);
} else { } else {
numOfRes = getNumOfResult(pRuntimeEnv); numOfRes = getNumOfResult(pRuntimeEnv);
@ -1679,28 +1710,21 @@ static bool onlyQueryTags(SQuery* pQuery) {
///////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////
void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *realWin, STimeWindow *win) { void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) {
assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime);
win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision);
/*
* if the realSkey > INT64_MAX - pQuery->intervalTime, the query duration between
* realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges.
*/
if (keyFirst > (INT64_MAX - pQuery->intervalTime)) { if (keyFirst > (INT64_MAX - pQuery->intervalTime)) {
/*
* if the realSkey > INT64_MAX - pQuery->intervalTime, the query duration between
* realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges.
*/
assert(keyLast - keyFirst < pQuery->intervalTime); assert(keyLast - keyFirst < pQuery->intervalTime);
realWin->skey = keyFirst;
realWin->ekey = keyLast;
win->ekey = INT64_MAX; win->ekey = INT64_MAX;
return; return;
} else {
win->ekey = win->skey + pQuery->intervalTime - 1;
} }
win->ekey = win->skey + pQuery->intervalTime - 1;
realWin->skey = (win->skey < keyFirst)? keyFirst : win->skey;
realWin->ekey = (win->ekey < keyLast) ? win->ekey : keyLast;
} }
static void setScanLimitationByResultBuffer(SQuery *pQuery) { static void setScanLimitationByResultBuffer(SQuery *pQuery) {
@ -1869,7 +1893,7 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) {
if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
num = 128; num = 128;
} else if (isIntervalQuery(pQuery)) { // time window query, allocate one page for each table } else if (QUERY_IS_INTERVAL_QUERY(pQuery)) { // time window query, allocate one page for each table
size_t s = pQInfo->tableqinfoGroupInfo.numOfTables; size_t s = pQInfo->tableqinfoGroupInfo.numOfTables;
num = MAX(s, INITIAL_RESULT_ROWS_VALUE); num = MAX(s, INITIAL_RESULT_ROWS_VALUE);
} else { // for super table query, one page for each subset } else { // for super table query, one page for each subset
@ -2019,7 +2043,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle,
r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId);
} }
if (pRuntimeEnv->pTSBuf > 0 || isIntervalQuery(pQuery)) { if (pRuntimeEnv->pTSBuf > 0 || QUERY_IS_INTERVAL_QUERY(pQuery)) {
r |= BLK_DATA_ALL_NEEDED; r |= BLK_DATA_ALL_NEEDED;
} }
} }
@ -2173,6 +2197,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
if (tmp == NULL) { // todo handle the oom if (tmp == NULL) { // todo handle the oom
assert(0); assert(0);
} else { } else {
memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (newSize - pRec->rows) * bytes);
pQuery->sdata[i] = (tFilePage *)tmp; pQuery->sdata[i] = (tFilePage *)tmp;
} }
@ -2203,34 +2228,32 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
pQuery->order.order); pQuery->order.order);
TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle;
SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
while (tsdbNextDataBlock(pQueryHandle)) { while (tsdbNextDataBlock(pQueryHandle)) {
summary->totalBlocks += 1; summary->totalBlocks += 1;
if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
return 0; return 0;
} }
SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo);
// todo extract methods // todo extract methods
if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) {
STimeWindow realWin = TSWINDOW_INITIALIZER, w = TSWINDOW_INITIALIZER; STimeWindow w = TSWINDOW_INITIALIZER;
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (QUERY_IS_ASC_QUERY(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) {
getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &realWin, &w); getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w);
pWindowResInfo->startTime = w.skey; pWindowResInfo->startTime = w.skey;
pWindowResInfo->prevSKey = w.skey; pWindowResInfo->prevSKey = w.skey;
} else { } else {
// the start position of the first time window in the endpoint that spreads beyond the queried last timestamp // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &realWin, &w); getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w);
pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->startTime = pQuery->window.skey;
pWindowResInfo->prevSKey = w.skey; pWindowResInfo->prevSKey = w.skey;
} }
if (pRuntimeEnv->pFillInfo != NULL) {
pRuntimeEnv->pFillInfo->start = w.skey;
}
} }
// in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block
@ -2260,10 +2283,10 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) {
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP; // int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP;
closeAllTimeWindow(&pRuntimeEnv->windowResInfo); closeAllTimeWindow(&pRuntimeEnv->windowResInfo);
removeRedundantWindow(&pRuntimeEnv->windowResInfo, pTableQueryInfo->lastKey - step, step); // removeRedundantWindow(&pRuntimeEnv->windowResInfo, pTableQueryInfo->lastKey - step, step);
pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window
} else { } else {
assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL));
@ -2880,6 +2903,9 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *
SWITCH_ORDER(pTableQueryInfo->cur.order); SWITCH_ORDER(pTableQueryInfo->cur.order);
pTableQueryInfo->cur.vgroupIndex = -1; pTableQueryInfo->cur.vgroupIndex = -1;
// set the index at the end of time window
pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1;
} }
static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) {
@ -2914,7 +2940,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) {
// group by normal columns and interval query on normal table // group by normal columns and interval query on normal table
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (pRuntimeEnv->groupbyNormalCol || isIntervalQuery(pQuery)) { if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order); disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order);
} else { // for simple result of table query, } else { // for simple result of table query,
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor
@ -3089,7 +3115,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
bool toContinue = false; bool toContinue = false;
if (pRuntimeEnv->groupbyNormalCol || isIntervalQuery(pQuery)) { if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
// for each group result, call the finalize function for each column // for each group result, call the finalize function for each column
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
@ -3281,7 +3307,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
if (pRuntimeEnv->groupbyNormalCol || isIntervalQuery(pQuery)) { if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
// for each group result, call the finalize function for each column // for each group result, call the finalize function for each column
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
if (pRuntimeEnv->groupbyNormalCol) { if (pRuntimeEnv->groupbyNormalCol) {
@ -3327,9 +3353,9 @@ static bool hasMainOutput(SQuery *pQuery) {
} }
static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void* pTable, STimeWindow win, void* buf) { static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void* pTable, STimeWindow win, void* buf) {
SQuery* pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
STableQueryInfo *pTableQueryInfo = buf;//calloc(1, sizeof(STableQueryInfo)); STableQueryInfo *pTableQueryInfo = buf;
pTableQueryInfo->win = win; pTableQueryInfo->win = win;
pTableQueryInfo->lastKey = win.skey; pTableQueryInfo->lastKey = win.skey;
@ -3337,16 +3363,14 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void
pTableQueryInfo->pTable = pTable; pTableQueryInfo->pTable = pTable;
pTableQueryInfo->cur.vgroupIndex = -1; pTableQueryInfo->cur.vgroupIndex = -1;
int32_t initialSize = 1;
int32_t initialThreshold = 1;
// set more initial size of interval/groupby query // set more initial size of interval/groupby query
if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) {
initialSize = 20; int32_t initialSize = 20;
initialThreshold = 100; int32_t initialThreshold = 100;
initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT);
} else { // in other aggregate query, do not initialize the windowResInfo
} }
initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT);
return pTableQueryInfo; return pTableQueryInfo;
} }
@ -3388,7 +3412,8 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
} }
int32_t GROUPRESULTID = 1; int32_t GROUPRESULTID = 1;
SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex)); SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex,
sizeof(groupIndex), true);
if (pWindowRes == NULL) { if (pWindowRes == NULL) {
return; return;
} }
@ -3521,12 +3546,12 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
* In ascending query, key is the first qualified timestamp. However, in the descending order query, additional * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional
* operations involve. * operations involve.
*/ */
STimeWindow w = TSWINDOW_INITIALIZER, realWin = TSWINDOW_INITIALIZER; STimeWindow w = TSWINDOW_INITIALIZER;
SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo;
TSKEY sk = MIN(win.skey, win.ekey); TSKEY sk = MIN(win.skey, win.ekey);
TSKEY ek = MAX(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey);
getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &realWin, &w); getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w);
pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp
if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
@ -3570,7 +3595,7 @@ static int32_t getNumOfSubset(SQInfo *pQInfo) {
SQuery *pQuery = pQInfo->runtimeEnv.pQuery; SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
int32_t totalSubset = 0; int32_t totalSubset = 0;
if (pQInfo->runtimeEnv.groupbyNormalCol || (isIntervalQuery(pQuery))) { if (pQInfo->runtimeEnv.groupbyNormalCol || (QUERY_IS_INTERVAL_QUERY(pQuery))) {
totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo); totalSubset = numOfClosedTimeWindow(&pQInfo->runtimeEnv.windowResInfo);
} else { } else {
totalSubset = GET_NUM_OF_TABLEGROUP(pQInfo); totalSubset = GET_NUM_OF_TABLEGROUP(pQInfo);
@ -3735,7 +3760,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) {
} else { } else {
// there are results waiting for returned to client. // there are results waiting for returned to client.
if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) &&
(pRuntimeEnv->groupbyNormalCol || isIntervalQuery(pQuery)) && (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) &&
(pRuntimeEnv->windowResInfo.size > 0)) { (pRuntimeEnv->windowResInfo.size > 0)) {
return true; return true;
} }
@ -3858,7 +3883,6 @@ static void queryCostStatis(SQInfo *pQInfo) {
// double total = pSummary->fileTimeUs + pSummary->cacheTimeUs; // double total = pSummary->fileTimeUs + pSummary->cacheTimeUs;
// double io = pSummary->loadCompInfoUs + pSummary->loadBlocksUs + pSummary->loadFieldUs; // double io = pSummary->loadCompInfoUs + pSummary->loadBlocksUs + pSummary->loadFieldUs;
// todo add the intermediate result save cost!!
// double computing = total - io; // double computing = total - io;
// //
// qDebug( // qDebug(
@ -3918,12 +3942,13 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
STableQueryInfo* pTableQueryInfo = pQuery->current; STableQueryInfo* pTableQueryInfo = pQuery->current;
TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle;
SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
while (tsdbNextDataBlock(pQueryHandle)) { while (tsdbNextDataBlock(pQueryHandle)) {
if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
return; return;
} }
SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo);
if (pQuery->limit.offset > blockInfo.rows) { if (pQuery->limit.offset > blockInfo.rows) {
pQuery->limit.offset -= blockInfo.rows; pQuery->limit.offset -= blockInfo.rows;
@ -3955,22 +3980,23 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
*/ */
assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL);
STimeWindow w = TSWINDOW_INITIALIZER, realWin = TSWINDOW_INITIALIZER; STimeWindow w = TSWINDOW_INITIALIZER;
SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
STableQueryInfo *pTableQueryInfo = pQuery->current; STableQueryInfo *pTableQueryInfo = pQuery->current;
SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) {
SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo);
if (QUERY_IS_ASC_QUERY(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) {
if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &realWin, &w); getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w);
pWindowResInfo->startTime = w.skey; pWindowResInfo->startTime = w.skey;
pWindowResInfo->prevSKey = w.skey; pWindowResInfo->prevSKey = w.skey;
} }
} else { } else {
getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &realWin, &w); getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w);
pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->startTime = pQuery->window.skey;
pWindowResInfo->prevSKey = w.skey; pWindowResInfo->prevSKey = w.skey;
@ -4067,7 +4093,7 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
return; return;
} }
if (isSTableQuery && (!isIntervalQuery(pQuery)) && (!isFixedOutputQuery(pQuery))) { if (isSTableQuery && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isFixedOutputQuery(pQuery))) {
return; return;
} }
@ -4081,7 +4107,7 @@ static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
if (!isSTableQuery if (!isSTableQuery
&& (pQInfo->tableqinfoGroupInfo.numOfTables == 1) && (pQInfo->tableqinfoGroupInfo.numOfTables == 1)
&& (cond.order == TSDB_ORDER_ASC) && (cond.order == TSDB_ORDER_ASC)
&& (!isIntervalQuery(pQuery)) && (!QUERY_IS_INTERVAL_QUERY(pQuery))
&& (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) && (!isGroupbyNormalCol(pQuery->pGroupbyExpr))
&& (!isFixedOutputQuery(pQuery)) && (!isFixedOutputQuery(pQuery))
) { ) {
@ -4174,7 +4200,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type);
} }
} else if (pRuntimeEnv->groupbyNormalCol || isIntervalQuery(pQuery)) { } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
int32_t rows = getInitialPageNum(pQInfo); int32_t rows = getInitialPageNum(pQInfo);
code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -4193,7 +4219,13 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) {
SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery); SFillColInfo* pColInfo = taosCreateFillColInfo(pQuery);
pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, 0, 0, pQuery->rec.capacity, pQuery->numOfOutput, STimeWindow w = TSWINDOW_INITIALIZER;
TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey);
TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey);
getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w);
pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, pQuery->rec.capacity, pQuery->numOfOutput,
pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision,
pQuery->fillType, pColInfo); pQuery->fillType, pColInfo);
} }
@ -4225,13 +4257,15 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle;
SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
while (tsdbNextDataBlock(pQueryHandle)) { while (tsdbNextDataBlock(pQueryHandle)) {
summary->totalBlocks += 1; summary->totalBlocks += 1;
if (isQueryKilled(pQInfo)) { if (isQueryKilled(pQInfo)) {
break; break;
} }
SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo);
STableQueryInfo **pTableQueryInfo = (STableQueryInfo**) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); STableQueryInfo **pTableQueryInfo = (STableQueryInfo**) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid));
if(pTableQueryInfo == NULL) { if(pTableQueryInfo == NULL) {
break; break;
@ -4244,7 +4278,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
if (!pRuntimeEnv->groupbyNormalCol) { if (!pRuntimeEnv->groupbyNormalCol) {
if (!isIntervalQuery(pQuery)) { if (!QUERY_IS_INTERVAL_QUERY(pQuery)) {
int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1; int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
setExecutionContext(pQInfo, (*pTableQueryInfo)->groupIndex, blockInfo.window.ekey + step); setExecutionContext(pQInfo, (*pTableQueryInfo)->groupIndex, blockInfo.window.ekey + step);
} else { // interval query } else { // interval query
@ -4653,9 +4687,9 @@ static void doRestoreContext(SQInfo *pQInfo) {
static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) {
SQuery *pQuery = pQInfo->runtimeEnv.pQuery; SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
if (isIntervalQuery(pQuery)) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo);
for (int32_t i = 0; i < numOfGroup; ++i) { for (int32_t i = 0; i < numOfGroup; ++i) {
SArray *group = GET_TABLEGROUP(pQInfo, i); SArray *group = GET_TABLEGROUP(pQInfo, i);
@ -4664,7 +4698,7 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) {
for (int32_t j = 0; j < num; ++j) { for (int32_t j = 0; j < num; ++j) {
STableQueryInfo* item = taosArrayGetP(group, j); STableQueryInfo* item = taosArrayGetP(group, j);
closeAllTimeWindow(&item->windowResInfo); closeAllTimeWindow(&item->windowResInfo);
removeRedundantWindow(&item->windowResInfo, item->lastKey - step, step); // removeRedundantWindow(&item->windowResInfo, item->lastKey - step, step);
} }
} }
} else { // close results for group result } else { // close results for group result
@ -4681,7 +4715,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
* if the groupIndex > 0, the query process must be completed yet, we only need to * if the groupIndex > 0, the query process must be completed yet, we only need to
* copy the data into output buffer * copy the data into output buffer
*/ */
if (isIntervalQuery(pQuery)) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
copyResToQueryResultBuf(pQInfo, pQuery); copyResToQueryResultBuf(pQInfo, pQuery);
#ifdef _DEBUG_VIEW #ifdef _DEBUG_VIEW
displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num);
@ -4716,7 +4750,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
el = scanMultiTableDataBlocks(pQInfo); el = scanMultiTableDataBlocks(pQInfo);
qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el);
doCloseAllTimeWindowAfterScan(pQInfo); // doCloseAllTimeWindowAfterScan(pQInfo);
doRestoreContext(pQInfo); doRestoreContext(pQInfo);
} else { } else {
qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo);
@ -4890,7 +4924,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
while (1) { while (1) {
tableIntervalProcessImpl(pRuntimeEnv, newStartKey); tableIntervalProcessImpl(pRuntimeEnv, newStartKey);
if (isIntervalQuery(pQuery)) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
pQInfo->groupIndex = 0; // always start from 0 pQInfo->groupIndex = 0; // always start from 0
pQuery->rec.rows = 0; pQuery->rec.rows = 0;
copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
@ -5788,7 +5822,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
qDebug("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, qDebug("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey,
pQuery->window.ekey, pQuery->order.order); pQuery->window.ekey, pQuery->order.order);
setQueryStatus(pQuery, QUERY_COMPLETED); setQueryStatus(pQuery, QUERY_COMPLETED);
pQInfo->tableqinfoGroupInfo.numOfTables = 0;
sem_post(&pQInfo->dataReady); sem_post(&pQInfo->dataReady);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -6460,7 +6494,7 @@ void qSetQueryMgmtClosed(void* pQMgmt) {
pQueryMgmt->closed = true; pQueryMgmt->closed = true;
pthread_mutex_unlock(&pQueryMgmt->lock); pthread_mutex_unlock(&pQueryMgmt->lock);
taosCacheEmpty(pQueryMgmt->qinfoPool, true); taosCacheRefresh(pQueryMgmt->qinfoPool, freeqinfoFn);
} }
void qCleanupQueryMgmt(void* pQMgmt) { void qCleanupQueryMgmt(void* pQMgmt) {
@ -6483,11 +6517,13 @@ void qCleanupQueryMgmt(void* pQMgmt) {
qDebug("vgId:%d querymgmt cleanup completed", vgId); qDebug("vgId:%d querymgmt cleanup completed", vgId);
} }
void** qRegisterQInfo(void* pMgmt, void* qInfo) { void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
if (pMgmt == NULL) { if (pMgmt == NULL) {
return NULL; return NULL;
} }
const int32_t DEFAULT_QHANDLE_LIFE_SPAN = tsShellActivityTimer * 2;
SQueryMgmt *pQueryMgmt = pMgmt; SQueryMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->qinfoPool == NULL) { if (pQueryMgmt->qinfoPool == NULL) {
return NULL; return NULL;
@ -6499,21 +6535,23 @@ void** qRegisterQInfo(void* pMgmt, void* qInfo) {
return NULL; return NULL;
} else { } else {
void** handle = taosCachePut(pQueryMgmt->qinfoPool, qInfo, POINTER_BYTES, &qInfo, POINTER_BYTES, tsShellActivityTimer*2); uint64_t handleVal = (uint64_t) qInfo;
void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(int64_t), &qInfo, POINTER_BYTES, DEFAULT_QHANDLE_LIFE_SPAN);
pthread_mutex_unlock(&pQueryMgmt->lock); pthread_mutex_unlock(&pQueryMgmt->lock);
return handle; return handle;
} }
} }
void** qAcquireQInfo(void* pMgmt, void** key) { void** qAcquireQInfo(void* pMgmt, uint64_t key) {
SQueryMgmt *pQueryMgmt = pMgmt; SQueryMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->qinfoPool == NULL || pQueryMgmt->closed) { if (pQueryMgmt->qinfoPool == NULL || pQueryMgmt->closed) {
return NULL; return NULL;
} }
void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, key, POINTER_BYTES); void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &key, sizeof(uint64_t));
if (handle == NULL || *handle == NULL) { if (handle == NULL || *handle == NULL) {
return NULL; return NULL;
} else { } else {

View File

@ -190,8 +190,7 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
} }
// get the result order // get the result order
int32_t resultOrder = (pWindowResInfo->pResult[0].window.skey < pWindowResInfo->pResult[1].window.skey)? int32_t resultOrder = (pWindowResInfo->pResult[0].window.skey < pWindowResInfo->pResult[1].window.skey)? 1:-1;
TSDB_ORDER_ASC:TSDB_ORDER_DESC;
if (order != resultOrder) { if (order != resultOrder) {
return; return;

View File

@ -13,25 +13,26 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h" #include "os.h"
#include "tulog.h"
#include "tutil.h" #include "tname.h"
#include "tbuffer.h"
#include "qast.h" #include "qast.h"
#include "tcompare.h" #include "tsdb.h"
#include "exception.h"
#include "qsqlparser.h" #include "qsqlparser.h"
#include "qsyntaxtreefunction.h" #include "qsyntaxtreefunction.h"
#include "taosdef.h" #include "taosdef.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tarray.h"
#include "tbuffer.h"
#include "tcompare.h"
#include "tskiplist.h"
#include "tsqlfunction.h" #include "tsqlfunction.h"
#include "tstoken.h" #include "tstoken.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tschemautil.h" #include "tulog.h"
#include "tarray.h" #include "tutil.h"
#include "tskiplist.h"
#include "queryLog.h"
#include "tsdbMain.h"
#include "exception.h"
/* /*
* *
@ -327,104 +328,6 @@ static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *st
} }
} }
void tSQLBinaryExprFromString(tExprNode **pExpr, SSchema *pSchema, int32_t numOfCols, char *src, int32_t len) {
*pExpr = NULL;
if (len <= 0 || src == NULL || pSchema == NULL || numOfCols <= 0) {
return;
}
int32_t pos = 0;
*pExpr = createSyntaxTree(pSchema, numOfCols, src, &pos);
if (*pExpr != NULL) {
assert((*pExpr)->nodeType == TSQL_NODE_EXPR);
}
}
int32_t tSQLBinaryExprToStringImpl(tExprNode *pNode, char *dst, uint8_t type) {
int32_t len = 0;
if (type == TSQL_NODE_EXPR) {
*dst = '(';
tSQLBinaryExprToString(pNode, dst + 1, &len);
len += 2;
*(dst + len - 1) = ')';
} else if (type == TSQL_NODE_COL) {
len = sprintf(dst, "%s", pNode->pSchema->name);
} else {
len = tVariantToString(pNode->pVal, dst);
}
return len;
}
// TODO REFACTOR WITH SQL PARSER
static char *tSQLOptrToString(uint8_t optr, char *dst) {
switch (optr) {
case TSDB_RELATION_LESS: {
*dst = '<';
dst += 1;
break;
}
case TSDB_RELATION_LESS_EQUAL: {
*dst = '<';
*(dst + 1) = '=';
dst += 2;
break;
}
case TSDB_RELATION_EQUAL: {
*dst = '=';
dst += 1;
break;
}
case TSDB_RELATION_GREATER: {
*dst = '>';
dst += 1;
break;
}
case TSDB_RELATION_GREATER_EQUAL: {
*dst = '>';
*(dst + 1) = '=';
dst += 2;
break;
}
case TSDB_RELATION_NOT_EQUAL: {
*dst = '<';
*(dst + 1) = '>';
dst += 2;
break;
}
case TSDB_RELATION_OR: {
memcpy(dst, "or", 2);
dst += 2;
break;
}
case TSDB_RELATION_AND: {
memcpy(dst, "and", 3);
dst += 3;
break;
}
default:;
}
return dst;
}
void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len) {
if (pExpr == NULL) {
*dst = 0;
*len = 0;
return;
}
int32_t lhs = tSQLBinaryExprToStringImpl(pExpr->_node.pLeft, dst, pExpr->_node.pLeft->nodeType);
dst += lhs;
*len = lhs;
char *start = tSQLOptrToString(pExpr->_node.optr, dst);
*len += (start - dst);
*len += tSQLBinaryExprToStringImpl(pExpr->_node.pRight, start, pExpr->_node.pRight->nodeType);
}
static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); } static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); }
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
@ -773,8 +676,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
SSkipListNode *pNode = tSkipListIterGet(iter); SSkipListNode *pNode = tSkipListIterGet(iter);
char * pData = SL_GET_NODE_DATA(pNode); char * pData = SL_GET_NODE_DATA(pNode);
// todo refactor: tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
tstr *name = (*(STable **)pData)->name;
// todo speed up by using hash // todo speed up by using hash
if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
if (pQueryInfo->optr == TSDB_RELATION_IN) { if (pQueryInfo->optr == TSDB_RELATION_IN) {
@ -976,27 +878,27 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
free(pRightOutput); free(pRightOutput);
} }
void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) { //void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) {
if (pExprs == NULL) { // if (pExprs == NULL) {
return; // return;
} // }
//
tExprNode *pLeft = pExprs->_node.pLeft; // tExprNode *pLeft = pExprs->_node.pLeft;
tExprNode *pRight = pExprs->_node.pRight; // tExprNode *pRight = pExprs->_node.pRight;
//
// recursive traverse left child branch // // recursive traverse left child branch
if (pLeft->nodeType == TSQL_NODE_EXPR) { // if (pLeft->nodeType == TSQL_NODE_EXPR) {
tSQLBinaryExprTrv(pLeft, res); // tSQLBinaryExprTrv(pLeft, res);
} else if (pLeft->nodeType == TSQL_NODE_COL) { // } else if (pLeft->nodeType == TSQL_NODE_COL) {
taosArrayPush(res, &pLeft->pSchema->colId); // taosArrayPush(res, &pLeft->pSchema->colId);
} // }
//
if (pRight->nodeType == TSQL_NODE_EXPR) { // if (pRight->nodeType == TSQL_NODE_EXPR) {
tSQLBinaryExprTrv(pRight, res); // tSQLBinaryExprTrv(pRight, res);
} else if (pRight->nodeType == TSQL_NODE_COL) { // } else if (pRight->nodeType == TSQL_NODE_COL) {
taosArrayPush(res, &pRight->pSchema->colId); // taosArrayPush(res, &pRight->pSchema->colId);
} // }
} //}
static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
tbufWriteUint8(bw, expr->nodeType); tbufWriteUint8(bw, expr->nodeType);

View File

@ -174,7 +174,7 @@ int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
return 0; return 0;
} }
return FILL_IS_ASC_FILL(pFillInfo) ? (pFillInfo->numOfRows - pFillInfo->rowIdx) : pFillInfo->rowIdx + 1; return pFillInfo->numOfRows - pFillInfo->rowIdx;
} }
// todo: refactor // todo: refactor

View File

@ -1,11 +1,10 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <qast.h>
#include <sys/time.h> #include <sys/time.h>
#include <cassert> #include <cassert>
#include <iostream> #include <iostream>
#include "qast.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "qast.h"
#include "tsdb.h" #include "tsdb.h"
#include "tskiplist.h" #include "tskiplist.h"
@ -24,8 +23,6 @@ static void initSchema_binary(SSchema *schema, int32_t numOfCols);
static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags); static SSkipList *createSkipList(SSchema *pSchema, int32_t numOfTags);
static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags); static SSkipList *createSkipList_binary(SSchema *pSchema, int32_t numOfTags);
static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, SSkipList *pSkipList, ResultObj *expectedVal);
static void dropMeter(SSkipList *pSkipList); static void dropMeter(SSkipList *pSkipList);
static void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList); static void Right2LeftTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList);
@ -239,44 +236,45 @@ static void initSchema(SSchema *schema, int32_t numOfCols) {
// return pSkipList; // return pSkipList;
//} //}
static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, SSkipList *pSkipList, ResultObj *pResult) { //static void testQueryStr(SSchema *schema, int32_t numOfCols, char *sql, SSkipList *pSkipList, ResultObj *pResult) {
tExprNode *pExpr = NULL; // tExprNode *pExpr = NULL;
tSQLBinaryExprFromString(&pExpr, schema, numOfCols, sql, strlen(sql)); // tSQLBinaryExprFromString(&pExpr, schema, numOfCols, sql, strlen(sql));
//
char str[512] = {0}; // char str[512] = {0};
int32_t len = 0; // int32_t len = 0;
if (pExpr == NULL) { // if (pExpr == NULL) {
printf("-----error in parse syntax:%s\n\n", sql); // printf("-----error in parse syntax:%s\n\n", sql);
assert(pResult == NULL); // assert(pResult == NULL);
return; // return;
} // }
//
tSQLBinaryExprToString(pExpr, str, &len); // tSQLBinaryExprToString(pExpr, str, &len);
printf("expr is: %s\n", str); // printf("expr is: %s\n", str);
//
SArray *result = NULL; // SArray *result = NULL;
// tExprTreeTraverse(pExpr, pSkipList, result, SSkipListNodeFilterCallback, &result); // // tExprTreeTraverse(pExpr, pSkipList, result, SSkipListNodeFilterCallback, &result);
// printf("the result is:%lld\n", result.num); // // printf("the result is:%lld\n", result.num);
// // //
// bool findResult = false; // // bool findResult = false;
// for (int32_t i = 0; i < result.num; ++i) { // // for (int32_t i = 0; i < result.num; ++i) {
// STabObj *pm = (STabObj *)result.pRes[i]; // // STabObj *pm = (STabObj *)result.pRes[i];
// printf("meterid:%s,\t", pm->meterId); // // printf("meterid:%s,\t", pm->meterId);
// // //
// for (int32_t j = 0; j < pResult->numOfResult; ++j) { // // for (int32_t j = 0; j < pResult->numOfResult; ++j) {
// if (strcmp(pm->meterId, pResult->resultName[j]) == 0) { // // if (strcmp(pm->meterId, pResult->resultName[j]) == 0) {
// findResult = true; // // findResult = true;
// break; // // break;
// } // // }
// } // // }
// assert(findResult == true); // // assert(findResult == true);
// findResult = false; // // findResult = false;
// } // // }
//
printf("\n\n"); // printf("\n\n");
tExprTreeDestroy(&pExpr, NULL); // tExprTreeDestroy(&pExpr, NULL);
} //}
#if 0
static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) { static void Left2RightTest(SSchema *schema, int32_t numOfCols, SSkipList *pSkipList) {
char str[256] = {0}; char str[256] = {0};
@ -632,4 +630,5 @@ void exprSerializeTest2() {
} // namespace } // namespace
TEST(testCase, astTest) { TEST(testCase, astTest) {
// exprSerializeTest2(); // exprSerializeTest2();
} }
#endif

View File

@ -156,6 +156,7 @@ int main(int argc, char *argv[]) {
} }
tInfo("client is initialized"); tInfo("client is initialized");
tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs);
gettimeofday(&systemTime, NULL); gettimeofday(&systemTime, NULL);
startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;

View File

@ -24,23 +24,21 @@ int msgSize = 128;
int commit = 0; int commit = 0;
int dataFd = -1; int dataFd = -1;
void *qhandle = NULL; void *qhandle = NULL;
void *qset = NULL;
void processShellMsg() { void processShellMsg() {
static int num = 0; static int num = 0;
taos_qall qall; taos_qall qall;
SRpcMsg *pRpcMsg, rpcMsg; SRpcMsg *pRpcMsg, rpcMsg;
int type; int type;
void *pvnode;
qall = taosAllocateQall(); qall = taosAllocateQall();
while (1) { while (1) {
int numOfMsgs = taosReadAllQitems(qhandle, qall); int numOfMsgs = taosReadAllQitemsFromQset(qset, qall, &pvnode);
if (numOfMsgs <= 0) {
usleep(100);
continue;
}
tDebug("%d shell msgs are received", numOfMsgs); tDebug("%d shell msgs are received", numOfMsgs);
if (numOfMsgs <= 0) break;
for (int i=0; i<numOfMsgs; ++i) { for (int i=0; i<numOfMsgs; ++i) {
taosGetQitem(qall, &type, (void **)&pRpcMsg); taosGetQitem(qall, &type, (void **)&pRpcMsg);
@ -82,15 +80,6 @@ void processShellMsg() {
} }
taosFreeQall(qall); taosFreeQall(qall);
/*
SRpcIpSet ipSet;
ipSet.numOfIps = 1;
ipSet.index = 0;
ipSet.port = 7000;
ipSet.ip[0] = inet_addr("192.168.0.2");
rpcSendRedirectRsp(ahandle, &ipSet);
*/
} }
@ -189,6 +178,8 @@ int main(int argc, char *argv[]) {
} }
qhandle = taosOpenQueue(sizeof(SRpcMsg)); qhandle = taosOpenQueue(sizeof(SRpcMsg));
qset = taosOpenQset();
taosAddIntoQset(qset, qhandle, NULL);
processShellMsg(); processShellMsg();

View File

@ -195,7 +195,6 @@ typedef struct {
typedef struct { typedef struct {
uint32_t len; uint32_t len;
uint32_t offset; uint32_t offset;
// uint32_t padding;
uint32_t hasLast : 2; uint32_t hasLast : 2;
uint32_t numOfBlocks : 30; uint32_t numOfBlocks : 30;
uint64_t uid; uint64_t uid;
@ -224,7 +223,7 @@ typedef struct {
typedef struct { typedef struct {
int16_t colId; int16_t colId;
int16_t len; int32_t len;
int32_t type : 8; int32_t type : 8;
int32_t offset : 24; int32_t offset : 24;
int64_t sum; int64_t sum;
@ -438,8 +437,9 @@ int tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); int tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, int16_t* colIds, int numOfColIds); int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds,
int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock); int numOfColIds);
int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo);
// ------------------ tsdbMain.c // ------------------ tsdbMain.c
#define REPO_ID(r) (r)->config.tsdbId #define REPO_ID(r) (r)->config.tsdbId

View File

@ -192,11 +192,6 @@ char *tsdbGetTableName(void* pTable) {
} }
} }
STableId tsdbGetTableId(void *pTable) {
assert(pTable);
return ((STable*)pTable)->tableId;
}
STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) {
if (pMsg == NULL) return NULL; if (pMsg == NULL) return NULL;

View File

@ -318,7 +318,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
ASSERT(pCompBlock->last); ASSERT(pCompBlock->last);
if (pCompBlock->numOfSubBlocks > 1) { if (pCompBlock->numOfSubBlocks > 1) {
if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1)) < 0) return -1; if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows > 0 && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); ASSERT(pHelper->pDataCols[0]->numOfRows > 0 && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0],
pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0) pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0)
@ -577,11 +577,12 @@ void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols)
} }
} }
int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds) { int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo, int16_t *colIds, int numOfColIds) {
ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block
int numOfSubBlocks = pCompBlock->numOfSubBlocks; int numOfSubBlocks = pCompBlock->numOfSubBlocks;
if (numOfSubBlocks > 1) pCompBlock = (SCompBlock *)POINTER_SHIFT(pHelper->pCompInfo, pCompBlock->offset); if (numOfSubBlocks > 1)
pCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset);
tdResetDataCols(pHelper->pDataCols[0]); tdResetDataCols(pHelper->pDataCols[0]);
if (tsdbLoadBlockDataColsImpl(pHelper, pCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err; if (tsdbLoadBlockDataColsImpl(pHelper, pCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err;
@ -598,10 +599,10 @@ _err:
return -1; return -1;
} }
int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock) { int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo) {
int numOfSubBlock = pCompBlock->numOfSubBlocks; int numOfSubBlock = pCompBlock->numOfSubBlocks;
if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)POINTER_SHIFT(pHelper->pCompInfo, pCompBlock->offset); if (numOfSubBlock > 1)
pCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset);
tdResetDataCols(pHelper->pDataCols[0]); tdResetDataCols(pHelper->pDataCols[0]);
if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err; if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err;
@ -703,6 +704,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
} }
// Add checksum // Add checksum
ASSERT(pCompCol->len > 0);
pCompCol->len += sizeof(TSCKSUM); pCompCol->len += sizeof(TSCKSUM);
taosCalcChecksumAppend(0, (uint8_t *)tptr, pCompCol->len); taosCalcChecksumAppend(0, (uint8_t *)tptr, pCompCol->len);
@ -792,7 +794,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err;
} else { } else {
// Load // Load
if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx)) < 0) goto _err; if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err;
ASSERT(pHelper->pDataCols[0]->numOfRows <= blockAtIdx(pHelper, blkIdx)->numOfRows); ASSERT(pHelper->pDataCols[0]->numOfRows <= blockAtIdx(pHelper, blkIdx)->numOfRows);
// Merge // Merge
if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err;
@ -848,7 +850,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err;
} else { // Load-Merge-Write } else { // Load-Merge-Write
// Load // Load
if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx)) < 0) goto _err; if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err;
if (blockAtIdx(pHelper, blkIdx)->last) pHelper->hasOldLastBlock = false; if (blockAtIdx(pHelper, blkIdx)->last) pHelper->hasOldLastBlock = false;
rowsWritten = rows3; rowsWritten = rows3;

View File

@ -22,7 +22,7 @@
#include "exception.h" #include "exception.h"
#include "../../../query/inc/qast.h" // todo move to common module #include "../../../query/inc/qast.h" // todo move to common module
#include "../../../query/inc/tlosertree.h" // todo move to util module #include "tlosertree.h"
#include "tsdb.h" #include "tsdb.h"
#include "tsdbMain.h" #include "tsdbMain.h"
@ -122,7 +122,7 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle);
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
SArray* sa); SArray* sa);
static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win,
STsdbQueryHandle* pQueryHandle); STsdbQueryHandle* pQueryHandle);
static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
@ -249,8 +249,6 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
pCheckInfo->initBuf = true; pCheckInfo->initBuf = true;
int32_t order = pHandle->order; int32_t order = pHandle->order;
// tsdbTakeMemSnapshot(pHandle->pTsdb, &pCheckInfo->mem, &pCheckInfo->imem);
// no data in buffer, abort // no data in buffer, abort
if (pHandle->mem == NULL && pHandle->imem == NULL) { if (pHandle->mem == NULL && pHandle->imem == NULL) {
return false; return false;
@ -392,8 +390,11 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
STable* pTable = pCheckInfo->pTableObj; STable* pTable = pCheckInfo->pTableObj;
assert(pTable != NULL); assert(pTable != NULL);
initTableMemIterator(pHandle, pCheckInfo); if (!pCheckInfo->initBuf) {
initTableMemIterator(pHandle, pCheckInfo);
}
SDataRow row = getSDataRowInTableMem(pCheckInfo); SDataRow row = getSDataRowInTableMem(pCheckInfo);
if (row == NULL) { if (row == NULL) {
return false; return false;
@ -411,8 +412,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1; int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1;
STimeWindow* win = &pHandle->cur.win; STimeWindow* win = &pHandle->cur.win;
pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, win, pHandle);
pHandle->outputCapacity, &win->skey, &win->ekey, pHandle); // todo refactor API
// update the last key value // update the last key value
pCheckInfo->lastKey = win->ekey + step; pCheckInfo->lastKey = win->ekey + step;
@ -576,6 +576,8 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
STsdbRepo *pRepo = pQueryHandle->pTsdb; STsdbRepo *pRepo = pQueryHandle->pTsdb;
// TODO refactor
SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
data->numOfCols = pBlock->numOfCols; data->numOfCols = pBlock->numOfCols;
@ -592,9 +594,12 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
} }
tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj)); STSchema* pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj);
tdInitDataCols(pCheckInfo->pDataCols, pSchema);
tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema);
tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema);
if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock) == 0) { if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo) == 0) {
SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo;
pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup;
@ -605,8 +610,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
} }
SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
assert(pCols->numOfRows != 0); assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
pBlock->numOfRows = pCols->numOfRows;
taosArrayDestroy(sa); taosArrayDestroy(sa);
tfree(data); tfree(data);
@ -636,7 +642,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step,
pQueryHandle->outputCapacity, &cur->win.skey, &cur->win.ekey, pQueryHandle); pQueryHandle->outputCapacity, &cur->win, pQueryHandle);
pQueryHandle->realNumOfRows = cur->rows; pQueryHandle->realNumOfRows = cur->rows;
// update the last key value // update the last key value
@ -1237,7 +1243,6 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void*
// assert(pLeftBlockInfoEx->compBlock->offset != pRightBlockInfoEx->compBlock->offset); // assert(pLeftBlockInfoEx->compBlock->offset != pRightBlockInfoEx->compBlock->offset);
if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset && if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset &&
pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) { pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) {
// todo add more information
tsdbError("error in header file, two block with same offset:%" PRId64, (int64_t)pLeftBlockInfoEx->compBlock->offset); tsdbError("error in header file, two block with same offset:%" PRId64, (int64_t)pLeftBlockInfoEx->compBlock->offset);
} }
@ -1477,7 +1482,8 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
assert(numOfTables > 0); assert(numOfTables > 0);
SDataBlockInfo blockInfo = {{0}, 0};
if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) { if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
pQueryHandle->order = TSDB_ORDER_DESC; pQueryHandle->order = TSDB_ORDER_DESC;
@ -1487,7 +1493,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
} }
SArray* sa = getDefaultLoadColumns(pQueryHandle, true); SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
/*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle); /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
/*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, sa); /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, sa);
if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) { if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
@ -1558,7 +1564,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
bool ret = tsdbNextDataBlock((void*) pSecQueryHandle); bool ret = tsdbNextDataBlock((void*) pSecQueryHandle);
assert(ret); assert(ret);
/*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle); /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
/*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pSecQueryHandle, sa); /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pSecQueryHandle, sa);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
@ -1693,11 +1699,11 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL};
} }
static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win,
STsdbQueryHandle* pQueryHandle) { STsdbQueryHandle* pQueryHandle) {
int numOfRows = 0; int numOfRows = 0;
int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
*skey = TSKEY_INITIAL_VAL; win->skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
@ -1717,11 +1723,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
break; break;
} }
if (*skey == INT64_MIN) { if (win->skey == INT64_MIN) {
*skey = key; win->skey = key;
} }
*ekey = key; win->ekey = key;
copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable); copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable);
if (++numOfRows >= maxRowsToRead) { if (++numOfRows >= maxRowsToRead) {
@ -1750,7 +1756,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
return numOfRows; return numOfRows;
} }
SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) { void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
SQueryFilePos* cur = &pHandle->cur; SQueryFilePos* cur = &pHandle->cur;
STable* pTable = NULL; STable* pTable = NULL;
@ -1763,16 +1769,12 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
pTable = pCheckInfo->pTableObj; pTable = pCheckInfo->pTableObj;
} }
SDataBlockInfo blockInfo = {
.uid = pTable->tableId.uid,
.tid = pTable->tableId.tid,
.rows = cur->rows,
.window = cur->win,
.numOfCols = QH_GET_NUM_OF_COLS(pHandle),
};
return blockInfo; pDataBlockInfo->uid = pTable->tableId.uid;
pDataBlockInfo->tid = pTable->tableId.tid;
pDataBlockInfo->rows = cur->rows;
pDataBlockInfo->window = cur->win;
pDataBlockInfo->numOfCols = QH_GET_NUM_OF_COLS(pHandle);
} }
/* /*
@ -1974,9 +1976,9 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
int32_t type = 0; int32_t type = 0;
int32_t bytes = 0; int32_t bytes = 0;
if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor extract method , to queryExecutor to generate tags values if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
f1 = (char*) pTable1->name; f1 = (char*) TABLE_NAME(pTable1);
f2 = (char*) pTable2->name; f2 = (char*) TABLE_NAME(pTable2);
type = TSDB_DATA_TYPE_BINARY; type = TSDB_DATA_TYPE_BINARY;
bytes = tGetTableNameColumnSchema().bytes; bytes = tGetTableNameColumnSchema().bytes;
} else { } else {
@ -2085,13 +2087,17 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
char* val = NULL; char* val = NULL;
if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
val = (char*) pTable->name; val = (char*) TABLE_NAME(pTable);
} else { } else {
val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);
} }
//todo :the val is possible to be null, so check it out carefully int32_t ret = 0;
int32_t ret = pInfo->compare(val, pInfo->q); if (val == NULL) { //the val is possible to be null, so check it out carefully
ret = -1; // val is missing in table tags value pairs
} else {
ret = pInfo->compare(val, pInfo->q);
}
switch (pInfo->optr) { switch (pInfo->optr) {
case TSDB_RELATION_EQUAL: { case TSDB_RELATION_EQUAL: {

View File

@ -24,14 +24,13 @@ extern "C" {
#include "tref.h" #include "tref.h"
#include "hash.h" #include "hash.h"
typedef void (*__cache_freeres_fn_t)(void*); typedef void (*__cache_free_fn_t)(void*);
typedef struct SCacheStatis { typedef struct SCacheStatis {
int64_t missCount; int64_t missCount;
int64_t hitCount; int64_t hitCount;
int64_t totalAccess; int64_t totalAccess;
int64_t refreshCount; int64_t refreshCount;
int32_t numOfCollision;
} SCacheStatis; } SCacheStatis;
typedef struct SCacheDataNode { typedef struct SCacheDataNode {
@ -70,7 +69,7 @@ typedef struct {
// void * pTimer; // void * pTimer;
SCacheStatis statistics; SCacheStatis statistics;
SHashObj * pHashTable; SHashObj * pHashTable;
__cache_freeres_fn_t freeFp; __cache_free_fn_t freeFp;
uint32_t numOfElemsInTrash; // number of element in trash uint32_t numOfElemsInTrash; // number of element in trash
uint8_t deleting; // set the deleting flag to stop refreshing ASAP. uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
pthread_t refreshWorker; pthread_t refreshWorker;
@ -91,15 +90,7 @@ typedef struct {
* @param fn free resource callback function * @param fn free resource callback function
* @return * @return
*/ */
SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn, const char *cacheName); SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName);
/**
* initialize the cache object and set the free object callback function
* @param refreshTimeInSeconds
* @param freeCb
* @return
*/
SCacheObj *taosCacheInitWithCb(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn, const char *cacheName);
/** /**
* add data into cache * add data into cache
@ -163,9 +154,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
/** /**
* move all data node into trash, clear node in trash can if it is not referenced by any clients * move all data node into trash, clear node in trash can if it is not referenced by any clients
* @param handle * @param handle
* @param _remove remove the data or not if refcount is greater than 0
*/ */
void taosCacheEmpty(SCacheObj *pCacheObj, bool _remove); void taosCacheEmpty(SCacheObj *pCacheObj);
/** /**
* release all allocated memory and destroy the cache object. * release all allocated memory and destroy the cache object.
@ -180,6 +170,14 @@ void taosCacheEmpty(SCacheObj *pCacheObj, bool _remove);
*/ */
void taosCacheCleanup(SCacheObj *pCacheObj); void taosCacheCleanup(SCacheObj *pCacheObj);
/**
*
* @param pCacheObj
* @param fp
* @return
*/
void taosCacheRefresh(SCacheObj *pCacheObj, __cache_free_fn_t fp);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -51,6 +51,7 @@ typedef struct SSkipListNode {
#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n))) #define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0))) #define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0)))
#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0)))
#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) #define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
@ -119,7 +120,6 @@ typedef struct SSkipList {
pthread_rwlock_t *lock; pthread_rwlock_t *lock;
SSkipListNode * pHead; // point to the first element SSkipListNode * pHead; // point to the first element
SSkipListNode * pTail; // point to the last element SSkipListNode * pTail; // point to the last element
void * lastKey; // last key in the skiplist
#if SKIP_LIST_RECORD_PERFORMANCE #if SKIP_LIST_RECORD_PERFORMANCE
tSkipListState state; // skiplist state tSkipListState state; // skiplist state
#endif #endif

View File

@ -223,9 +223,9 @@ static void doCleanupDataCache(SCacheObj *pCacheObj);
* refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime
* @param handle Cache object handle * @param handle Cache object handle
*/ */
static void* taosCacheRefresh(void *handle); static void* taosCacheTimedRefresh(void *pCacheObj);
SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_freeres_fn_t fn, const char* cacheName) { SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char* cacheName) {
if (refreshTimeInSeconds <= 0) { if (refreshTimeInSeconds <= 0) {
return NULL; return NULL;
} }
@ -261,7 +261,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_create(&pCacheObj->refreshWorker, &thattr, taosCacheRefresh, pCacheObj); pthread_create(&pCacheObj->refreshWorker, &thattr, taosCacheTimedRefresh, pCacheObj);
pthread_attr_destroy(&thattr); pthread_attr_destroy(&thattr);
return pCacheObj; return pCacheObj;
@ -450,7 +450,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
} }
} }
void taosCacheEmpty(SCacheObj *pCacheObj, bool _remove) { void taosCacheEmpty(SCacheObj *pCacheObj) {
SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
__cache_wr_lock(pCacheObj); __cache_wr_lock(pCacheObj);
@ -459,8 +459,8 @@ void taosCacheEmpty(SCacheObj *pCacheObj, bool _remove) {
break; break;
} }
SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); SCacheDataNode *pNode = *(SCacheDataNode **) taosHashIterGet(pIter);
if (T_REF_VAL_GET(pNode) == 0 || _remove) { if (T_REF_VAL_GET(pNode) == 0) {
taosCacheReleaseNode(pCacheObj, pNode); taosCacheReleaseNode(pCacheObj, pNode);
} else { } else {
taosCacheMoveToTrash(pCacheObj, pNode); taosCacheMoveToTrash(pCacheObj, pNode);
@ -469,7 +469,7 @@ void taosCacheEmpty(SCacheObj *pCacheObj, bool _remove) {
__cache_unlock(pCacheObj); __cache_unlock(pCacheObj);
taosHashDestroyIter(pIter); taosHashDestroyIter(pIter);
taosTrashCanEmpty(pCacheObj, _remove); taosTrashCanEmpty(pCacheObj, false);
} }
void taosCacheCleanup(SCacheObj *pCacheObj) { void taosCacheCleanup(SCacheObj *pCacheObj) {
@ -623,8 +623,29 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
free(pCacheObj); free(pCacheObj);
} }
void* taosCacheRefresh(void *handle) { static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t fp) {
SCacheObj *pCacheObj = (SCacheObj *)handle; SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
__cache_wr_lock(pCacheObj);
while (taosHashIterNext(pIter)) {
SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
if ((pNode->addedTime + pNode->lifespan * pNode->extendFactor) <= time && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
continue;
}
if (fp) {
fp(pNode->data);
}
}
__cache_unlock(pCacheObj);
taosHashDestroyIter(pIter);
}
void* taosCacheTimedRefresh(void *handle) {
SCacheObj* pCacheObj = handle;
if (pCacheObj == NULL) { if (pCacheObj == NULL) {
uDebug("object is destroyed. no refresh retry"); uDebug("object is destroyed. no refresh retry");
return NULL; return NULL;
@ -657,21 +678,8 @@ void* taosCacheRefresh(void *handle) {
// refresh data in hash table // refresh data in hash table
if (elemInHash > 0) { if (elemInHash > 0) {
int64_t expiredTime = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
doCacheRefresh(pCacheObj, now, NULL);
SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
__cache_wr_lock(pCacheObj);
while (taosHashIterNext(pIter)) {
SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
if ((pNode->addedTime + pNode->lifespan * pNode->extendFactor) <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
}
}
__cache_unlock(pCacheObj);
taosHashDestroyIter(pIter);
} }
taosTrashCanEmpty(pCacheObj, false); taosTrashCanEmpty(pCacheObj, false);
@ -679,3 +687,12 @@ void* taosCacheRefresh(void *handle) {
return NULL; return NULL;
} }
void taosCacheRefresh(SCacheObj *pCacheObj, __cache_free_fn_t fp) {
if (pCacheObj == NULL) {
return;
}
int64_t now = taosGetTimestampMs();
doCacheRefresh(pCacheObj, now, fp);
}

View File

@ -13,10 +13,10 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tlosertree.h"
#include "os.h" #include "os.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tlosertree.h" #include "tulog.h"
#include "queryLog.h"
// set initial value for loser tree // set initial value for loser tree
void tLoserTreeInit(SLoserTreeInfo* pTree) { void tLoserTreeInit(SLoserTreeInfo* pTree) {
@ -45,7 +45,7 @@ uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* pa
*pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries); *pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries);
if ((*pTree) == NULL) { if ((*pTree) == NULL) {
qError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); uError("allocate memory for loser-tree failed. reason:%s", strerror(errno));
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }

View File

@ -93,16 +93,18 @@ void taosCloseQueue(taos_queue param) {
void *taosAllocateQitem(int size) { void *taosAllocateQitem(int size) {
STaosQnode *pNode = (STaosQnode *)calloc(sizeof(STaosQnode) + size, 1); STaosQnode *pNode = (STaosQnode *)calloc(sizeof(STaosQnode) + size, 1);
if (pNode == NULL) return NULL; if (pNode == NULL) return NULL;
uTrace("item:%p, node:%p is allocated", pNode->item, pNode);
return (void *)pNode->item; return (void *)pNode->item;
} }
void taosFreeQitem(void *param) { void taosFreeQitem(void *param) {
if (param == NULL) return; if (param == NULL) return;
uTrace("item:%p is freed", param);
char *temp = (char *)param; char *temp = (char *)param;
temp -= sizeof(STaosQnode); temp -= sizeof(STaosQnode);
uTrace("item:%p, node:%p is freed", param, temp);
free(temp); free(temp);
} }

View File

@ -5,6 +5,7 @@
* it under the terms of the GNU Affero General Public License, version 3 * it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation. * or later ("AGPL"), as published by the Free Software Foundation.
* *
*
* This program is distributed in the hope that it will be useful, but WITHOUT * This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. * FITNESS FOR A PARTICULAR PURPOSE.
@ -238,7 +239,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
// if the new key is greater than the maximum key of skip list, push back this node at the end of skip list // if the new key is greater than the maximum key of skip list, push back this node at the end of skip list
char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode);
if (pSkipList->size == 0 || pSkipList->comparFn(pSkipList->lastKey, newDatakey) < 0) { if (pSkipList->size == 0 || pSkipList->comparFn(SL_GET_SL_MAX_KEY(pSkipList), newDatakey) < 0) {
return tSkipListPushBack(pSkipList, pNode); return tSkipListPushBack(pSkipList, pNode);
} }
@ -498,7 +499,7 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) {
if (pSkipList->lock) { if (pSkipList->lock) {
pthread_rwlock_wrlock(pSkipList->lock); pthread_rwlock_wrlock(pSkipList->lock);
} }
for (int32_t j = level - 1; j >= 0; --j) { for (int32_t j = level - 1; j >= 0; --j) {
SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pNode, j); SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pNode, j);
SSkipListNode* next = SL_GET_FORWARD_POINTER(pNode, j); SSkipListNode* next = SL_GET_FORWARD_POINTER(pNode, j);
@ -699,8 +700,6 @@ SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode) {
SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode;
} }
pSkipList->lastKey = SL_GET_NODE_KEY(pSkipList, pNode);
atomic_add_fetch_32(&pSkipList->size, 1); atomic_add_fetch_32(&pSkipList->size, 1);
if (pSkipList->lock) { if (pSkipList->lock) {
pthread_rwlock_unlock(pSkipList->lock); pthread_rwlock_unlock(pSkipList->lock);

View File

@ -383,10 +383,15 @@ int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
return -1; return -1;
} }
if (taosKeepTcpAlive(sockFd) < 0) return -1; if (taosKeepTcpAlive(sockFd) < 0) {
uError("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno));
close(sockFd);
return -1;
}
if (listen(sockFd, 10) < 0) { if (listen(sockFd, 10) < 0) {
uError("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); uError("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno));
close(sockFd);
return -1; return -1;
} }

View File

@ -161,11 +161,17 @@ int32_t vnodeDrop(int32_t vgId) {
int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) { int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
SVnodeObj *pVnode = param; SVnodeObj *pVnode = param;
if (pVnode->status != TAOS_VN_STATUS_READY) // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS
return TSDB_CODE_VND_INVALID_STATUS; // cfgVersion can be corrected by status msg
if (pVnode->status != TAOS_VN_STATUS_READY) {
vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId);
return TSDB_CODE_SUCCESS;
}
if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) // the vnode may always fail to synchronize because of it in low cfgVersion
return TSDB_CODE_VND_NOT_SYNCED; // so cannot use the following codes
// if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED)
// return TSDB_CODE_VND_NOT_SYNCED;
pVnode->status = TAOS_VN_STATUS_UPDATING; pVnode->status = TAOS_VN_STATUS_UPDATING;
@ -415,13 +421,19 @@ void *vnodeGetWal(void *pVnode) {
} }
static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
int64_t totalStorage = 0;
int64_t compStorage = 0;
int64_t pointsWritten = 0;
if (pVnode->status != TAOS_VN_STATUS_READY) return; if (pVnode->status != TAOS_VN_STATUS_READY) return;
if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return;
if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) return;
if (pVnode->tsdb == NULL) return;
int64_t totalStorage, compStorage, pointsWritten = 0; // still need report status when unsynced
tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) {
} else if (pVnode->tsdb == NULL) {
} else {
tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage);
}
SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++];
pLoad->vgId = htonl(pVnode->vgId); pLoad->vgId = htonl(pVnode->vgId);

View File

@ -61,7 +61,7 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
} }
static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
void * pCont = pReadMsg->pCont; void *pCont = pReadMsg->pCont;
int32_t contLen = pReadMsg->contLen; int32_t contLen = pReadMsg->contLen;
SRspRet *pRet = &pReadMsg->rspRet; SRspRet *pRet = &pReadMsg->rspRet;
@ -74,19 +74,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
killQueryMsg->free = htons(killQueryMsg->free); killQueryMsg->free = htons(killQueryMsg->free);
killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle); killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
void* handle = NULL; vWarn("QInfo:%p connection %p broken, kill query", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
if ((void**) killQueryMsg->qhandle != NULL) {
handle = *(void**) killQueryMsg->qhandle;
}
vWarn("QInfo:%p connection %p broken, kill query", handle, pReadMsg->rpcMsg.handle);
assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1); assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1);
void** qhandle = qAcquireQInfo(pVnode->qMgmt, (void**) killQueryMsg->qhandle); void** qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) killQueryMsg->qhandle);
if (qhandle == NULL || *qhandle == NULL) { if (qhandle == NULL || *qhandle == NULL) {
vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
} else { } else {
assert(qhandle == (void**) killQueryMsg->qhandle); assert(*qhandle == (void*) killQueryMsg->qhandle);
qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true); qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true);
} }
@ -94,10 +89,10 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
} }
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
qinfo_t pQInfo = NULL;
void** handle = NULL; void** handle = NULL;
if (contLen != 0) { if (contLen != 0) {
qinfo_t pQInfo = NULL;
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, NULL, &pQInfo); code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, NULL, &pQInfo);
SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
@ -110,48 +105,49 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
// current connect is broken // current connect is broken
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
handle = qRegisterQInfo(pVnode->qMgmt, pQInfo); handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo);
if (handle == NULL) { // failed to register qhandle if (handle == NULL) { // failed to register qhandle
pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE; pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE;
qKillQuery(pQInfo); qKillQuery(pQInfo);
qKillQuery(pQInfo); qKillQuery(pQInfo);
pQInfo = NULL;
} else { } else {
assert(*handle == pQInfo); assert(*handle == pQInfo);
pRsp->qhandle = htobe64((uint64_t) (handle)); pRsp->qhandle = htobe64((uint64_t) pQInfo);
} }
if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { pQInfo = NULL;
vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, pQInfo, pReadMsg->rpcMsg.handle); if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
// NOTE: there two refcount, needs to kill twice // NOTE: there two refcount, needs to kill twice
// query has not been put into qhandle pool, kill it directly. // query has not been put into qhandle pool, kill it directly.
qKillQuery(pQInfo); qKillQuery(*handle);
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
return pRsp->code; return pRsp->code;
} }
} else { } else {
assert(pQInfo == NULL); assert(pQInfo == NULL);
} }
if (handle != NULL) {
dnodePutItemIntoReadQueue(pVnode, *handle);
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
}
vDebug("vgId:%d, QInfo:%p, dnode query msg disposed", vgId, pQInfo); vDebug("vgId:%d, QInfo:%p, dnode query msg disposed", vgId, pQInfo);
} else { } else {
assert(pCont != NULL); assert(pCont != NULL);
pQInfo = *(void**)(pCont); handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont);
handle = pCont; if (handle == NULL) {
code = TSDB_CODE_VND_ACTION_IN_PROGRESS; vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle);
code = TSDB_CODE_QRY_INVALID_QHANDLE;
vDebug("vgId:%d, QInfo:%p, dnode query msg in progress", pVnode->vgId, pQInfo); } else {
} vDebug("vgId:%d, QInfo:%p, dnode query msg in progress", pVnode->vgId, (void*) pCont);
code = TSDB_CODE_VND_ACTION_IN_PROGRESS;
if (pQInfo != NULL) { qTableQuery(*handle); // do execute query
qTableQuery(pQInfo); // do execute query }
assert(handle != NULL);
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
} }
return code; return code;
} }
@ -160,57 +156,64 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
SRspRet *pRet = &pReadMsg->rspRet; SRspRet *pRet = &pReadMsg->rspRet;
SRetrieveTableMsg *pRetrieve = pCont; SRetrieveTableMsg *pRetrieve = pCont;
void **pQInfo = (void*) htobe64(pRetrieve->qhandle); pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
pRetrieve->free = htons(pRetrieve->free); pRetrieve->free = htons(pRetrieve->free);
vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, *pQInfo); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, *(void**) pRetrieve->qhandle);
memset(pRet, 0, sizeof(SRspRet)); memset(pRet, 0, sizeof(SRspRet));
int32_t ret = 0;
void** handle = qAcquireQInfo(pVnode->qMgmt, pQInfo); int32_t code = TSDB_CODE_SUCCESS;
if (handle == NULL || handle != pQInfo) { void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle);
ret = TSDB_CODE_QRY_INVALID_QHANDLE; if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) {
code = TSDB_CODE_QRY_INVALID_QHANDLE;
vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle);
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
pRet->len = sizeof(SRetrieveTableRsp);
memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
SRetrieveTableRsp* pRsp = pRet->rsp;
pRsp->numOfRows = 0;
pRsp->useconds = 0;
pRsp->completed = true;
return code;
} }
if (pRetrieve->free == 1) { if (pRetrieve->free == 1) {
if (ret == TSDB_CODE_SUCCESS) { vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle);
vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pQInfo); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
pRet->len = sizeof(SRetrieveTableRsp); pRet->len = sizeof(SRetrieveTableRsp);
memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
SRetrieveTableRsp* pRsp = pRet->rsp; SRetrieveTableRsp* pRsp = pRet->rsp;
pRsp->numOfRows = 0; pRsp->numOfRows = 0;
pRsp->completed = true; pRsp->completed = true;
pRsp->useconds = 0; pRsp->useconds = 0;
} else { // todo handle error
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); return code;
}
return ret;
} }
int32_t code = qRetrieveQueryResultInfo(*pQInfo); bool freeHandle = true;
if (code != TSDB_CODE_SUCCESS || ret != TSDB_CODE_SUCCESS) { code = qRetrieveQueryResultInfo(*handle);
//TODO if (code != TSDB_CODE_SUCCESS) {
//TODO handle malloc failure
pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
} else { // if failed to dump result, free qhandle immediately
} else { if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len)) == TSDB_CODE_SUCCESS) {
// todo check code and handle error in build result set if (qHasMoreResultsToRetrieve(*handle)) {
code = qDumpRetrieveResult(*pQInfo, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len); dnodePutItemIntoReadQueue(pVnode, *handle);
pRet->qhandle = *handle;
if (qHasMoreResultsToRetrieve(*handle)) { freeHandle = false;
dnodePutItemIntoReadQueue(pVnode, handle); }
pRet->qhandle = handle;
code = TSDB_CODE_SUCCESS;
} else { // no further execution invoked, release the ref to vnode
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
} }
} }
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle);
return code; return code;
} }

View File

@ -49,19 +49,26 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) {
SVnodeObj *pVnode = (SVnodeObj *)param1; SVnodeObj *pVnode = (SVnodeObj *)param1;
SWalHead *pHead = param2; SWalHead *pHead = param2;
if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) {
vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]);
return TSDB_CODE_VND_MSG_NOT_PROCESSED; return TSDB_CODE_VND_MSG_NOT_PROCESSED;
}
if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) {
vDebug("vgId:%d, msgType:%s not processed, no write auth", pVnode->vgId, taosMsg[pHead->msgType]);
return TSDB_CODE_VND_NO_WRITE_AUTH; return TSDB_CODE_VND_NO_WRITE_AUTH;
} }
if (pHead->version == 0) { // from client or CQ if (pHead->version == 0) { // from client or CQ
if (pVnode->status != TAOS_VN_STATUS_READY) if (pVnode->status != TAOS_VN_STATUS_READY) {
vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->status);
return TSDB_CODE_VND_INVALID_STATUS; // it may be in deleting or closing state return TSDB_CODE_VND_INVALID_STATUS; // it may be in deleting or closing state
}
if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->syncCfg.replica, pVnode->role);
return TSDB_CODE_RPC_NOT_READY; return TSDB_CODE_RPC_NOT_READY;
}
// assign version // assign version
pVnode->version++; pVnode->version++;

View File

@ -78,5 +78,6 @@ class TDTestCase:
tdSql.close() tdSql.close()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase())

View File

@ -40,6 +40,11 @@ class TDTestCase:
ret = tdSql.query('select server_status() as result') ret = tdSql.query('select server_status() as result')
tdSql.checkData(0, 0, 1) tdSql.checkData(0, 0, 1)
ret = tdSql.query('show dnodes')
ret = tdSql.execute('alter dnode "%s" debugFlag 135' % tdSql.getData(0,1))
tdLog.info('alter dnode "%s" debugFlag 135 -> ret: %d' % (tdSql.getData(0, 1), ret))
def stop(self): def stop(self):
tdSql.close() tdSql.close()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)

File diff suppressed because it is too large Load Diff

View File

@ -38,4 +38,4 @@ export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib
# Now we are all let, and let's see if we can find a crash. Note we pass all params # Now we are all let, and let's see if we can find a crash. Note we pass all params
./crash_gen.py $@ python3 ./crash_gen.py $@

View File

@ -1,7 +1,6 @@
#!/bin/bash #!/bin/bash
ulimit -c unlimited ulimit -c unlimited
python3 ./test.py -f client/client.py
python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/basic.py
python3 ./test.py -f insert/int.py python3 ./test.py -f insert/int.py
python3 ./test.py -f insert/float.py python3 ./test.py -f insert/float.py
@ -146,8 +145,14 @@ python3 ./test.py -f query/queryJoin.py
python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/select_last_crash.py
#stream #stream
python3 ./test.py -f stream/metric_1.py
python3 ./test.py -f stream/new.py
python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py python3 ./test.py -f stream/stream2.py
python3 ./test.py -f stream/parser.py
#alter table #alter table
python3 ./test.py -f alter/alter_table_crash.py python3 ./test.py -f alter/alter_table_crash.py
# client
python3 ./test.py -f client/client.py

View File

@ -67,7 +67,7 @@ class DBWriteNonStop:
self.cursor.execute( self.cursor.execute(
"select first(ts), last(ts), min(speed), max(speed), avg(speed), count(*) from st") "select first(ts), last(ts), min(speed), max(speed), avg(speed), count(*) from st")
data = self.cursor.fetchall() data = self.cursor.fetchall()
end = datetime.now() end = datetime.now()
self.writeDataToCSVFile(data, (end - start).seconds) self.writeDataToCSVFile(data, (end - start).seconds)
time.sleep(.001) time.sleep(.001)
@ -75,8 +75,9 @@ class DBWriteNonStop:
self.cursor.close() self.cursor.close()
self.conn.close() self.conn.close()
test = DBWriteNonStop() test = DBWriteNonStop()
test.connectDB() test.connectDB()
test.createTable() test.createTable()
test.insertData() test.insertData()
test.closeConn() test.closeConn()

View File

@ -89,17 +89,25 @@ class TDTestCase:
tdSql.checkRows(101) tdSql.checkRows(101)
# range for int type on column # range for int type on column
tdSql.query("select * from st%s where num > 50 and num < 100" % curType) tdSql.query(
tdSql.checkRows(49) "select * from st%s where num > 50 and num < 100" %
curType)
tdSql.checkRows(49)
tdSql.query("select * from st%s where num >= 50 and num < 100" % curType) tdSql.query(
tdSql.checkRows(50) "select * from st%s where num >= 50 and num < 100" %
curType)
tdSql.checkRows(50)
tdSql.query("select * from st%s where num > 50 and num <= 100" % curType) tdSql.query(
tdSql.checkRows(50) "select * from st%s where num > 50 and num <= 100" %
curType)
tdSql.checkRows(50)
tdSql.query("select * from st%s where num >= 50 and num <= 100" % curType) tdSql.query(
tdSql.checkRows(51) "select * from st%s where num >= 50 and num <= 100" %
curType)
tdSql.checkRows(51)
# > for int type on tag # > for int type on tag
tdSql.query("select * from st%s where id > 5" % curType) tdSql.query("select * from st%s where id > 5" % curType)
@ -135,16 +143,22 @@ class TDTestCase:
# range for int type on tag # range for int type on tag
tdSql.query("select * from st%s where id > 5 and id < 7" % curType) tdSql.query("select * from st%s where id > 5 and id < 7" % curType)
tdSql.checkRows(10) tdSql.checkRows(10)
tdSql.query("select * from st%s where id >= 5 and id < 7" % curType) tdSql.query(
tdSql.checkRows(20) "select * from st%s where id >= 5 and id < 7" %
curType)
tdSql.checkRows(20)
tdSql.query("select * from st%s where id > 5 and id <= 7" % curType) tdSql.query(
tdSql.checkRows(20) "select * from st%s where id > 5 and id <= 7" %
curType)
tdSql.checkRows(20)
tdSql.query("select * from st%s where id >= 5 and id <= 7" % curType) tdSql.query(
tdSql.checkRows(30) "select * from st%s where id >= 5 and id <= 7" %
curType)
tdSql.checkRows(30)
print( print(
"======= Verify filter for %s type finished =========" % "======= Verify filter for %s type finished =========" %

View File

@ -77,7 +77,7 @@ class TDTestCase:
# join queries # join queries
tdSql.query( tdSql.query(
"select * from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") "select * from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
tdSql.checkRows(6) tdSql.checkRows(6)
tdSql.error( tdSql.error(
"select ts, pressure, temperature, id, dscrption from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") "select ts, pressure, temperature, id, dscrption from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
@ -108,7 +108,7 @@ class TDTestCase:
tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
tdSql.checkRows(6) tdSql.checkRows(6)
tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id")
tdSql.checkRows(6) tdSql.checkRows(6)

View File

@ -55,9 +55,9 @@ class MetadataQuery:
def createTablesAndInsertData(self, threadID): def createTablesAndInsertData(self, threadID):
cursor = self.connectDB() cursor = self.connectDB()
cursor.execute("use test") cursor.execute("use test")
tablesPerThread = int (self.tables / self.numOfTherads) tablesPerThread = int(self.tables / self.numOfTherads)
base = threadID * tablesPerThread base = threadID * tablesPerThread
for i in range(tablesPerThread): for i in range(tablesPerThread):
cursor.execute( cursor.execute(
@ -68,24 +68,72 @@ class MetadataQuery:
%d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d',
%d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d',
%d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')''' % %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')''' %
(base + i + 1, (base + i + 1, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 100, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 10000, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 1000000, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 100000000, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 100 * 1.1, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100)) 100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100))
cursor.execute( cursor.execute(
"insert into t%d values(%d, 1) (%d, 2) (%d, 3) (%d, 4) (%d, 5)" % "insert into t%d values(%d, 1) (%d, 2) (%d, 3) (%d, 4) (%d, 5)" %
(base + i + 1, self.ts + 1, self.ts + 2, self.ts + 3, self.ts + 4, self.ts + 5)) (base + i + 1, self.ts + 1, self.ts + 2, self.ts + 3, self.ts + 4, self.ts + 5))
cursor.close() cursor.close()
def queryData(self, query): def queryData(self, query):
cursor = self.connectDB() cursor = self.connectDB()
cursor.execute("use test") cursor.execute("use test")
print("================= query tag data =================") print("================= query tag data =================")
startTime = datetime.now() startTime = datetime.now()
cursor.execute(query) cursor.execute(query)
cursor.fetchall() cursor.fetchall()
@ -107,15 +155,15 @@ if __name__ == '__main__':
print( print(
"================= Create %d tables and insert %d records into each table =================" % "================= Create %d tables and insert %d records into each table =================" %
(t.tables, t.records)) (t.tables, t.records))
startTime = datetime.now() startTime = datetime.now()
threads = [] threads = []
for i in range(t.numOfTherads): for i in range(t.numOfTherads):
thread = threading.Thread( thread = threading.Thread(
target=t.createTablesAndInsertData, args=(i,)) target=t.createTablesAndInsertData, args=(i,))
thread.start() thread.start()
threads.append(thread) threads.append(thread)
for th in threads: for th in threads:
th.join() th.join()
endTime = datetime.now() endTime = datetime.now()

View File

@ -19,6 +19,7 @@ import time
from datetime import datetime from datetime import datetime
import numpy as np import numpy as np
class MyThread(threading.Thread): class MyThread(threading.Thread):
def __init__(self, func, args=()): def __init__(self, func, args=()):
@ -35,17 +36,23 @@ class MyThread(threading.Thread):
except Exception: except Exception:
return None return None
class MetadataQuery: class MetadataQuery:
def initConnection(self): def initConnection(self):
self.tables = 100 self.tables = 100
self.records = 10 self.records = 10
self.numOfTherads =5 self.numOfTherads = 5
self.ts = 1537146000000 self.ts = 1537146000000
self.host = "127.0.0.1" self.host = "127.0.0.1"
self.user = "root" self.user = "root"
self.password = "taosdata" self.password = "taosdata"
self.config = "/etc/taos" self.config = "/etc/taos"
self.conn = taos.connect( self.host, self.user, self.password, self.config) self.conn = taos.connect(
self.host,
self.user,
self.password,
self.config)
def connectDB(self): def connectDB(self):
return self.conn.cursor() return self.conn.cursor()
@ -69,7 +76,7 @@ class MetadataQuery:
cursor.execute("use test") cursor.execute("use test")
base = threadID * self.tables base = threadID * self.tables
tablesPerThread = int (self.tables / self.numOfTherads) tablesPerThread = int(self.tables / self.numOfTherads)
for i in range(tablesPerThread): for i in range(tablesPerThread):
cursor.execute( cursor.execute(
'''create table t%d using meters tags( '''create table t%d using meters tags(
@ -79,20 +86,69 @@ class MetadataQuery:
%d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d',
%d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d',
%d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')''' % %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')''' %
(base + i + 1, (base + i + 1, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 100, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 10000, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 1000000, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 100000000, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, 100 * 1.1, (base + i) %
(base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100)) 100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100, (base + i) %
100, (base + i) %
10000, (base + i) %
1000000, (base + i) %
100000000, (base + i) %
100 * 1.1, (base + i) %
100 * 2.3, (base + i) %
2, (base + i) %
100, (base + i) %
100))
for j in range(self.records): for j in range(self.records):
cursor.execute( cursor.execute(
"insert into t%d values(%d, %d)" % "insert into t%d values(%d, %d)" %
(base + i + 1, self.ts + j, j)) (base + i + 1, self.ts + j, j))
cursor.close() cursor.close()
def queryWithTagId(self, threadId, tagId, queryNum):
print("---------thread%d start-----------"%threadId) def queryWithTagId(self, threadId, tagId, queryNum):
print("---------thread%d start-----------" % threadId)
query = '''select tgcol1, tgcol2, tgcol3, tgcol4, tgcol5, tgcol6, tgcol7, tgcol8, tgcol9, query = '''select tgcol1, tgcol2, tgcol3, tgcol4, tgcol5, tgcol6, tgcol7, tgcol8, tgcol9,
tgcol10, tgcol11, tgcol12, tgcol13, tgcol14, tgcol15, tgcol16, tgcol17, tgcol18, tgcol10, tgcol11, tgcol12, tgcol13, tgcol14, tgcol15, tgcol16, tgcol17, tgcol18,
tgcol19, tgcol20, tgcol21, tgcol22, tgcol23, tgcol24, tgcol25, tgcol26, tgcol27, tgcol19, tgcol20, tgcol21, tgcol22, tgcol23, tgcol24, tgcol25, tgcol26, tgcol27,
@ -103,18 +159,19 @@ class MetadataQuery:
latancy = [] latancy = []
cursor = self.connectDB() cursor = self.connectDB()
cursor.execute("use test") cursor.execute("use test")
for i in range(queryNum): for i in range(queryNum):
startTime = time.time() startTime = time.time()
cursor.execute(query.format(id = tagId, condition = i)) cursor.execute(query.format(id=tagId, condition=i))
cursor.fetchall() cursor.fetchall()
latancy.append((time.time() - startTime)) latancy.append((time.time() - startTime))
print("---------thread%d end-----------"%threadId) print("---------thread%d end-----------" % threadId)
return latancy return latancy
def queryData(self, query): def queryData(self, query):
cursor = self.connectDB() cursor = self.connectDB()
cursor.execute("use test") cursor.execute("use test")
print("================= query tag data =================") print("================= query tag data =================")
startTime = datetime.now() startTime = datetime.now()
cursor.execute(query) cursor.execute(query)
cursor.fetchall() cursor.fetchall()
@ -124,7 +181,7 @@ class MetadataQuery:
(endTime - startTime).seconds) (endTime - startTime).seconds)
cursor.close() cursor.close()
#self.conn.close() # self.conn.close()
if __name__ == '__main__': if __name__ == '__main__':
@ -132,18 +189,33 @@ if __name__ == '__main__':
t = MetadataQuery() t = MetadataQuery()
t.initConnection() t.initConnection()
latancys = [] latancys = []
threads = [] threads = []
tagId = 1 tagId = 1
queryNum = 1000 queryNum = 1000
for i in range(t.numOfTherads): for i in range(t.numOfTherads):
thread = MyThread(t.queryWithTagId, args = (i, tagId, queryNum)) thread = MyThread(t.queryWithTagId, args=(i, tagId, queryNum))
threads.append(thread) threads.append(thread)
thread.start() thread.start()
for i in range(t.numOfTherads): for i in range(t.numOfTherads):
threads[i].join() threads[i].join()
latancys.extend(threads[i].get_result()) latancys.extend(threads[i].get_result())
print("Total query: %d"%(queryNum * t.numOfTherads)) print("Total query: %d" % (queryNum * t.numOfTherads))
print("statistic(s): mean= %f, P50 = %f, P75 = %f, P95 = %f, P99 = %f" print(
%(sum(latancys)/(queryNum * t.numOfTherads), np.percentile(latancys, 50), np.percentile(latancys, 75), np.percentile(latancys, 95), np.percentile(latancys, 99))) "statistic(s): mean= %f, P50 = %f, P75 = %f, P95 = %f, P99 = %f" %
(sum(latancys) /
(
queryNum *
t.numOfTherads),
np.percentile(
latancys,
50),
np.percentile(
latancys,
75),
np.percentile(
latancys,
95),
np.percentile(
latancys,
99)))

View File

@ -23,7 +23,6 @@ class TDTestCase:
tdLog.debug("start to execute %s" % __file__) tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor()) tdSql.init(conn.cursor())
self.rowNum = 5000 self.rowNum = 5000
self.ts = 1537146000000 self.ts = 1537146000000
@ -36,7 +35,9 @@ class TDTestCase:
"create table t1 using st tags('dev_001')") "create table t1 using st tags('dev_001')")
for i in range(self.rowNum): for i in range(self.rowNum):
tdSql.execute("insert into t1 values(%d, 'taosdata%d', %d)" % (self.ts + i, i + 1, i + 1)) tdSql.execute(
"insert into t1 values(%d, 'taosdata%d', %d)" %
(self.ts + i, i + 1, i + 1))
tdSql.query("select last(*) from st") tdSql.query("select last(*) from st")
tdSql.checkRows(1) tdSql.checkRows(1)

View File

@ -330,7 +330,6 @@ class Test (Thread):
self.q.put(-1) self.q.put(-1)
tdLog.exit("second thread failed, first thread exit too") tdLog.exit("second thread failed, first thread exit too")
elif (self.threadId == 2): elif (self.threadId == 2):
while True: while True:
self.dbEvent.wait() self.dbEvent.wait()

View File

@ -1,7 +1,6 @@
#!/bin/bash #!/bin/bash
ulimit -c unlimited ulimit -c unlimited
python3 ./test.py -f client/client.py
python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/basic.py
python3 ./test.py -f insert/int.py python3 ./test.py -f insert/int.py
python3 ./test.py -f insert/float.py python3 ./test.py -f insert/float.py
@ -138,6 +137,9 @@ python3 ./test.py -f query/filterOtherTypes.py
python3 ./test.py -f query/queryError.py python3 ./test.py -f query/queryError.py
python3 ./test.py -f query/querySort.py python3 ./test.py -f query/querySort.py
python3 ./test.py -f query/queryJoin.py python3 ./test.py -f query/queryJoin.py
python3 ./test.py -f query/filterCombo.py
python3 ./test.py -f query/queryNormal.py
python3 ./test.py -f query/select_last_crash.py
#stream #stream
python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream1.py
@ -146,4 +148,5 @@ python3 ./test.py -f stream/stream2.py
#alter table #alter table
python3 ./test.py -f alter/alter_table_crash.py python3 ./test.py -f alter/alter_table_crash.py
# client
python3 ./test.py -f client/client.py

View File

@ -1,10 +1,6 @@
#!/bin/bash #!/bin/bash
ulimit -c unlimited ulimit -c unlimited
# client
python3 ./test.py $1 -f client/client.py
python3 ./test.py $1 -s && sleep 1
# insert # insert
python3 ./test.py $1 -f insert/basic.py python3 ./test.py $1 -f insert/basic.py
python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -s && sleep 1
@ -35,3 +31,7 @@ python3 ./test.py $1 -s && sleep 1
python3 ./test.py $1 -f query/filter.py python3 ./test.py $1 -f query/filter.py
python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -s && sleep 1
# client
python3 ./test.py $1 -f client/client.py
python3 ./test.py $1 -s && sleep 1

View File

@ -0,0 +1,104 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import time
import taos
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def createFuncStream(self, expr, suffix, value):
tbname = "strm_" + suffix
tdLog.info("create stream table %s" % tbname)
tdSql.query("select %s from stb interval(1d)" % expr)
tdSql.checkData(0, 1, value)
tdSql.execute("create table %s as select %s from stb interval(1d)" % (tbname, expr))
def checkStreamData(self, suffix, value):
sql = "select * from strm_" + suffix
tdSql.waitedQuery(sql, 1, 120)
tdSql.checkData(0, 1, value)
def run(self):
tbNum = 10
rowNum = 20
tdSql.prepare()
tdLog.info("===== preparing data =====")
tdSql.execute(
"create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)")
for i in range(tbNum):
tdSql.execute("create table tb%d using stb tags(%d)" % (i, i))
for j in range(rowNum):
tdSql.execute(
"insert into tb%d values (now - %dm, %d, %d)" %
(i, 1440 - j, j, j))
time.sleep(0.1)
self.createFuncStream("count(*)", "c1", 200)
self.createFuncStream("count(tbcol)", "c2", 200)
self.createFuncStream("count(tbcol2)", "c3", 200)
self.createFuncStream("avg(tbcol)", "av", 9.5)
self.createFuncStream("sum(tbcol)", "su", 1900)
self.createFuncStream("min(tbcol)", "mi", 0)
self.createFuncStream("max(tbcol)", "ma", 19)
self.createFuncStream("first(tbcol)", "fi", 0)
self.createFuncStream("last(tbcol)", "la", 19)
#tdSql.query("select stddev(tbcol) from stb interval(1d)")
#tdSql.query("select leastsquares(tbcol, 1, 1) from stb interval(1d)")
tdSql.query("select top(tbcol, 1) from stb interval(1d)")
tdSql.query("select bottom(tbcol, 1) from stb interval(1d)")
#tdSql.query("select percentile(tbcol, 1) from stb interval(1d)")
#tdSql.query("select diff(tbcol) from stb interval(1d)")
tdSql.query("select count(tbcol) from stb where ts < now + 4m interval(1d)")
tdSql.checkData(0, 1, 200)
#tdSql.execute("create table strm_wh as select count(tbcol) from stb where ts < now + 4m interval(1d)")
self.createFuncStream("count(tbcol)", "as", 200)
tdSql.query("select count(tbcol) from stb interval(1d) group by tgcol")
tdSql.checkData(0, 1, 20)
tdSql.query("select count(tbcol) from stb where ts < now + 4m interval(1d) group by tgcol")
tdSql.checkData(0, 1, 20)
self.checkStreamData("c1", 200)
self.checkStreamData("c2", 200)
self.checkStreamData("c3", 200)
self.checkStreamData("av", 9.5)
self.checkStreamData("su", 1900)
self.checkStreamData("mi", 0)
self.checkStreamData("ma", 19)
self.checkStreamData("fi", 0)
self.checkStreamData("la", 19)
#self.checkStreamData("wh", 200)
self.checkStreamData("as", 200)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

View File

@ -0,0 +1,71 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import time
import taos
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
rowNum = 200
totalNum = 200
tdSql.prepare()
tdLog.info("=============== step1")
tdSql.execute("create table mt(ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int)")
for i in range(5):
tdSql.execute("create table tb%d using mt tags(%d)" % (i, i))
for j in range(rowNum):
tdSql.execute("insert into tb%d values(now + %ds, %d, %d)" % (i, j, j, j))
time.sleep(0.1)
tdLog.info("=============== step2")
tdSql.query("select count(*), count(tbcol), count(tbcol2) from mt interval(10s)")
tdSql.execute("create table st as select count(*), count(tbcol), count(tbcol2) from mt interval(10s)")
tdLog.info("=============== step3")
tdSql.waitedQuery("select * from st", 1, 120)
v = tdSql.getData(0, 3)
if v >= 51:
tdLog.exit("value is %d, which is larger than 51" % v)
tdLog.info("=============== step4")
for i in range(5, 10):
tdSql.execute("create table tb%d using mt tags(%d)" % (i, i))
for j in range(rowNum):
tdSql.execute("insert into tb%d values(now + %ds, %d, %d)" % (i, j, j, j))
tdLog.info("=============== step5")
tdLog.sleep(30)
tdSql.waitedQuery("select * from st order by ts desc", 1, 120)
v = tdSql.getData(0, 3)
if v <= 51:
tdLog.exit("value is %d, which is smaller than 51" % v)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

View File

@ -0,0 +1,182 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import time
import taos
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
'''
def bug2222(self):
tdSql.prepare()
tdSql.execute("create table superreal(ts timestamp, addr binary(5), val float) tags (deviceNo binary(20))")
tdSql.execute("create table real_001 using superreal tags('001')")
tdSql.execute("create table tj_001 as select sum(val) from real_001 interval(1m)")
t = datetime.datetime.now()
for i in range(60):
ts = t.strftime("%Y-%m-%d %H:%M")
t += datetime.timedelta(minutes=1)
sql = "insert into real_001 values('%s:0%d', '1', %d)" % (ts, 0, i)
for j in range(4):
sql += ",('%s:0%d', '%d', %d)" % (ts, j + 1, j + 1, i)
tdSql.execute(sql)
time.sleep(60 + random.random() * 60 - 30)
'''
def tbase300(self):
tdLog.debug("begin tbase300")
tdSql.prepare()
tdSql.execute("create table mt(ts timestamp, c1 int, c2 int) tags(t1 int)")
tdSql.execute("create table tb1 using mt tags(1)");
tdSql.execute("create table tb2 using mt tags(2)");
tdSql.execute("create table strm as select count(*), avg(c1), sum(c2), max(c1), min(c2),first(c1), last(c2) from mt interval(4s) sliding(2s)")
#tdSql.execute("create table strm as select count(*), avg(c1), sum(c2), max(c1), min(c2), first(c1) from mt interval(4s) sliding(2s)")
tdLog.sleep(10)
tdSql.execute("insert into tb2 values(now, 1, 1)");
tdSql.execute("insert into tb1 values(now, 1, 1)");
tdLog.sleep(4)
tdSql.query("select * from mt")
tdSql.query("select * from strm")
tdSql.execute("drop table tb1")
tdSql.waitedQuery("select * from strm", 1, 100)
if tdSql.queryRows < 1 or tdSql.queryRows > 2:
tdLog.exit("rows should be 1 or 2")
tdSql.execute("drop table tb2")
tdSql.execute("drop table mt")
tdSql.execute("drop table strm")
def tbase304(self):
tdLog.debug("begin tbase304")
# we cannot reset query cache in server side, as a workaround,
# set super table name to mt304, need to change back to mt later
tdSql.execute("create table mt304 (ts timestamp, c1 int) tags(t1 int, t2 int)")
tdSql.execute("create table tb1 using mt304 tags(1, 1)")
tdSql.execute("create table tb2 using mt304 tags(1, -1)")
time.sleep(0.1)
tdSql.execute("create table strm as select count(*), avg(c1) from mt304 where t2 >= 0 interval(4s) sliding(2s)")
tdSql.execute("insert into tb1 values (now,1)")
tdSql.execute("insert into tb2 values (now,2)")
tdSql.waitedQuery("select * from strm", 1, 100)
if tdSql.queryRows < 1 or tdSql.queryRows > 2:
tdLog.exit("rows should be 1 or 2")
tdSql.checkData(0, 1, 1)
tdSql.checkData(0, 2, 1.000000000)
tdSql.execute("alter table mt304 drop tag t2")
tdSql.execute("insert into tb2 values (now,2)")
tdSql.execute("insert into tb1 values (now,1)")
tdSql.query("select * from strm")
tdSql.execute("alter table mt304 add tag t2 int")
tdLog.sleep(1)
tdSql.query("select * from strm")
def wildcardFilterOnTags(self):
tdLog.debug("begin wildcardFilterOnTag")
tdSql.prepare()
tdSql.execute("create table stb (ts timestamp, c1 int, c2 binary(10)) tags(t1 binary(10))")
tdSql.execute("create table tb1 using stb tags('a1')")
tdSql.execute("create table tb2 using stb tags('b2')")
tdSql.execute("create table tb3 using stb tags('a3')")
tdSql.execute("create table strm as select count(*), avg(c1), first(c2) from stb where t1 like 'a%' interval(4s) sliding(2s)")
tdSql.query("describe strm")
tdSql.checkRows(4)
tdLog.sleep(1)
tdSql.execute("insert into tb1 values (now, 0, 'tb1')")
tdLog.sleep(4)
tdSql.execute("insert into tb2 values (now, 2, 'tb2')")
tdLog.sleep(4)
tdSql.execute("insert into tb3 values (now, 0, 'tb3')")
tdSql.waitedQuery("select * from strm", 4, 60)
tdSql.checkRows(4)
tdSql.checkData(0, 2, 0.000000000)
if tdSql.getData(0, 3) == 'tb2':
tdLog.exit("unexpected value of data03")
if tdSql.getData(1, 3) == 'tb2':
tdLog.exit("unexpected value of data13")
if tdSql.getData(2, 3) == 'tb2':
tdLog.exit("unexpected value of data23")
if tdSql.getData(3, 3) == 'tb2':
tdLog.exit("unexpected value of data33")
tdLog.info("add table tb4 to see if stream still works correctly")
# The vnode client needs to refresh metadata cache to allow strm calculate tb4's data.
# But the current refreshing frequency is every 10 min
# commented out the case below to save running time
tdSql.execute("create table tb4 using stb tags('a4')")
tdSql.execute("insert into tb4 values(now, 4, 'tb4')")
tdSql.waitedQuery("select * from strm order by ts desc", 6, 60)
tdSql.checkRows(6)
tdSql.checkData(0, 2, 4)
tdSql.checkData(0, 3, "tb4")
tdLog.info("change tag values to see if stream still works correctly")
tdSql.execute("alter table tb4 set tag t1='b4'")
tdLog.sleep(3)
tdSql.execute("insert into tb1 values (now, 1, 'tb1_a1')")
tdLog.sleep(4)
tdSql.execute("insert into tb4 values (now, -4, 'tb4_b4')")
tdSql.waitedQuery("select * from strm order by ts desc", 8, 100)
tdSql.checkRows(8)
tdSql.checkData(0, 2, 1)
tdSql.checkData(0, 3, "tb1_a1")
def datatypes(self):
tdLog.debug("begin data types")
tdSql.prepare()
tdSql.execute("create table stb3 (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(15), c6 nchar(15), c7 bool) tags(t1 int, t2 binary(15))")
tdSql.execute("create table tb0 using stb3 tags(0, 'tb0')")
tdSql.execute("create table tb1 using stb3 tags(1, 'tb1')")
tdSql.execute("create table tb2 using stb3 tags(2, 'tb2')")
tdSql.execute("create table tb3 using stb3 tags(3, 'tb3')")
tdSql.execute("create table tb4 using stb3 tags(4, 'tb4')")
tdSql.execute("create table strm0 as select count(ts), count(c1), max(c2), min(c4), first(c5), last(c6) from stb3 where ts < now + 30s interval(4s) sliding(2s)")
#tdSql.execute("create table strm0 as select count(ts), count(c1), max(c2), min(c4), first(c5) from stb where ts < now + 30s interval(4s) sliding(2s)")
tdLog.sleep(1)
tdSql.execute("insert into tb0 values (now, 0, 0, 0, 0, 'binary0', '涛思0', true) tb1 values (now, 1, 1, 1, 1, 'binary1', '涛思1', false) tb2 values (now, 2, 2, 2, 2, 'binary2', '涛思2', true) tb3 values (now, 3, 3, 3, 3, 'binary3', '涛思3', false) tb4 values (now, 4, 4, 4, 4, 'binary4', '涛思4', true) ")
tdSql.waitedQuery("select * from strm0 order by ts desc", 2, 120)
tdSql.checkRows(2)
tdSql.execute("insert into tb0 values (now, 10, 10, 10, 10, 'binary0', '涛思0', true) tb1 values (now, 11, 11, 11, 11, 'binary1', '涛思1', false) tb2 values (now, 12, 12, 12, 12, 'binary2', '涛思2', true) tb3 values (now, 13, 13, 13, 13, 'binary3', '涛思3', false) tb4 values (now, 14, 14, 14, 14, 'binary4', '涛思4', true) ")
tdSql.waitedQuery("select * from strm0 order by ts desc", 4, 120)
tdSql.checkRows(4)
def run(self):
self.tbase300()
self.tbase304()
self.wildcardFilterOnTags()
self.datatypes()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

View File

@ -55,10 +55,7 @@ class TDTestCase:
tdSql.checkRows(tbNum + 1) tdSql.checkRows(tbNum + 1)
tdLog.info("===== step3 =====") tdLog.info("===== step3 =====")
tdLog.info("sleeping 120 seconds") tdSql.waitedQuery("select * from s0", 1, 120)
time.sleep(120)
tdSql.query("select * from s0")
try: try:
tdSql.checkData(0, 1, rowNum) tdSql.checkData(0, 1, rowNum)
tdSql.checkData(0, 2, rowNum) tdSql.checkData(0, 2, rowNum)
@ -82,10 +79,7 @@ class TDTestCase:
tdSql.checkRows(tbNum + 1) tdSql.checkRows(tbNum + 1)
tdLog.info("===== step7 =====") tdLog.info("===== step7 =====")
tdLog.info("sleeping 120 seconds") tdSql.waitedQuery("select * from s0", 1, 120)
time.sleep(120)
tdSql.query("select * from s0")
try: try:
tdSql.checkData(0, 1, rowNum) tdSql.checkData(0, 1, rowNum)
tdSql.checkData(0, 2, rowNum) tdSql.checkData(0, 2, rowNum)
@ -108,10 +102,7 @@ class TDTestCase:
tdSql.checkRows(tbNum + 2) tdSql.checkRows(tbNum + 2)
tdLog.info("===== step9 =====") tdLog.info("===== step9 =====")
tdLog.info("sleeping 120 seconds") tdSql.waitedQuery("select * from s1", 1, 120)
time.sleep(120)
tdSql.query("select * from s1")
try: try:
tdSql.checkData(0, 1, rowNum * tbNum) tdSql.checkData(0, 1, rowNum * tbNum)
tdSql.checkData(0, 2, rowNum * tbNum) tdSql.checkData(0, 2, rowNum * tbNum)
@ -134,9 +125,7 @@ class TDTestCase:
tdSql.checkRows(tbNum + 2) tdSql.checkRows(tbNum + 2)
tdLog.info("===== step13 =====") tdLog.info("===== step13 =====")
tdLog.info("sleeping 120 seconds") tdSql.waitedQuery("select * from s1", 1, 120)
time.sleep(120)
tdSql.query("select * from s1")
try: try:
tdSql.checkData(0, 1, rowNum * tbNum) tdSql.checkData(0, 1, rowNum * tbNum)
tdSql.checkData(0, 2, rowNum * tbNum) tdSql.checkData(0, 2, rowNum * tbNum)

View File

@ -53,8 +53,7 @@ class TDTestCase:
tdSql.checkRows(tbNum + 1) tdSql.checkRows(tbNum + 1)
tdLog.info("===== step3 =====") tdLog.info("===== step3 =====")
time.sleep(120) tdSql.waitedQuery("select * from s0", 1, 120)
tdSql.query("select * from s0")
try: try:
tdSql.checkData(0, 1, rowNum) tdSql.checkData(0, 1, rowNum)
except Exception as e: except Exception as e:
@ -81,8 +80,7 @@ class TDTestCase:
tdLog.info(repr(e)) tdLog.info(repr(e))
tdLog.info("===== step7 =====") tdLog.info("===== step7 =====")
time.sleep(120) tdSql.waitedQuery("select * from s0", 1, 120)
tdSql.query("select * from s0")
try: try:
tdSql.checkData(0, 1, rowNum) tdSql.checkData(0, 1, rowNum)
tdSql.checkData(0, 2, rowNum) tdSql.checkData(0, 2, rowNum)
@ -107,8 +105,7 @@ class TDTestCase:
tdSql.checkRows(tbNum + 2) tdSql.checkRows(tbNum + 2)
tdLog.info("===== step9 =====") tdLog.info("===== step9 =====")
time.sleep(120) tdSql.waitedQuery("select * from s1", 1, 120)
tdSql.query("select * from s1")
try: try:
tdSql.checkData(0, 1, totalNum) tdSql.checkData(0, 1, totalNum)
tdSql.checkData(0, 2, totalNum) tdSql.checkData(0, 2, totalNum)
@ -137,8 +134,7 @@ class TDTestCase:
tdLog.info(repr(e)) tdLog.info(repr(e))
tdLog.info("===== step13 =====") tdLog.info("===== step13 =====")
time.sleep(120) tdSql.waitedQuery("select * from s1", 1, 120)
tdSql.query("select * from s1")
try: try:
tdSql.checkData(0, 1, totalNum) tdSql.checkData(0, 1, totalNum)
#tdSql.checkData(0, 2, None) #tdSql.checkData(0, 2, None)

View File

@ -190,32 +190,31 @@ class TDDnode:
"dnode:%d is deployed and configured by %s" % "dnode:%d is deployed and configured by %s" %
(self.index, self.cfgPath)) (self.index, self.cfgPath))
def start(self): def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__)) selfPath = os.path.dirname(os.path.realpath(__file__))
binPath = ""
if ("community" in selfPath): if ("community" in selfPath):
projPath = selfPath + "/../../../../" projPath = selfPath[:selfPath.find("community")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
binPath = os.path.join(root, "taosd")
break
else: else:
projPath = selfPath + "/../../../" projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
binPath = os.path.join(root, "taosd")
break
if (binPath == ""): for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root)-len("/build/bin")]
break
return buildPath
def start(self):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!") tdLog.exit("taosd not found!")
else: else:
tdLog.info("taosd found in %s" % rootRealPath) tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath + "/build/bin/taosd"
if self.deployed == 0: if self.deployed == 0:
tdLog.exit("dnode:%d is not deployed" % (self.index)) tdLog.exit("dnode:%d is not deployed" % (self.index))

View File

@ -29,10 +29,8 @@ class TDSql:
self.cursor = cursor self.cursor = cursor
if (log): if (log):
frame = inspect.stack()[1] caller = inspect.getframeinfo(inspect.stack()[1][0])
callerModule = inspect.getmodule(frame[0]) self.cursor.log(caller.filename + ".sql")
callerFilename = callerModule.__file__
self.cursor.log(callerFilename + ".sql")
def close(self): def close(self):
self.cursor.close() self.cursor.close()
@ -55,12 +53,8 @@ class TDSql:
except BaseException: except BaseException:
expectErrNotOccured = False expectErrNotOccured = False
if expectErrNotOccured: if expectErrNotOccured:
frame = inspect.stack()[1] caller = inspect.getframeinfo(inspect.stack()[1][0])
callerModule = inspect.getmodule(frame[0]) tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql))
callerFilename = callerModule.__file__
tdLog.exit(
"%s failed: sql:%s, expect error not occured" %
(callerFilename, sql))
else: else:
self.queryRows = 0 self.queryRows = 0
self.queryCols = 0 self.queryCols = 0
@ -69,75 +63,68 @@ class TDSql:
def query(self, sql): def query(self, sql):
self.sql = sql self.sql = sql
self.cursor.execute(sql) try:
self.queryResult = self.cursor.fetchall() self.cursor.execute(sql)
self.queryRows = len(self.queryResult) self.queryResult = self.cursor.fetchall()
self.queryCols = len(self.cursor.description) self.queryRows = len(self.queryResult)
# if self.queryRows == 1 and self.queryCols == 1: self.queryCols = len(self.cursor.description)
# tdLog.info("sql:%s, rows:%d cols:%d data:%s" % (self.sql, self.queryRows, self.queryCols, self.queryResult[0][0])) except Exception as e:
# else: caller = inspect.getframeinfo(inspect.stack()[1][0])
# tdLog.info("sql:%s, rows:%d cols:%d" % (self.sql, self.queryRows, self.queryCols)) args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.exit("%s(%d) failed: sql:%s, %s" % args)
return self.queryRows return self.queryRows
def waitedQuery(self, sql, expectRows, timeout):
tdLog.info("sql: %s, try to retrieve %d rows in %d seconds" % (sql, expectRows, timeout))
self.sql = sql
try:
for i in range(timeout):
self.cursor.execute(sql)
self.queryResult = self.cursor.fetchall()
self.queryRows = len(self.queryResult)
self.queryCols = len(self.cursor.description)
if self.queryRows >= expectRows:
return (self.queryRows, i)
time.sleep(1)
except Exception as e:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.exit("%s(%d) failed: sql:%s, %s" % args)
return (self.queryRows, timeout)
def checkRows(self, expectRows): def checkRows(self, expectRows):
if self.queryRows != expectRows: if self.queryRows == expectRows:
frame = inspect.stack()[1] tdLog.info("sql:%s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows))
callerModule = inspect.getmodule(frame[0]) else:
callerFilename = callerModule.__file__ caller = inspect.getframeinfo(inspect.stack()[1][0])
tdLog.exit( args = (caller.filename, caller.lineno, self.sql, self.queryRows, expectRows)
"%s failed: sql:%s, queryRows:%d != expect:%d" % tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args)
(callerFilename, self.sql, self.queryRows, expectRows))
tdLog.info("sql:%s, queryRows:%d == expect:%d" % def checkRowCol(self, row, col):
(self.sql, self.queryRows, expectRows)) caller = inspect.getframeinfo(inspect.stack()[2][0])
if row < 0:
args = (caller.filename, caller.lineno, self.sql, row)
tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args)
if col < 0:
args = (caller.filename, caller.lineno, self.sql, row)
tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args)
if row > self.queryRows:
args = (caller.filename, caller.lineno, self.sql, row, self.queryRows)
tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args)
if col > self.queryCols:
args = (caller.filename, caller.lineno, self.sql, col, self.queryCols)
tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args)
def checkDataType(self, row, col, dataType): def checkDataType(self, row, col, dataType):
frame = inspect.stack()[1] self.checkRowCol(row, col)
callerModule = inspect.getmodule(frame[0])
callerFilename = callerModule.__file__
if row < 0:
tdLog.exit(
"%s failed: sql:%s, row:%d is smaller than zero" %
(callerFilename, self.sql, row))
if col < 0:
tdLog.exit(
"%s failed: sql:%s, col:%d is smaller than zero" %
(callerFilename, self.sql, col))
if row > self.queryRows:
tdLog.exit(
"%s failed: sql:%s, row:%d is larger than queryRows:%d" %
(callerFilename, self.sql, row, self.queryRows))
if col > self.queryCols:
tdLog.exit(
"%s failed: sql:%s, col:%d is larger than queryCols:%d" %
(callerFilename, self.sql, col, self.queryCols))
return self.cursor.istype(col, dataType) return self.cursor.istype(col, dataType)
def checkData(self, row, col, data): def checkData(self, row, col, data):
frame = inspect.stack()[1] self.checkRowCol(row, col)
callerModule = inspect.getmodule(frame[0])
callerFilename = callerModule.__file__
if row < 0:
tdLog.exit(
"%s failed: sql:%s, row:%d is smaller than zero" %
(callerFilename, self.sql, row))
if col < 0:
tdLog.exit(
"%s failed: sql:%s, col:%d is smaller than zero" %
(callerFilename, self.sql, col))
if row > self.queryRows:
tdLog.exit(
"%s failed: sql:%s, row:%d is larger than queryRows:%d" %
(callerFilename, self.sql, row, self.queryRows))
if col > self.queryCols:
tdLog.exit(
"%s failed: sql:%s, col:%d is larger than queryCols:%d" %
(callerFilename, self.sql, col, self.queryCols))
if self.queryResult[row][col] != data: if self.queryResult[row][col] != data:
tdLog.exit("%s failed: sql:%s row:%d col:%d data:%s != expect:%s" % ( caller = inspect.getframeinfo(inspect.stack()[1][0])
callerFilename, self.sql, row, col, self.queryResult[row][col], data)) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data)
tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args)
if data is None: if data is None:
tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" %
@ -153,26 +140,7 @@ class TDSql:
(self.sql, row, col, self.queryResult[row][col], data)) (self.sql, row, col, self.queryResult[row][col], data))
def getData(self, row, col): def getData(self, row, col):
frame = inspect.stack()[1] self.checkRowCol(row, col)
callerModule = inspect.getmodule(frame[0])
callerFilename = callerModule.__file__
if row < 0:
tdLog.exit(
"%s failed: sql:%s, row:%d is smaller than zero" %
(callerFilename, self.sql, row))
if col < 0:
tdLog.exit(
"%s failed: sql:%s, col:%d is smaller than zero" %
(callerFilename, self.sql, col))
if row > self.queryRows:
tdLog.exit(
"%s failed: sql:%s, row:%d is larger than queryRows:%d" %
(callerFilename, self.sql, row, self.queryRows))
if col > self.queryCols:
tdLog.exit(
"%s failed: sql:%s, col:%d is larger than queryCols:%d" %
(callerFilename, self.sql, col, self.queryCols))
return self.queryResult[row][col] return self.queryResult[row][col]
def executeTimes(self, sql, times): def executeTimes(self, sql, times):
@ -185,20 +153,21 @@ class TDSql:
def execute(self, sql): def execute(self, sql):
self.sql = sql self.sql = sql
self.affectedRows = self.cursor.execute(sql) try:
self.affectedRows = self.cursor.execute(sql)
except Exception as e:
caller = inspect.getframeinfo(inspect.stack()[1][0])
args = (caller.filename, caller.lineno, sql, repr(e))
tdLog.exit("%s(%d) failed: sql:%s, %s" % args)
return self.affectedRows return self.affectedRows
def checkAffectedRows(self, expectAffectedRows): def checkAffectedRows(self, expectAffectedRows):
if self.affectedRows != expectAffectedRows: if self.affectedRows != expectAffectedRows:
frame = inspect.stack()[1] caller = inspect.getframeinfo(inspect.stack()[1][0])
callerModule = inspect.getmodule(frame[0]) args = (caller.filename, caller.lineno, self.sql, self.affectedRows, expectAffectedRows)
callerFilename = callerModule.__file__ tdLog.exit("%s(%d) failed: sql:%s, affectedRows:%d != expect:%d" % args)
tdLog.exit( tdLog.info("sql:%s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows))
"%s failed: sql:%s, affectedRows:%d != expect:%d" %
(callerFilename, self.sql, self.affectedRows, expectAffectedRows))
tdLog.info("sql:%s, affectedRows:%d == expect:%d" %
(self.sql, self.affectedRows, expectAffectedRows))
tdSql = TDSql() tdSql = TDSql()

View File

@ -68,7 +68,7 @@ endi
if $data01 != 1 then if $data01 != 1 then
return -1 return -1
endi endi
if $data02 != 1 then if $data02 != NULL then
return -1 return -1
endi endi
@ -80,7 +80,7 @@ endi
if $data01 != 1 then if $data01 != 1 then
return -1 return -1
endi endi
if $data02 != 1 then if $data02 != NULL then
return -1 return -1
endi endi

View File

@ -843,6 +843,14 @@ if $data81 != 4 then
return -1 return -1
endi endi
# desc fill query
print desc fill query
sql select count(*) from m_fl_tb0 where ts>='2018-9-17 9:0:0' and ts<='2018-9-17 9:11:00' interval(1m) fill(value,10) order by ts desc;
if $rows != 12 then
return -1
endi
print =============== clear print =============== clear
sql drop database $db sql drop database $db
sql show databases sql show databases

File diff suppressed because it is too large Load Diff

View File

@ -422,4 +422,63 @@ if $data97 != @group_tb0@ then
return -1 return -1
endi endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
#=========================== group by multi tags ======================
sql create table st (ts timestamp, c int) tags (t1 int, t2 int, t3 int, t4 int);
sql create table t1 using st tags(1, 1, 1, 1);
sql create table t2 using st tags(1, 2, 2, 2);
sql insert into t1 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ;
sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ;
sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ;
sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ;
sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2;
if $rows != 40 then
return -1
endi
if $data01 != 1.000000000 then
return -1
endi
if $data02 != t1 then
return -1
endi
if $data03 != 1 then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data11 != 1.000000000 then
return -1
endi
if $data12 != t1 then
return -1
endi
if $data13 != 1 then
return -1
endi
if $data14 != 1 then
return -1
endi
sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1;
if $rows != 2 then
return -1
endi
if $data11 != 1.000000000 then
return -1
endi
if $data12 != t2 then
return -1
endi
if $data13 != 1 then
return -1
endi
if $data14 != 2 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -40,7 +40,7 @@ while $x < $rowNum
endw endw
print ====== tables created print ====== tables created
sleep 60000 sleep 6000
$ts = $ts0 + $delta $ts = $ts0 + $delta
$ts = $ts + 1 $ts = $ts + 1

View File

@ -39,7 +39,7 @@ while $x < $rowNum
endw endw
print ====== tables created print ====== tables created
sleep 60000 sleep 6000
$ts = $ts0 + $delta $ts = $ts0 + $delta
$ts = $ts + 1 $ts = $ts + 1

View File

@ -39,7 +39,7 @@ while $x < $rowNum
endw endw
print ====== tables created print ====== tables created
sleep 60000 sleep 6000
$ts = $ts + 1 $ts = $ts + 1
sql insert into $tb values ( $ts , -1, -1, -1, -1, -1) sql insert into $tb values ( $ts , -1, -1, -1, -1, -1)
@ -47,7 +47,7 @@ $ts = $ts0 + $delta
$ts = $ts + 1 $ts = $ts + 1
sql import into $tb values ( $ts , -2, -2, -2, -2, -2) sql import into $tb values ( $ts , -2, -2, -2, -2, -2)
sleep 60000 sleep 6000
sql show databases sql show databases

View File

@ -210,6 +210,11 @@ if $data10 != @70-01-01 08:01:40.200@ then
return -1 return -1
endi endi
print data06 = $data06
print data07 = $data07
print data08 = $data08
print data00 = $data00
if $data07 != 0 then if $data07 != 0 then
return -1 return -1
endi endi
@ -255,14 +260,17 @@ endi
print 3 print 3
#agg + where condition #agg + where condition
sql select count(join_tb1.c3), count(join_tb0.ts) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts <= 100002 and join_tb0.c7 = true; sql select count(join_tb1.c3), count(join_tb0.ts) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts <= 100002 and join_tb0.c7 = true;
if $rows != 1 then
$val = 2
if $data00 != $val then
print expect 2, actaul: $data00
return -1 return -1
endi endi
if $data01 != $val then print $data00
if $data00 != 2 then
return -1
endi
if $data01 != 2 then
return -1 return -1
endi endi
@ -412,7 +420,7 @@ endi
#======================limit offset=================================== #======================limit offset===================================
# tag values not int # tag values not int
sql_error select count(*) from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t2=join_mt1.t2; #!!!!! sql_error select count(*) from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t2=join_mt1.t2;
# tag type not identical # tag type not identical
sql_error select count(*) from join_mt0, join_mt1 where join_mt1.t2 = join_mt0.t1 and join_mt1.ts=join_mt0.ts; sql_error select count(*) from join_mt0, join_mt1 where join_mt1.t2 = join_mt0.t1 and join_mt1.ts=join_mt0.ts;
@ -447,4 +455,15 @@ sql insert into um2 using m2 tags(9) values(1000001, 10)(2000000, 20);
sql_error select count(*) from m1,m2 where m1.a=m2.a and m1.ts=m2.ts; sql_error select count(*) from m1,m2 where m1.a=m2.a and m1.ts=m2.ts;
#empty table join test, add for no result join test
sql create database ux1;
sql use ux1;
sql create table m1(ts timestamp, k int) tags(a binary(12), b int);
sql create table tm0 using m1 tags('abc', 1);
sql create table m2(ts timestamp, k int) tags(a int, b binary(12));
sql create table tm2 using m2 tags(2, 'abc');
sql select count(*) from tm0, tm2 where tm0.ts=tm2.ts;
sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a
sql drop database ux1;
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -327,3 +327,7 @@ endi
if $data98 != 9 then if $data98 != 9 then
return -1 return -1
endi endi
#add one more test case
sql select max(c1), last(c8) from lm2_db0.lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(linear) limit 10 offset 4089;"

View File

@ -636,6 +636,15 @@ if $data00 != $data01 then
return -1 return -1
endi endi
sql select first(ts), ts from select_tags_tb1
if $row != 1 then
return -1
endi
if $data01 != @70-01-01 08:01:50.001@ then
return -1
endi
print ======= selectivity + tags + group by + tags + filter + interval ================ print ======= selectivity + tags + group by + tags + filter + interval ================
sql select first(c1), t2, t1, tbname from select_tags_mt0 where c1<=2 interval(1d) group by tbname; sql select first(c1), t2, t1, tbname from select_tags_mt0 where c1<=2 interval(1d) group by tbname;
if $row != 3 then if $row != 3 then

Some files were not shown because too many files have changed in this diff Show More