xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

This commit is contained in:
Shengliang Guan 2020-05-25 03:09:47 +00:00
parent 9c3ddab8c5 0f69f280ba
commit 5ab5dcea68
29 changed files with 439 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/client/src/tscFunctionImpl.c
View File

@ -396,10 +396,6 @@ static void function_finalizer(SQLFunctionCtx *pCtx) {
doFinalizer(pCtx);
}
static bool usePreVal(SQLFunctionCtx *pCtx) {
return pCtx->preAggVals.isSet && pCtx->size == pCtx->preAggVals.size;
}
/*
* count function does need the finalize, if data is missing, the default value, which is 0, is used
* count function does not use the pCtx->interResBuf to keep the intermediate buffer
@ -412,7 +408,7 @@ static void count_function(SQLFunctionCtx *pCtx) {
* 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->preAggVals.isSet == true;
* 3. for primary key column, pCtx->hasNull always be false, pCtx->preAggVals.isSet == false;
*/
if (usePreVal(pCtx)) {
if (pCtx->preAggVals.isSet) {
numOfElem = pCtx->size - pCtx->preAggVals.statis.numOfNull;
} else {
if (pCtx->hasNull) {
@ -537,7 +533,7 @@ static void do_sum(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
// Only the pre-computing information loaded and actual data does not loaded
if (pCtx->preAggVals.isSet && pCtx->preAggVals.size == pCtx->size) {
if (pCtx->preAggVals.isSet) {
notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull);
@ -768,7 +764,7 @@ static void avg_function(SQLFunctionCtx *pCtx) {
SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf;
double * pVal = &pAvgInfo->sum;
if (usePreVal(pCtx)) {
if (pCtx->preAggVals.isSet) {
// Pre-aggregation
notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
assert(notNullElems >= 0);
@ -932,7 +928,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) {
static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) {
// data in current data block are qualified to the query
if (usePreVal(pCtx)) {
if (pCtx->preAggVals.isSet) {
*notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
assert(*notNullElems >= 0);
@ -947,6 +943,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
index = pCtx->preAggVals.statis.maxIndex;
}
TSKEY key = TSKEY_INITIAL_VAL;
if (pCtx->ptsList != NULL) {
/**
* NOTE: work around the bug caused by invalid pre-calculated function.
* Here the selectivity + ts will not return correct value.
@ -957,7 +955,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
index = 0;
}
TSKEY key = pCtx->ptsList[index];
key = pCtx->ptsList[index];
}
if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) {
int64_t val = GET_INT64_VAL(tval);
@ -2913,10 +2912,6 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
}
static void date_col_output_function(SQLFunctionCtx *pCtx) {
if (pCtx->scanFlag == REVERSE_SCAN) {
return;
}
SET_VAL(pCtx, pCtx->size, 1);
*(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp;
}
@ -3081,7 +3076,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
pOutput += 1;
pTimestamp += 1;
} else {
*pOutput = pData[i] - pData[i - step];
*pOutput = pData[i] - pCtx->param[1].i64Key; // direct previous may be null
*pTimestamp = pCtx->ptsList[i];
pOutput += 1;
@ -3113,7 +3108,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
pOutput += 1;
pTimestamp += 1;
} else {
*pOutput = pData[i] - pData[i - step];
*pOutput = pData[i] - pCtx->param[1].i64Key;
*pTimestamp = pCtx->ptsList[i];
pOutput += 1;
@ -3144,7 +3139,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
pOutput += 1;
pTimestamp += 1;
} else {
*pOutput = pData[i] - pData[i - step];
*pOutput = pData[i] - pCtx->param[1].i64Key;
*pTimestamp = pCtx->ptsList[i];
pOutput += 1;
pTimestamp += 1;
@ -3175,7 +3170,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
pOutput += 1;
pTimestamp += 1;
} else {
*pOutput = pData[i] - pData[i - step];
*pOutput = pData[i] - pCtx->param[1].i64Key;
*pTimestamp = pCtx->ptsList[i];
pOutput += 1;
@ -3207,7 +3202,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
pOutput += 1;
pTimestamp += 1;
} else {
*pOutput = pData[i] - pData[i - step];
*pOutput = pData[i] - pCtx->param[1].i64Key;
*pTimestamp = pCtx->ptsList[i];
pOutput += 1;
@ -3239,7 +3234,7 @@ static void diff_function(SQLFunctionCtx *pCtx) {
pOutput += 1;
pTimestamp += 1;
} else {
*pOutput = pData[i] - pData[i - step];
*pOutput = pData[i] - pCtx->param[1].i64Key;
*pTimestamp = pCtx->ptsList[i];
pOutput += 1;
@ -3420,7 +3415,7 @@ static void spread_function(SQLFunctionCtx *pCtx) {
// todo : opt with pre-calculated result
// column missing cause the hasNull to be true
if (usePreVal(pCtx)) {
if (pCtx->preAggVals.isSet) {
numOfElems = pCtx->size - pCtx->preAggVals.statis.numOfNull;
// all data are null in current data block, ignore current data block
@ -3446,14 +3441,8 @@ static void spread_function(SQLFunctionCtx *pCtx) {
pInfo->max = GET_DOUBLE_VAL(&(pCtx->preAggVals.statis.max));
}
}
} else {
// if (pInfo->min > pCtx->param[1].dKey) {
// pInfo->min = pCtx->param[1].dKey;
// }
//
// if (pInfo->max < pCtx->param[2].dKey) {
// pInfo->max = pCtx->param[2].dKey;
// }
goto _spread_over;
}
void *pData = GET_INPUT_CHAR(pCtx);

5
src/client/src/tscLocal.c
View File

@ -384,14 +384,11 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
// keep the code in local variable in order to avoid invalid read in case of async query
int32_t code = pSql->res.code;
if (pSql->fp != NULL) { // callback function
if (code == 0) {
if (code == TSDB_CODE_SUCCESS) {
(*pSql->fp)(pSql->param, pSql, 0);
} else {
tscQueueAsyncRes(pSql);
}
}
return code;
}

44
src/client/src/tscParseInsert.c
View File

@ -42,35 +42,42 @@ enum {
static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
// int32_t numType = isValidNumber(pToken);
// if (TK_ILLEGAL == numType) {
// return numType;
// }
if (pToken->n == 0) {
return TK_ILLEGAL;
}
int32_t radix = 10;
if (pToken->type == TK_HEX) {
radix = 16;
} else if (pToken->type == TK_OCT) {
radix = 8;
} else if (pToken->type == TK_BIN) {
radix = 2;
int32_t radixList[3] = {16, 8, 2};
if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) {
radix = radixList[pToken->type - TK_HEX];
}
errno = 0;
*value = strtoll(pToken->z, endPtr, radix);
// not a valid integer number, return error
if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
return TK_ILLEGAL;
}
return pToken->type;
}
static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
// int32_t numType = isValidNumber(pToken);
// if (TK_ILLEGAL == numType) {
// return numType;
// }
if (pToken->n == 0) {
return TK_ILLEGAL;
}
errno = 0;
*value = strtod(pToken->z, endPtr);
// not a valid integer number, return error
if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) {
return TK_ILLEGAL;
} else {
return pToken->type;
}
}
int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) {
@ -422,9 +429,9 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[
return -1;
}
if (((sToken.type != TK_NOW) && (sToken.type != TK_INTEGER) && (sToken.type != TK_STRING) &&
(sToken.type != TK_FLOAT) && (sToken.type != TK_BOOL) && (sToken.type != TK_NULL)) ||
(sToken.n == 0) || (sToken.type == TK_RP)) {
int16_t type = sToken.type;
if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL &&
type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) {
tscInvalidSQLErrMsg(error, "invalid data or symbol", sToken.z);
*code = TSDB_CODE_INVALID_SQL;
return -1;
@ -1306,8 +1313,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
SQueryInfo *pQueryInfo = NULL;
tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo);
uint16_t type = (sToken.type == TK_INSERT)? TSDB_QUERY_TYPE_INSERT:TSDB_QUERY_TYPE_IMPORT;
TSDB_QUERY_SET_TYPE(pQueryInfo->type, type);
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT);
sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL);
if (sToken.type != TK_INTO) {

33
src/client/src/tscSQLParser.c
View File

@ -3290,13 +3290,14 @@ static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg
static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SCondExpr* pCondExpr, int32_t* type,
int32_t parentOptr) {
const char* msg1 = "meter query cannot use tags filter";
const char* msg1 = "table query cannot use tags filter";
const char* msg2 = "illegal column name";
const char* msg3 = "only one query time range allowed";
const char* msg4 = "only one join condition allowed";
const char* msg5 = "not support ordinary column join";
const char* msg6 = "only one query condition on tbname allowed";
const char* msg7 = "only in/like allowed in filter table name";
const char* msg8 = "wildcard string should be less than 20 characters";
tSQLExpr* pLeft = (*pExpr)->pLeft;
tSQLExpr* pRight = (*pExpr)->pRight;
@ -3344,7 +3345,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
// check for like expression
if ((*pExpr)->nSQLOptr == TK_LIKE) {
if (pRight->val.nLen > TSDB_PATTERN_STRING_MAX_LEN) {
return TSDB_CODE_INVALID_SQL;
return invalidSqlErrMsg(pQueryInfo->msg, msg8);
}
SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
@ -3361,6 +3362,10 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S
return invalidSqlErrMsg(pQueryInfo->msg, msg7);
}
if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
return invalidSqlErrMsg(pQueryInfo->msg, msg1);
}
if (pCondExpr->pTableCond == NULL) {
pCondExpr->pTableCond = *pExpr;
pCondExpr->relType = parentOptr;
@ -3808,9 +3813,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
const char* msg2 = "invalid filter expression";
int32_t ret = TSDB_CODE_SUCCESS;
pQueryInfo->window.skey = 0;
pQueryInfo->window.ekey = INT64_MAX;
pQueryInfo->window = TSWINDOW_INITIALIZER;
// tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space
SStringBuilder sb; memset(&sb, 0, sizeof(sb));
@ -5334,13 +5337,6 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
}
}
SColumnIndex ind = {0};
SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT,
tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
switch (index) {
case 0:
pQueryInfo->command = TSDB_SQL_CURRENT_DB;
@ -5359,6 +5355,13 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
return TSDB_CODE_SUCCESS;
default: { return invalidSqlErrMsg(pQueryInfo->msg, msg3); }
}
SColumnIndex ind = {0};
SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT,
tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false);
const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name;
strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName));
}
// can only perform the parameters based on the macro definitation
@ -5606,6 +5609,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
const char* msg3 = "fill only available for interval query";
const char* msg4 = "fill option not supported in stream computing";
const char* msg5 = "sql too long"; // todo ADD support
const char* msg6 = "from missing in subclause";
SSqlCmd* pCmd = &pSql->cmd;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
@ -5623,8 +5627,11 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
}
tVariantList* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
tVariant* pVar = &pSrcMeterName->a[0].pVar;
if (pSrcMeterName == NULL || pSrcMeterName->nExpr == 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
tVariant* pVar = &pSrcMeterName->a[0].pVar;
SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING};
if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(pQueryInfo->msg, msg1);

6
src/client/src/tscSub.c
View File

@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
SSub* pSub = (SSub*)sub;
SSubscriptionProgress target = {.uid = uid, .key = 0};
SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
if (p == NULL) {
return dflt;
}
@ -74,7 +74,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
SSub* pSub = (SSub*)sub;
SSubscriptionProgress target = {.uid = uid, .key = ts};
SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
if (p != NULL) {
p->key = ts;
}
@ -211,7 +211,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0};
SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target);
SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
if (p == NULL) {
taosArrayClear(pSub->progress);
taosArrayPush(pSub->progress, &target);

2
src/common/src/tglobal.c
View File

@ -142,7 +142,7 @@ int32_t rpcDebugFlag = 135;
int32_t uDebugFlag = 131;
int32_t debugFlag = 131;
int32_t sDebugFlag = 135;
int32_t tsdbDebugFlag = 131;
int32_t tsdbDebugFlag = 135;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp

35
src/common/src/ttypes.c
View File

@ -42,9 +42,6 @@ static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t nu
ASSERT(numOfRow <= INT16_MAX);
// int64_t lastKey = 0;
// int8_t lastVal = TSDB_DATA_TINYINT_NULL;
for (int32_t i = 0; i < numOfRow; ++i) {
if (isNull((char *)&data[i], TSDB_DATA_TYPE_TINYINT)) {
(*numOfNull) += 1;
@ -213,15 +210,6 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num
fmax = fv;
*maxIndex = i;
}
// if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) {
// lastKey = primaryKey[i];
// lastVal = data[i];
// } else {
// *wsum = lastVal * (primaryKey[i] - lastKey);
// lastKey = primaryKey[i];
// lastVal = data[i];
// }
}
double csum = 0;
@ -232,9 +220,9 @@ static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t num
SET_DOUBLE_VAL_ALIGN(max, &fmax);
SET_DOUBLE_VAL_ALIGN(min, &fmin);
#else
*sum = csum;
*max = fmax;
*min = fmin;
*(double*)sum = csum;
*(double*)max = fmax;
*(double*)min = fmin;
#endif
}
@ -267,15 +255,6 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num
dmax = dv;
*maxIndex = i;
}
// if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) {
// lastKey = primaryKey[i];
// lastVal = data[i];
// } else {
// *wsum = lastVal * (primaryKey[i] - lastKey);
// lastKey = primaryKey[i];
// lastVal = data[i];
// }
}
double csum = 0;
@ -288,15 +267,15 @@ static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t num
SET_DOUBLE_VAL_ALIGN(max, &dmax);
SET_DOUBLE_VAL_ALIGN(min, &dmin);
#else
*sum = csum;
*max = dmax;
*min = dmin;
*(double*) sum = csum;
*(double*) max = dmax;
*(double*) min = dmin;
#endif
}
tDataTypeDescriptor tDataTypeDesc[11] = {
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, getStatics_i8},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint, getStatics_i16},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt, getStatics_i32},

3
src/inc/taosdef.h
View File

@ -326,8 +326,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type
#define TSDB_QUERY_TYPE_IMPORT 0x200u // import data
#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x800u
#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0)
#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type))

1
src/mnode/inc/mgmtDef.h
View File

@ -63,7 +63,6 @@ typedef struct SMnodeObj {
int8_t updateEnd[1];
int32_t refCount;
int8_t role;
SDnodeObj *pDnode;
} SMnodeObj;
typedef struct {

24
src/mnode/src/mgmtMnode.c
View File

@ -65,7 +65,6 @@ static int32_t mgmtMnodeActionInsert(SSdbOper *pOper) {
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
pMnode->pDnode = pDnode;
pDnode->isMgmt = true;
mgmtDecDnodeRef(pDnode);
@ -220,22 +219,27 @@ void mgmtUpdateMnodeIpSet() {
pIter = mgmtGetNextMnode(pIter, &pMnode);
if (pMnode == NULL) break;
strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn);
ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort);
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode != NULL) {
strcpy(ipSet->fqdn[ipSet->numOfIps], pDnode->dnodeFqdn);
ipSet->port[ipSet->numOfIps] = htons(pDnode->dnodePort);
mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp);
strcpy(mnodes->nodeInfos[index].nodeEp, pDnode->dnodeEp);
if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
ipSet->inUse = ipSet->numOfIps;
mnodes->inUse = index;
}
mPrint("mnode:%d, ep:%s %s", index, pMnode->pDnode->dnodeEp, pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
mPrint("mnode:%d, ep:%s %s", index, pDnode->dnodeEp,
pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
ipSet->numOfIps++;
index++;
}
mgmtDecDnodeRef(pDnode);
mgmtDecMnodeRef(pMnode);
}
@ -385,7 +389,15 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pMnode->pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode != NULL) {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
} else {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols] - VARSTR_HEADER_SIZE);
}
mgmtDecDnodeRef(pDnode);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

12
src/mnode/src/mgmtSdb.c
View File

@ -28,6 +28,7 @@
#include "mgmtDef.h"
#include "mgmtInt.h"
#include "mgmtMnode.h"
#include "mgmtDnode.h"
#include "mgmtSdb.h"
typedef enum {
@ -259,10 +260,15 @@ void sdbUpdateSync() {
if (pMnode == NULL) break;
syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
syncCfg.nodeInfo[index].nodePort = pMnode->pDnode->dnodePort + TSDB_PORT_SYNC;
strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
index++;
SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
if (pDnode != NULL) {
syncCfg.nodeInfo[index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC;
strcpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeEp);
index++;
}
mgmtDecDnodeRef(pDnode);
mgmtDecMnodeRef(pMnode);
}
sdbFreeIter(pIter);

2
src/query/inc/tsqlfunction.h
View File

@ -126,7 +126,6 @@ typedef struct SArithmeticSupport {
typedef struct SQLPreAggVal {
bool isSet;
int32_t size;
SDataStatis statis;
} SQLPreAggVal;
@ -174,7 +173,6 @@ typedef struct SQLFunctionCtx {
int16_t outputBytes; // size of results, determined by function and input column data type
bool hasNull; // null value exist in current block
int16_t functionId; // function id
int32_t blockStatus; // Indicate if data is loaded, it is first/last/internal block. Only for file blocks
void * aInputElemBuf;
char * aOutputBuf; // final result output buffer, point to sdata->data
uint8_t currentStage; // record current running step, default: 0

109
src/query/src/qExecutor.c
View File

@ -110,7 +110,7 @@ static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInf
static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId);
static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow);
static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* pData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag);
static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols);
@ -403,19 +403,20 @@ static bool isTopBottomQuery(SQuery *pQuery) {
return false;
}
static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) {
static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, int32_t numOfCols, int32_t index) {
// for a tag column, no corresponding field info
SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].base.colInfo;
if (TSDB_COL_IS_TAG(pColIndexEx->flag)) {
SColIndex *pColIndex = &pQuery->pSelectExpr[index].base.colInfo;
if (TSDB_COL_IS_TAG(pColIndex->flag)) {
return NULL;
}
/*
* Choose the right column field info by field id, since the file block may be out of date,
* which means the newest table schema is not equalled to the schema of this block.
* TODO: speedup by using bsearch
*/
for (int32_t i = 0; i < pDataBlockInfo->numOfCols; ++i) {
if (pColIndexEx->colId == pStatis[i].colId) {
for (int32_t i = 0; i < numOfCols; ++i) {
if (pColIndex->colId == pStatis[i].colId) {
return &pStatis[i];
}
}
@ -431,8 +432,7 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo
* @param pColStatis
* @return
*/
static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis,
SDataStatis **pColStatis) {
static bool hasNullValue(SQuery *pQuery, int32_t col, int32_t numOfCols, SDataStatis *pStatis, SDataStatis **pColStatis) {
SColIndex *pColIndex = &pQuery->pSelectExpr[col].base.colInfo;
if (TSDB_COL_IS_TAG(pColIndex->flag)) {
return false;
@ -444,7 +444,7 @@ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlock
}
if (pStatis != NULL) {
*pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col);
*pColStatis = getStatisInfo(pQuery, pStatis, numOfCols, col);
} else {
*pColStatis = NULL;
}
@ -733,7 +733,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
}
static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin,
int32_t startPos, int32_t forwardStep, TSKEY *tsBuf) {
int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) {
SQuery * pQuery = pRuntimeEnv->pQuery;
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
@ -743,10 +743,15 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat
pCtx[k].nStartQueryTimestamp = pWin->skey;
pCtx[k].size = forwardStep;
pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1);
pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset];
pCtx[k].ptsList = tsBuf;
}
// not a whole block involved in query processing, statistics data can not be used
if (forwardStep != numOfTotal) {
pCtx[k].preAggVals.isSet = false;
}
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
@ -890,7 +895,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
}
assert(dataBlock != NULL);
sas->data[i] = dataBlock + pCtx->startOffset * pQuery->colList[i].bytes; // start from the offset
sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/; // start from the offset
}
} else { // other type of query function
@ -933,14 +938,15 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
SDataStatis *tpField = NULL;
bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField);
bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &tpField);
char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull,
&sasArray[k], pRuntimeEnv->scanFlag);
setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, tpField, hasNull,
&sasArray[k], colId);
}
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
@ -958,7 +964,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true);
SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol);
doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
int32_t index = pWindowResInfo->curIndex;
STimeWindow nextWin = win;
@ -978,7 +984,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true);
pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol);
doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
}
pWindowResInfo->curIndex = index;
@ -1154,14 +1160,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
SDataStatis *pColStatis = NULL;
bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis);
bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &pColStatis);
char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull,
&sasArray[k], pRuntimeEnv->scanFlag);
setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, pColStatis, hasNull,
&sasArray[k], colId);
}
// set the input column data
@ -1214,7 +1221,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
}
// all startOffset are identical
offset -= pCtx[0].startOffset;
// offset -= pCtx[0].startOffset;
SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset);
@ -1255,9 +1262,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
}
}
// all startOffset are identical
offset -= pCtx[0].startOffset;
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
@ -1333,27 +1337,28 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
return numOfRes;
}
void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) {
pCtx->scanFlag = scanFlag;
pCtx->aInputElemBuf = inputData;
void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t colId) {
pCtx->hasNull = hasNull;
pCtx->aInputElemBuf = inputData;
if (pStatis != NULL) {
pCtx->preAggVals.isSet = true;
pCtx->preAggVals.size = size;
pCtx->preAggVals.statis = *pStatis;
if (pCtx->preAggVals.statis.numOfNull == -1) {
pCtx->preAggVals.statis.numOfNull = pBlockInfo->rows; // todo :can not be -1
}
} else {
pCtx->preAggVals.isSet = false;
}
pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0;
pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? size - pQuery->pos : pQuery->pos + 1;
// limit/offset query will affect this value
pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0;
pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1;
uint32_t status = aAggs[functionId].nStatus;
if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {
pCtx->ptsList = &tsCol[pCtx->startOffset];
pCtx->ptsList = tsCol;
}
if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) {
@ -1363,7 +1368,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
} else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA ||
functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) {
/*
* leastsquares function needs two columns of input, currently, the x value of linear equation is set to
* least squares function needs two columns of input, currently, the x value of linear equation is set to
* timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer
*
* top/bottom function needs timestamp to indicate when the
@ -1377,6 +1382,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
} else if (functionId == TSDB_FUNC_ARITHM) {
pCtx->param[1].pz = param;
} else if (functionId == TSDB_FUNC_SPREAD) { // set the statistics data for primary time stamp column
if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
pCtx->preAggVals.isSet = true;
pCtx->preAggVals.statis.min = pBlockInfo->window.skey;
pCtx->preAggVals.statis.max = pBlockInfo->window.ekey;
}
}
#if defined(_DEBUG_VIEW)
@ -2455,9 +2466,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
}
SDataStatis *pStatis = NULL;
SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1;
SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock);
qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, numOfRes:%d", GET_QINFO_ADDR(pRuntimeEnv),
@ -3633,12 +3644,14 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
STimeWindow w = {0};
SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo;
getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w);
TSKEY sk = MIN(win.skey, win.ekey);
TSKEY ek = MAX(win.skey, win.ekey);
getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &skey1, &ekey1, &w);
pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp
if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
if (!QUERY_IS_ASC_QUERY(pQuery)) {
assert(win.ekey == pQuery->window.skey);
assert(win.ekey == pQuery->window.ekey);
}
pWindowResInfo->prevSKey = w.skey;
@ -3673,10 +3686,6 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) {
return loadPrimaryTS;
}
bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) {
return (pQuery->intervalTime == 0) || ((queryRangeSet == 1) && (isIntervalQuery(pQuery)));
}
static int32_t getNumOfSubset(SQInfo *pQInfo) {
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
@ -5610,18 +5619,18 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) {
continue;
}
SColIndex *pColIndexEx = &pSqlExprMsg->colInfo;
if (!TSDB_COL_IS_TAG(pColIndexEx->flag)) {
SColIndex *pColIndex = &pSqlExprMsg->colInfo;
if (!TSDB_COL_IS_TAG(pColIndex->flag)) {
for (int32_t f = 0; f < pQuery->numOfCols; ++f) {
if (pColIndexEx->colId == pQuery->colList[f].colId) {
pColIndexEx->colIndex = f;
if (pColIndex->colId == pQuery->colList[f].colId) {
pColIndex->colIndex = f;
break;
}
}
} else {
for (int32_t f = 0; f < pQuery->numOfTags; ++f) {
if (pColIndexEx->colId == pQuery->tagColList[f].colId) {
pColIndexEx->colIndex = f;
if (pColIndex->colId == pQuery->tagColList[f].colId) {
pColIndex->colIndex = f;
break;
}
}
@ -5745,7 +5754,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
SGroupItem item = { .id = id };
// NOTE: compare STableIdInfo with STableId
// not a problem at present because we only use their 1st int64_t field
STableIdInfo* pTableId = taosArraySearch( pTableIdList, compareTableIdInfo, &id );
STableIdInfo* pTableId = taosArraySearch( pTableIdList, &id, compareTableIdInfo);
if (pTableId != NULL ) {
window.skey = pTableId->key;
} else {

24
src/query/src/qast.c
View File

@ -773,9 +773,6 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
// todo refactor:
tstr *name = ((STableIndexElem *)pData)->pTable->name;
// char* name = NULL;
// tsdbGetTableName(pQueryInfo->, pTable, &name);
// todo speed up by using hash
if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
if (pQueryInfo->optr == TSDB_RELATION_IN) {
@ -1051,7 +1048,7 @@ static void* exception_malloc(size_t size) {
return p;
}
static char* exception_strdup(const char* str) {
static UNUSED_FUNC char* exception_strdup(const char* str) {
char* p = strdup(str);
if (p == NULL) {
THROW(TSDB_CODE_SERV_OUT_OF_MEMORY);
@ -1154,28 +1151,33 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
tVariant* pVal = exception_calloc(1, sizeof(tVariant));
right->pVal = pVal;
pVal->nType = TSDB_DATA_TYPE_ARRAY;
pVal->arr = taosArrayInit(2, sizeof(char*));
pVal->arr = taosArrayInit(2, POINTER_BYTES);
const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN;
for (const char *e = cond; *e != 0; e++) {
if (*e == TS_PATH_DELIMITER[0]) {
cond = e + 1;
} else if (*e == ',') {
size_t len = e - cond + 1;
char* p = exception_malloc( len );
memcpy(p, cond, len);
p[len - 1] = 0;
size_t len = e - cond + VARSTR_HEADER_SIZE;
char* p = exception_malloc(len);
varDataSetLen(p, len - VARSTR_HEADER_SIZE);
memcpy(varDataVal(p), cond, len);
cond += len;
taosArrayPush(pVal->arr, &p);
}
}
if (*cond != 0) {
char* p = exception_strdup( cond );
size_t len = strlen(cond) + VARSTR_HEADER_SIZE;
char* p = exception_malloc(len);
varDataSetLen(p, len - VARSTR_HEADER_SIZE);
memcpy(varDataVal(p), cond, len);
taosArrayPush(pVal->arr, &p);
}
taosArraySortString(pVal->arr);
taosArraySortString(pVal->arr, taosArrayCompareString);
}
CLEANUP_EXECUTE_TO(anchor, false);

2
src/tsdb/src/tsdbMain.c
View File

@ -1196,7 +1196,7 @@ uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *
// Map index to the file name
int fid = (*index) / 3;
if (fid > pFileH->numOfFGroups) {
if (fid >= pFileH->numOfFGroups) {
// return meta data file
if ((*index) % 3 > 0) { // it is finished
tfree(spath);

184
src/tsdb/src/tsdbRead.c
View File

@ -40,10 +40,6 @@ enum {
TSDB_QUERY_TYPE_EXTERNAL = 3,
};
typedef struct SField {
// todo need the definition
} SField;
typedef struct SQueryFilePos {
int32_t fid;
int32_t slot;
@ -71,28 +67,20 @@ typedef struct STableCheckInfo {
STableId tableId;
TSKEY lastKey;
STable* pTableObj;
int32_t start;
SCompInfo* pCompInfo;
int32_t compSize;
int32_t numOfBlocks; // number of qualified data blocks not the original blocks
SDataCols* pDataCols;
SSkipListIterator* iter; // skip list iterator
SSkipListIterator* iiter; // imem iterator
bool initBuf; // if we should initialize the in-memory skip list iterator
bool initBuf; // whether to initialize the in-memory skip list iterator or not
SSkipListIterator* iter; // mem buffer skip list iterator
SSkipListIterator* iiter; // imem buffer skip list iterator
} STableCheckInfo;
typedef struct {
SCompBlock* compBlock;
SField* fields;
} SCompBlockFields;
typedef struct STableBlockInfo {
SCompBlockFields pBlock;
SCompBlock* compBlock;
STableCheckInfo* pTableCheckInfo;
int32_t blockIndex;
int32_t groupIdx; /* number of group is less than the total number of tables */
// int32_t blockIndex;
// int32_t groupIdx; /* number of group is less than the total number of tables */
} STableBlockInfo;
typedef struct SBlockOrderSupporter {
@ -105,15 +93,11 @@ typedef struct SBlockOrderSupporter {
typedef struct STsdbQueryHandle {
STsdbRepo* pTsdb;
SQueryFilePos cur; // current position
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
int16_t order;
STimeWindow window; // the primary query time window that applies to all queries
SCompBlock* pBlock;
SDataStatis* statis; // query level statistics, only one table block statistics info exists at any time
int32_t numOfBlocks;
SField** pFields;
SArray* pColumns; // column list, SColumnInfoData array list
bool locateStart;
int32_t outputCapacity;
@ -123,11 +107,13 @@ typedef struct STsdbQueryHandle {
bool checkFiles; // check file stage
void* qinfo; // query info handle, for debug purpose
int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
STableBlockInfo* pDataBlockInfo;
SFileGroup* pFileGroup;
SFileGroupIter fileIter;
SRWHelper rhelper;
STableBlockInfo* pDataBlockInfo;
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
} STsdbQueryHandle;
static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle);
@ -152,15 +138,35 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
pQueryHandle->window = pCond->twindow;
pQueryHandle->pTsdb = tsdb;
pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb);
pQueryHandle->cur.fid = -1;
pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
pQueryHandle->checkFiles = true;//ASCENDING_TRAVERSE(pQueryHandle->order);
pQueryHandle->activeIndex = 0; // current active table index
pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock;
tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb);
size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList);
assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0);
// allocate buffer in order to load data blocks from file
int32_t numOfCols = pCond->numOfCols;
pQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis));
pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array?
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {{0}, 0};
colInfo.info = pCond->colList[i];
colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes);
taosArrayPush(pQueryHandle->pColumns, &colInfo);
pQueryHandle->statis[i].colId = colInfo.info.colId;
}
pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo));
STsdbMeta* pMeta = tsdbGetMeta(tsdb);
assert(pMeta != NULL);
for (int32_t i = 0; i < sizeOfGroup; ++i) {
SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
@ -174,7 +180,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
STableCheckInfo info = {
.lastKey = pQueryHandle->window.skey,
.tableId = *id,
.pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id->uid),
.pTableObj = tsdbGetTableByUid(pMeta, id->uid),
};
assert(info.pTableObj != NULL && info.pTableObj->tableId.tid == id->tid);
@ -182,28 +188,11 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable
}
}
uTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
/*
* For ascending timestamp order query, query starts from data files. In contrast, buffer will be checked in the first place
* in case of descending timestamp order query.
*/
pQueryHandle->checkFiles = true;//ASCENDING_TRAVERSE(pQueryHandle->order);
pQueryHandle->activeIndex = 0;
// allocate buffer in order to load data blocks from file
int32_t numOfCols = pCond->numOfCols;
pQueryHandle->outputCapacity = 4096;
pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
SColumnInfoData colInfo = {{0}, 0};
colInfo.info = pCond->colList[i];
colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes);
taosArrayPush(pQueryHandle->pColumns, &colInfo);
for(int32_t i = 0; i < numOfCols; ++i) {
}
uTrace("%p total numOfTable:%d in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
@ -499,8 +488,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
bool blockLoaded = false;
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
if (pCheckInfo->pDataCols == NULL) { // todo: why not the real data?
pCheckInfo->pDataCols = tdNewDataCols(pRepo->tsdbMeta->maxRowBytes, pRepo->tsdbMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
if (pCheckInfo->pDataCols == NULL) {
STsdbMeta* pMeta = tsdbGetMeta(pRepo);
pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
}
tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj));
@ -522,8 +512,6 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
}
static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlock);
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
@ -592,8 +580,11 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
return;
}
SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa);
taosArrayDestroy(sa);
} else {
pQueryHandle->realNumOfRows = binfo.rows;
@ -1074,7 +1065,9 @@ static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t n
tfree(pSupporter->blockIndexArray);
for (int32_t i = 0; i < numOfTables; ++i) {
tfree(pSupporter->pDataBlockInfo[i]);
STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i];
// tfree(pBlockInfo->statis);
tfree(pBlockInfo);
}
tfree(pSupporter->pDataBlockInfo);
@ -1100,14 +1093,14 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void*
STableBlockInfo* pLeftBlockInfoEx = &pSupporter->pDataBlockInfo[leftTableIndex][leftTableBlockIndex];
STableBlockInfo* pRightBlockInfoEx = &pSupporter->pDataBlockInfo[rightTableIndex][rightTableBlockIndex];
// assert(pLeftBlockInfoEx->pBlock.compBlock->offset != pRightBlockInfoEx->pBlock.compBlock->offset);
if (pLeftBlockInfoEx->pBlock.compBlock->offset == pRightBlockInfoEx->pBlock.compBlock->offset &&
pLeftBlockInfoEx->pBlock.compBlock->last == pRightBlockInfoEx->pBlock.compBlock->last) {
// assert(pLeftBlockInfoEx->compBlock->offset != pRightBlockInfoEx->compBlock->offset);
if (pLeftBlockInfoEx->compBlock->offset == pRightBlockInfoEx->compBlock->offset &&
pLeftBlockInfoEx->compBlock->last == pRightBlockInfoEx->compBlock->last) {
// todo add more information
uError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->pBlock.compBlock->offset);
uError("error in header file, two block with same offset:%p", pLeftBlockInfoEx->compBlock->offset);
}
return pLeftBlockInfoEx->pBlock.compBlock->offset > pRightBlockInfoEx->pBlock.compBlock->offset ? 1 : -1;
return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1;
}
static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) {
@ -1116,7 +1109,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
return TSDB_CODE_SERV_OUT_OF_MEMORY;
}
pQueryHandle->pDataBlockInfo = (STableBlockInfo*)tmp;
pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp;
memset(pQueryHandle->pDataBlockInfo, 0, sizeof(STableBlockInfo) * numOfBlocks);
*numOfAllocBlocks = numOfBlocks;
@ -1135,6 +1128,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
int32_t cnt = 0;
int32_t numOfQualTables = 0;
for (int32_t j = 0; j < numOfTables; ++j) {
STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j);
if (pTableCheck->numOfBlocks <= 0) {
@ -1153,14 +1147,12 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
sup.pDataBlockInfo[numOfQualTables] = (STableBlockInfo*)buf;
for (int32_t k = 0; k < pTableCheck->numOfBlocks; ++k) {
STableBlockInfo* pBlockInfoEx = &sup.pDataBlockInfo[numOfQualTables][k];
STableBlockInfo* pBlockInfo = &sup.pDataBlockInfo[numOfQualTables][k];
pBlockInfoEx->pBlock.compBlock = &pBlock[k];
pBlockInfoEx->pBlock.fields = NULL;
pBlockInfoEx->pTableCheckInfo = pTableCheck;
// pBlockInfoEx->groupIdx = pTableCheckInfo[j]->groupIdx; // set the group index
// pBlockInfoEx->blockIndex = pTableCheckInfo[j]->start + k; // set the block index in original table
pBlockInfo->compBlock = &pBlock[k];
pBlockInfo->pTableCheckInfo = pTableCheck;
// pBlockInfo->groupIdx = pTableCheckInfo[j]->groupIdx; // set the group index
// pBlockInfo->blockIndex = pTableCheckInfo[j]->start + k; // set the block index in original table
cnt++;
}
@ -1185,8 +1177,8 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
int32_t pos = pTree->pNode[0].index;
int32_t index = sup.blockIndexArray[pos]++;
STableBlockInfo* pBlocksInfoEx = sup.pDataBlockInfo[pos];
pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfoEx[index];
STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos];
pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index];
// set data block index overflow, in order to disable the offset comparator
if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) {
@ -1199,7 +1191,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
/*
* available when no import exists
* for(int32_t i = 0; i < cnt - 1; ++i) {
* assert((*pDataBlockInfo)[i].pBlock.compBlock->offset < (*pDataBlockInfo)[i+1].pBlock.compBlock->offset);
* assert((*pDataBlockInfo)[i].compBlock->offset < (*pDataBlockInfo)[i+1].compBlock->offset);
* }
*/
@ -1255,7 +1247,7 @@ static bool getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle) {
cur->fid = pQueryHandle->pFileGroup->fileId;
STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
return loadFileDataBlock(pQueryHandle, pBlockInfo->pBlock.compBlock, pBlockInfo->pTableCheckInfo);
return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo);
}
static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
@ -1291,10 +1283,10 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
cur->blockCompleted = false;
STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
return loadFileDataBlock(pQueryHandle, pNext->pBlock.compBlock, pNext->pTableCheckInfo);
return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo);
}
} else {
handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->pBlock.compBlock, pCheckInfo);
handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo);
return pQueryHandle->realNumOfRows > 0;
}
}
@ -1484,35 +1476,33 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
// there are data in file
if (pHandle->cur.fid >= 0) {
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
STable* pTable = pBlockInfo->pTableCheckInfo->pTableObj;
STable* pTable = pCheckInfo->pTableObj;
if (pHandle->cur.mixBlock) {
SDataBlockInfo blockInfo = {
.uid = pTable->tableId.uid,
.tid = pTable->tableId.tid,
.rows = pHandle->cur.rows,
.window = pHandle->cur.win,
.numOfCols = QH_GET_NUM_OF_COLS(pHandle),
};
return blockInfo;
} else {
return getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock);
}
} else {
STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
SQueryFilePos* cur = &pHandle->cur;
STable* pTable = pCheckInfo->pTableObj;
if (pTable->mem != NULL) { // create mem table iterator if it is not created yet
assert(pCheckInfo->iter != NULL);
STimeWindow* win = &pHandle->cur.win;
STimeWindow* win = &cur->win;
pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey,
pHandle->outputCapacity, &win->skey, &win->ekey, pHandle); // todo refactor API
// update the last key value
pCheckInfo->lastKey = win->ekey + step;
cur->lastKey = win->ekey + step;
cur->mixBlock = true;
}
if (!ASCENDING_TRAVERSE(pHandle->order)) {
@ -1524,15 +1514,34 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
.tid = pTable->tableId.tid,
.rows = pHandle->cur.rows,
.window = pHandle->cur.win,
.numOfCols = QH_GET_NUM_OF_COLS(pHandle),
};
return blockInfo;
}
}
// return null for data block in cache
/*
* return null for mixed data block, if not a complete file data block, the statistics value will always return NULL
*/
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
SQueryFilePos* cur = &pHandle->cur;
if (cur->mixBlock) {
*pBlockStatis = NULL;
return TSDB_CODE_SUCCESS;
}
assert((cur->slot >= 0 && cur->slot < pHandle->numOfBlocks) ||
((cur->slot == pHandle->numOfBlocks) && (cur->slot == 0)));
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot];
tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL);
tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, QH_GET_NUM_OF_COLS(pHandle));
*pBlockStatis = pHandle->statis;
return TSDB_CODE_SUCCESS;
}
@ -1546,13 +1555,13 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
if (pHandle->cur.fid < 0) {
return pHandle->pColumns;
} else {
STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot];
STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo;
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
if (pHandle->cur.mixBlock) {
return pHandle->pColumns;
} else {
SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock);
SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfo->compBlock);
assert(pHandle->realNumOfRows <= binfo.rows);
// data block has been loaded, todo extract method
@ -1562,7 +1571,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
return pHandle->pColumns;
} else { // only load the file block
SCompBlock* pBlock = pBlockInfoEx->pBlock.compBlock;
SCompBlock* pBlock = pBlockInfo->compBlock;
doLoadFileDataBlock(pHandle, pBlock, pCheckInfo);
// todo refactor
@ -1669,7 +1678,7 @@ void filterPrepare(void* expr, void* param) {
tVariant* pCond = pExpr->_node.pRight->pVal;
SSchema* pSchema = pExpr->_node.pLeft->pSchema;
// todo : if current super table does not change schema yet, this function may failed, add test case
// todo : if current super table does not change schema yet, this function may fail to get correct schema, test case
int32_t index = getTagColumnIndex(pTSSchema, pSchema);
assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX));
@ -2006,8 +2015,9 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
}
taosArrayDestroy(pQueryHandle->pColumns);
tfree(pQueryHandle->pDataBlockInfo);
tfree(pQueryHandle->statis);
tsdbDestroyHelper(&pQueryHandle->rhelper);
tfree(pQueryHandle);

6
src/util/inc/tarray.h
View File

@ -129,7 +129,7 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*));
* sort string array
* @param pArray
*/
void taosArraySortString(SArray* pArray);
void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
/**
* search the array
@ -137,14 +137,14 @@ void taosArraySortString(SArray* pArray);
* @param compar
* @param key
*/
void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key);
void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn);
/**
* search the array
* @param pArray
* @param key
*/
char* taosArraySearchString(const SArray* pArray, const char* key);
char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn);
#ifdef __cplusplus
}

8
src/util/inc/tcompare.h
View File

@ -34,16 +34,18 @@ typedef struct SPatternCompareInfo {
char matchOne; // symbol for match one wildcard, default: '_'
} SPatternCompareInfo;
int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo);
int patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo);
int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size);
int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
__compar_fn_t getKeyComparFunc(int32_t keyType);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
int32_t taosArrayCompareString(const void* a, const void* b);
#ifdef __cplusplus
}
#endif

21
src/util/src/tarray.c
View File

@ -197,30 +197,23 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
qsort(pArray->pData, pArray->size, pArray->elemSize, compar);
}
void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key) {
assert(pArray != NULL);
assert(compar != NULL);
void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn) {
assert(pArray != NULL && comparFn != NULL);
assert(key != NULL);
return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar);
return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
}
static int taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a;
const char* y = *(const char**)b;
return strcmp(x, y);
}
void taosArraySortString(SArray* pArray) {
void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
assert(pArray != NULL);
qsort(pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString);
qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn);
}
char* taosArraySearchString(const SArray* pArray, const char* key) {
char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn) {
assert(pArray != NULL);
assert(key != NULL);
void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString);
void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
if (p == NULL) {
return NULL;
}

9
src/util/src/tcompare.c
View File

@ -227,9 +227,16 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
int32_t taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a;
const char* y = *(const char**)b;
return compareLenPrefixedStr(x, y);
}
static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
const SArray* arr = (const SArray*) pRight;
return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1;
return taosArraySearchString(arr, pLeft, taosArrayCompareString) == NULL ? 0 : 1;
}
static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {

4
tests/script/general/parser/create_db.sim
View File

@ -104,14 +104,14 @@ $replica = 1 # max=3
$days = 10
$keep = 365
$rows_db = 1000
$cache = 4096 # 4 kb
$cache = 16 # 16MB
$ablocks = 100
$tblocks = 32 # max=512, automatically trimmed when exceeding
$ctime = 36000 # 10 hours
$wal = 0 # valid value is 0, 1, 2
$comp = 1 # max=32, automatically trimmed when exceeding
sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ablocks $ablocks tblocks $tblocks ctime $ctime wal $wal comp $comp
sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ctime $ctime wal $wal comp $comp
sql show databases
if $rows != 1 then
return -1

7
tests/script/general/parser/first_last_query.sim
View File

@ -27,32 +27,25 @@ endi
if $data00 != @18-09-17 08:59:00.000@ then
return -1
endi
#if $data01 != NULL then
if $data01 != 0 then
return -1
endi
#if $data02 != NULL then
if $data02 != 0 then
return -1
endi
#if $data03 != NULL then
print data03 = $data03
if $data03 != 0.00000 then
return -1
endi
#if $data04 != NULL then
if $data04 != 0.000000000 then
return -1
endi
#if $data05 != NULL then
if $data05 != 0 then
return -1
endi
#if $data06 != NULL then
if $data06 != 0 then
return -1
endi
#if $data07 != NULL then
if $data07 != 1 then
return -1
endi

12
tests/script/general/parser/null_char.sim
View File

@ -177,7 +177,7 @@ sql_error insert into st34 using mt3 tags ('NULL', '123aBc', 105, NULL) values
#### case 3: set tag value
sql create table mt4 (ts timestamp, c1 int) tags (tag_binary binary(16), tag_nchar nchar(16), tag_int int, tag_bool bool, tag_float float, tag_double double)
sql create table st41 using mt4 tags ("beijing", 'nchar_tag', 100, false, 9.12345, 7.123456789)
sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double st41
sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
if $rows != 1 then
return -1
endi
@ -190,13 +190,17 @@ endi
if $data02 != 100 then
return -1
endi
if $data03 != false then
if $data03 != 0 then
return -1
endi
if $dat04 != 9.123450 then
if $data04 != 9.12345 then
print expect 9.12345 , actual: $data04
return -1
endi
if $data05 != 7.123457 then
if $data05 != 7.123456789 then
print expect 7.123456789 , actual: $data05
return -1
endi

22
tests/script/general/parser/projection_limit_offset.sim
View File

@ -133,7 +133,7 @@ if $rows != 4007 then
return -1
endi
if $data00 != @70-01-01 08:01:43.499@ then
if $data00 != @70-01-01 08:01:43.500@ then
return -1
endi
@ -143,7 +143,7 @@ if $rows != 3907 then
return -1
endi
if $data00 != @70-01-01 08:01:43.399@ then
if $data00 != @70-01-01 08:01:43.488@ then
return -1
endi
@ -152,7 +152,7 @@ if $rows != 3106 then
return -1
endi
if $data00 != @70-01-01 08:01:43.099@ then
if $data00 != @70-01-01 08:01:43.388@ then
return -1
endi
@ -161,7 +161,7 @@ if $rows != 3608 then
return -1
endi
if $data00 != @70-01-01 08:01:43.100@ then
if $data00 != @70-01-01 08:01:43.450@ then
return -1
endi
@ -358,6 +358,15 @@ if $row != 8 then
return -1
endi
sql select diff(k) from tm0
if $row != 3 then
return -1
endi
if $data21 != -1 then
return -1
endi
#error sql
sql_error select * from 1;
sql_error select 1;
@ -371,8 +380,11 @@ sql_error select 1 interval(1h);
sql_error select count(*);
sql_error select sum(k);
sql_error select 'abc';
sql_error select k+1,sum(k) from tm0;
sql_error select k, sum(k) from tm0;
sql_error select k, sum(k)+1 from tm0;
#=============================tbase-1205
sql select count(*) from tm1 where ts<now and ts>= now -1d interval(1h) fill(NULL);
system sh/exec.sh -n dnode1 -s stop -x SIGINT

4
tests/script/general/parser/slimit.sim
View File

@ -22,7 +22,7 @@ $db = $dbPrefix . $i
$stb = $stbPrefix . $i
sql drop database if exists $db
sql create database $db maxrows 200 cache 1024 tblocks 200 maxTables 4
sql create database $db maxrows 200 cache 16 maxTables 4
print ====== create tables
sql use $db
sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 binary(15), t2 int, t3 bigint, t4 nchar(10), t5 double, t6 bool)
@ -64,7 +64,7 @@ print ====== $db tables created
$db = $dbPrefix . 1
sql drop database if exists $db
sql create database $db maxrows 200 cache 1024
sql create database $db maxrows 200 cache 16
sql use $db
sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 binary(15), t2 int, t3 bigint, t4 nchar(10), t5 double, t6 bool)

13
tests/script/general/parser/testSuite.sim
View File

@ -56,15 +56,17 @@ sleep 2000
run general/parser/limit1_tblocks100.sim
sleep 2000
run general/parser/select_across_vnodes.sim
sleep 2000
run general/parser/slimit1.sim
sleep 2000
run general/parser/tbnameIn.sim
sleep 2000
run general/parser/projection_limit_offset.sim
sleep 2000
run general/parser/limit2.sim
sleep 2000
run general/parser/tbnameIn.sim
sleep 2000
run general/parser/slimit.sim
sleep 2000
run general/parser/slimit1.sim
sleep 2000
run general/parser/fill.sim
@ -94,8 +96,7 @@ sleep 2000
run general/parser/join.sim
sleep 2000
run general/parser/join_multivnode.sim
sleep 2000
run general/parser/projection_limit_offset.sim
sleep 2000
run general/parser/select_with_tags.sim
sleep 2000

42
tests/script/jenkins/basic.txt
View File

@ -109,9 +109,9 @@ cd ../../../debug; make
./test.sh -f general/parser/auto_create_tb.sim
./test.sh -f general/parser/auto_create_tb_drop_tb.sim
./test.sh -f general/parser/col_arithmetic_operation.sim
#/test.sh -f general/parser/columnValue.sim
./test.sh -f general/parser/columnValue.sim
./test.sh -f general/parser/commit.sim
# ./test.sh -f general/parser/create_db.sim
#./test.sh -f general/parser/create_db.sim #there are bugs in this sim script
./test.sh -f general/parser/create_mt.sim
./test.sh -f general/parser/create_tb.sim
./test.sh -f general/parser/dbtbnameValidate.sim
@ -122,38 +122,38 @@ cd ../../../debug; make
./test.sh -f general/parser/first_last.sim
# ./test.sh -f general/parser/import_file.sim
./test.sh -f general/parser/lastrow.sim
# ./test.sh -f general/parser/nchar.sim
# ./test.sh -f general/parser/null_char.sim
./test.sh -f general/parser/nchar.sim
./test.sh -f general/parser/null_char.sim
./test.sh -f general/parser/single_row_in_tb.sim
./test.sh -f general/parser/select_from_cache_disk.sim
./test.sh -f general/parser/limit.sim
# ./test.sh -f general/parser/fill.sim
# ./test.sh -f general/parser/fill_stb.sim
# ./test.sh -f general/parser/tags_dynamically_specifiy.sim
# ./test.sh -f general/parser/interp.sim
./test.sh -f general/parser/limit1.sim
./test.sh -f general/parser/limit1_tblocks100.sim
# ./test.sh -f general/parser/limit2.sim
./test.sh -f general/parser/mixed_blocks.sim
./test.sh -f general/parser/selectResNum.sim
./test.sh -f general/parser/select_across_vnodes.sim
# ./test.sh -f general/parser/set_tag_vals.sim
# ./test.sh -f general/parser/slimit.sim
./test.sh -f general/parser/slimit1.sim
./test.sh -f general/parser/tbnameIn.sim
./test.sh -f general/parser/binary_escapeCharacter.sim
./test.sh -f general/parser/projection_limit_offset.sim
# ./test.sh -f general/parser/limit2.sim
# ./test.sh -f general/parser/slimit.sim
# ./test.sh -f general/parser/fill.sim
# ./test.sh -f general/parser/fill_stb.sim
# ./test.sh -f general/parser/interp.sim
# ./test.sh -f general/parser/where.sim
# ./test.sh -f general/parser/join.sim
# ./test.sh -f general/parser/join_multivnode.sim
# ./test.sh -f general/parser/select_with_tags.sim
# ./test.sh -f general/parser/groupby.sim
# ./test.sh -f general/parser/bug.sim
#unsupport ./test.sh -f general/parser/tags_dynamically_specifiy.sim
#unsupport ./test.sh -f general/parser/set_tag_vals.sim
#unsupport ./test.sh -f general/parser/repeatAlter.sim
#unsupport ./test.sh -f general/parser/slimit_alter_tags.sim
#unsupport ./test.sh -f general/parser/stream_on_sys.sim
#unsupport ./test.sh -f general/parser/stream.sim
# ./test.sh -f general/parser/tbnameIn.sim
# ./test.sh -f general/parser/where.sim
# ./test.sh -f general/parser/repeatAlter.sim
#unsupport ./test.sh -f general/parser/repeatStream.sim
# ./test.sh -f general/parser/join.sim
# ./test.sh -f general/parser/join_multivnode.sim
# ./test.sh -f general/parser/projection_limit_offset.sim
# ./test.sh -f general/parser/select_with_tags.sim
# ./test.sh -f general/parser/groupby.sim
./test.sh -f general/parser/binary_escapeCharacter.sim
#./test.sh -f general/parser/bug.sim
./test.sh -f general/stable/disk.sim
./test.sh -f general/stable/dnode3.sim

14
tests/script/tmp/prepare.sim
View File

@ -1,14 +1,4 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/cfg.sh -n dnode1 -c numOfMPeers -v 2
system sh/cfg.sh -n dnode2 -c numOfMPeers -v 2
system sh/cfg.sh -n dnode3 -c numOfMPeers -v 2
return
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
@ -34,3 +24,7 @@ system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
system sh/cfg.sh -n dnode1 -c http -v 1
system sh/cfg.sh -n dnode2 -c http -v 1
system sh/cfg.sh -n dnode3 -c http -v 1