xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-32] support multi-table query

This commit is contained in:
hjxilinx 2020-03-28 10:34:51 +08:00
parent 6fae35afd0
commit ff14492181
10 changed files with 173 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/client/src/tscParseInsert.c
View File

@ -1238,8 +1238,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
goto _clean; goto _clean;
} }
if (pCmd->pDataBlocks->nSize > 0) { if (pCmd->pDataBlocks->nSize > 0) { // merge according to vgId
// merge according to vgId
if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) {
goto _error_clean; goto _error_clean;
} }
@ -1294,12 +1293,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) { int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion) {
int32_t ret = TSDB_CODE_SUCCESS; int32_t ret = TSDB_CODE_SUCCESS;
tscTrace("continue parse sql: %s", pSql->asyncTblPos);
// if (NULL == pSql->asyncTblPos) {
// tscCleanSqlCmd(&pSql->cmd);
// } else {
tscTrace("continue parse sql: %s", pSql->asyncTblPos);
// }
if (tscIsInsertOrImportData(pSql->sqlstr)) { if (tscIsInsertOrImportData(pSql->sqlstr)) {
/* /*

8
src/client/src/tscSQLParser.c
View File

@ -2789,7 +2789,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) {
tSQLExprList* pList = pExpr->pParam; tSQLExprList* pList = pExpr->pParam;
if (pList->nExpr <= 0) { if (pList->nExpr <= 0) {
return TSDB_CODE_INVALID_SQL; return TSDB_CODE_INVALID_SQL;
@ -2815,7 +2815,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuild
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { static int32_t tablenameCondToString(tSQLExpr* pExpr, SStringBuilder* sb) {
taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN);
taosStringBuilderAppendString(sb, pExpr->val.pz); taosStringBuilderAppendString(sb, pExpr->val.pz);
@ -3756,8 +3756,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
const char* msg = "invalid filter expression";
const char* msg1 = "invalid expression"; const char* msg1 = "invalid expression";
const char* msg2 = "invalid filter expression";
int32_t ret = TSDB_CODE_SUCCESS; int32_t ret = TSDB_CODE_SUCCESS;
@ -3819,7 +3819,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
taosStringBuilderDestroy(&sb); taosStringBuilderDestroy(&sb);
if (!validateFilterExpr(pQueryInfo)) { if (!validateFilterExpr(pQueryInfo)) {
return invalidSqlErrMsg(pQueryInfo->msg, msg); return invalidSqlErrMsg(pQueryInfo->msg, msg2);
} }
doAddJoinTagsColumnsIntoTagList(pQueryInfo, &condExpr); doAddJoinTagsColumnsIntoTagList(pQueryInfo, &condExpr);

11
src/client/src/tscServer.c
View File

@ -847,6 +847,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder);
} }
// serialize tag column query condition
if (pQueryInfo->tagCond.numOfTagCond > 0) { if (pQueryInfo->tagCond.numOfTagCond > 0) {
STagCond* pTagCond = &pQueryInfo->tagCond; STagCond* pTagCond = &pQueryInfo->tagCond;
@ -865,6 +866,16 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
} }
} }
// tbname in/like query expression should be sent to mgmt node
STagCond* pTagCond = &pQueryInfo->tagCond;
if (pTagCond->tbnameCond.cond != NULL) {
size_t s = strlen(pTagCond->tbnameCond.cond);
memcpy(pMsg, pTagCond->tbnameCond.cond, s);
pQueryMsg->nameCondLen = htons(s);
pMsg += s;
}
msgLen = pMsg - pStart; msgLen = pMsg - pStart;
tscTrace("%p msg built success,len:%d bytes", pSql, msgLen); tscTrace("%p msg built success,len:%d bytes", pSql, msgLen);

2
src/client/src/tscUtil.c
View File

@ -608,7 +608,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
* the payloadLen should be actual message body size * the payloadLen should be actual message body size
* the old value of payloadLen is the allocated payload size * the old value of payloadLen is the allocated payload size
*/ */
pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize; pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize - sizeof(SMsgDesc);
assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0); assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

10
src/dnode/src/dnodeRead.c
View File

@ -243,16 +243,6 @@ static void dnodeContinueExecuteQuery(void* pVnode, void* qhandle, SReadMsg *pMs
taos_queue queue = dnodeGetVnodeRworker(pVnode); taos_queue queue = dnodeGetVnodeRworker(pVnode);
taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead);
// SReadMsg readMsg = {
// .rpcMsg = {0},
// .pCont = qhandle,
// .contLen = 0,
// .pRpcContext = pMsg->pRpcContext,
// };
//
// taos_queue queue = dnodeGetVnodeRworker(pVnode);
// taosWriteQitem(queue, TSDB_MSG_TYPE_QUERY, &readMsg);
} }
static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) { static void dnodeProcessQueryMsg(void *pVnode, SReadMsg *pMsg) {

3
src/inc/taosmsg.h
View File

@ -475,9 +475,10 @@ typedef struct {
int64_t slidingTime; // value for sliding window int64_t slidingTime; // value for sliding window
char slidingTimeUnit; // time interval type, for revisement of interval(1d) char slidingTimeUnit; // time interval type, for revisement of interval(1d)
uint16_t tagCondLen; // tag length in current query uint16_t tagCondLen; // tag length in current query
uint16_t nameCondLen; // table name in/like query expression string length
int16_t numOfGroupCols; // num of group by columns int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx; int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx int16_t orderType; // used in group by xx order by xxx
uint64_t groupbyTagIds; uint64_t groupbyTagIds;
int64_t limit; int64_t limit;
int64_t offset; int64_t offset;

1
src/query/inc/queryExecutor.h
View File

@ -172,7 +172,6 @@ typedef struct SQueryRuntimeEnv {
typedef struct SQInfo { typedef struct SQInfo {
void* signature; void* signature;
// void* param; // pointer to the RpcReadMsg
TSKEY startTime; TSKEY startTime;
TSKEY elapsedTime; TSKEY elapsedTime;
int32_t pointsInterpo; int32_t pointsInterpo;

10
src/query/src/qast.c
View File

@ -16,6 +16,7 @@
#include "qast.h" #include "qast.h"
#include <tarray.h> #include <tarray.h>
#include <tskiplist.h> #include <tskiplist.h>
#include "../../client/inc/tschemautil.h"
#include "os.h" #include "os.h"
#include "qsqlparser.h" #include "qsqlparser.h"
#include "qsyntaxtreefunction.h" #include "qsyntaxtreefunction.h"
@ -107,7 +108,7 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols,
return NULL; return NULL;
} }
size_t nodeSize = sizeof(tSQLSyntaxNode); size_t nodeSize = sizeof(tSQLSyntaxNode);
tSQLSyntaxNode *pNode = NULL; tSQLSyntaxNode *pNode = NULL;
if (pToken->type == TK_ID || pToken->type == TK_TBNAME) { if (pToken->type == TK_ID || pToken->type == TK_TBNAME) {
@ -239,9 +240,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tSQLSyntaxNode
} }
static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { static tSQLSyntaxNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) {
SSQLToken t0; SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL);
t0 = tStrGetToken(str, i, false, 0, NULL);
if (t0.n == 0) { if (t0.n == 0) {
return NULL; return NULL;
} }
@ -343,7 +342,8 @@ void tSQLBinaryExprFromString(tSQLBinaryExpr **pExpr, SSchema *pSchema, int32_t
return; return;
} }
int32_t pos = 0; int32_t pos = 0;
tSQLSyntaxNode *pStxNode = createSyntaxTree(pSchema, numOfCols, src, &pos); tSQLSyntaxNode *pStxNode = createSyntaxTree(pSchema, numOfCols, src, &pos);
if (pStxNode != NULL) { if (pStxNode != NULL) {
assert(pStxNode->nodeType == TSQL_NODE_EXPR); assert(pStxNode->nodeType == TSQL_NODE_EXPR);

199
src/query/src/queryExecutor.c
View File

@ -5265,11 +5265,11 @@ void qTableQuery(SQInfo *pQInfo) {
// vnodeDecRefCount(pQInfo); // vnodeDecRefCount(pQInfo);
} }
static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) {
int32_t j = 0; int32_t j = 0;
while (j < pQueryTableMsg->numOfCols) { while (j < pQueryMsg->numOfCols) {
if (pExprMsg->colInfo.colId == pQueryTableMsg->colList[j].colId) { if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) {
break; break;
} }
@ -5279,44 +5279,44 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryTableMsg, SSqlFuncEx
return j; return j;
} }
bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryTableMsg, SSqlFuncExprMsg *pExprMsg) { bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) {
int32_t j = getColumnIndexInSource(pQueryTableMsg, pExprMsg); int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg);
return j < pQueryTableMsg->numOfCols; return j < pQueryMsg->numOfCols;
} }
static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryTableMsg) { static int32_t validateQueryMeterMsg(SQueryTableMsg *pQueryMsg) {
if (pQueryTableMsg->intervalTime < 0) { if (pQueryMsg->intervalTime < 0) {
dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryTableMsg, pQueryTableMsg->intervalTime); dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime);
return -1; return -1;
} }
if (pQueryTableMsg->numOfCols <= 0 || pQueryTableMsg->numOfCols > TSDB_MAX_COLUMNS) { if (pQueryMsg->numOfCols <= 0 || pQueryMsg->numOfCols > TSDB_MAX_COLUMNS) {
dError("qmsg:%p illegal value of numOfCols %d", pQueryTableMsg, pQueryTableMsg->numOfCols); dError("qmsg:%p illegal value of numOfCols %d", pQueryMsg, pQueryMsg->numOfCols);
return -1; return -1;
} }
if (pQueryTableMsg->numOfTables <= 0) { if (pQueryMsg->numOfTables <= 0) {
dError("qmsg:%p illegal value of numOfTables %d", pQueryTableMsg, pQueryTableMsg->numOfTables); dError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables);
return -1; return -1;
} }
if (pQueryTableMsg->numOfGroupCols < 0) { if (pQueryMsg->numOfGroupCols < 0) {
dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryTableMsg, pQueryTableMsg->numOfGroupCols); dError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols);
return -1; return -1;
} }
if (pQueryTableMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryTableMsg->numOfOutputCols <= 0) { if (pQueryMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutputCols <= 0) {
dError("qmsg:%p illegal value of output columns %d", pQueryTableMsg, pQueryTableMsg->numOfOutputCols); dError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutputCols);
return -1; return -1;
} }
return 0; return 0;
} }
static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArray** pTableIdList) { static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) {
assert(pQueryTableMsg->numOfTables > 0); assert(pQueryMsg->numOfTables > 0);
*pTableIdList = taosArrayInit(pQueryTableMsg->numOfTables, sizeof(STableIdInfo)); *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo));
STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->sid = htonl(pTableIdInfo->sid);
@ -5326,7 +5326,7 @@ static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArra
taosArrayPush(*pTableIdList, pTableIdInfo); taosArrayPush(*pTableIdList, pTableIdInfo);
pMsg += sizeof(STableIdInfo); pMsg += sizeof(STableIdInfo);
for (int32_t j = 1; j < pQueryTableMsg->numOfTables; ++j) { for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) {
pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo = (STableIdInfo *)pMsg;
pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->sid = htonl(pTableIdInfo->sid);
@ -5341,49 +5341,47 @@ static char* createTableIdList(SQueryTableMsg* pQueryTableMsg, char* pMsg, SArra
} }
/** /**
* pQueryTableMsg->head has been converted before this function is called. * pQueryMsg->head has been converted before this function is called.
* *
* @param pQueryTableMsg * @param pQueryMsg
* @param pTableIdList * @param pTableIdList
* @param pExpr * @param pExpr
* @return * @return
*/ */
static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr,
wchar_t** tagCond) { wchar_t** tagCond, char** nameCond) {
pQueryTableMsg->numOfTables = htonl(pQueryTableMsg->numOfTables); pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables);
pQueryTableMsg->window.skey = htobe64(pQueryTableMsg->window.skey); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey);
pQueryTableMsg->window.ekey = htobe64(pQueryTableMsg->window.ekey); pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey);
pQueryTableMsg->intervalTime = htobe64(pQueryTableMsg->intervalTime); pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime);
pQueryTableMsg->slidingTime = htobe64(pQueryTableMsg->slidingTime); pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime);
pQueryMsg->limit = htobe64(pQueryMsg->limit);
pQueryMsg->offset = htobe64(pQueryMsg->offset);
pQueryTableMsg->limit = htobe64(pQueryTableMsg->limit); pQueryMsg->order = htons(pQueryMsg->order);
pQueryTableMsg->offset = htobe64(pQueryTableMsg->offset); pQueryMsg->orderColId = htons(pQueryMsg->orderColId);
pQueryMsg->queryType = htons(pQueryMsg->queryType);
pQueryTableMsg->order = htons(pQueryTableMsg->order);
pQueryTableMsg->orderColId = htons(pQueryTableMsg->orderColId);
pQueryTableMsg->queryType = htons(pQueryTableMsg->queryType); pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols);
pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols);
pQueryTableMsg->numOfCols = htons(pQueryTableMsg->numOfCols); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols);
pQueryTableMsg->numOfOutputCols = htons(pQueryTableMsg->numOfOutputCols); pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen);
pQueryTableMsg->numOfGroupCols = htons(pQueryTableMsg->numOfGroupCols); pQueryMsg->nameCondLen = htons(pQueryMsg->nameCondLen);
pQueryTableMsg->tagCondLen = htons(pQueryTableMsg->tagCondLen); pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset);
pQueryMsg->tsLen = htonl(pQueryMsg->tsLen);
pQueryTableMsg->tsOffset = htonl(pQueryTableMsg->tsOffset); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks);
pQueryTableMsg->tsLen = htonl(pQueryTableMsg->tsLen); pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder);
pQueryTableMsg->tsNumOfBlocks = htonl(pQueryTableMsg->tsNumOfBlocks);
pQueryTableMsg->tsOrder = htonl(pQueryTableMsg->tsOrder);
// query msg safety check // query msg safety check
if (validateQueryMeterMsg(pQueryTableMsg) != 0) { if (validateQueryMeterMsg(pQueryMsg) != 0) {
return TSDB_CODE_INVALID_QUERY_MSG; return TSDB_CODE_INVALID_QUERY_MSG;
} }
char *pMsg = (char *)(pQueryTableMsg->colList) + sizeof(SColumnInfo) * pQueryTableMsg->numOfCols; char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols;
for (int32_t col = 0; col < pQueryTableMsg->numOfCols; ++col) { for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) {
SColumnInfo* pColInfo = &pQueryTableMsg->colList[col]; SColumnInfo* pColInfo = &pQueryMsg->colList[col];
pColInfo->colId = htons(pColInfo->colId); pColInfo->colId = htons(pColInfo->colId);
pColInfo->type = htons(pColInfo->type); pColInfo->type = htons(pColInfo->type);
@ -5423,10 +5421,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId
bool hasArithmeticFunction = false; bool hasArithmeticFunction = false;
*pExpr = calloc(pQueryTableMsg->numOfOutputCols, POINTER_BYTES); *pExpr = calloc(pQueryMsg->numOfOutputCols, POINTER_BYTES);
SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg;
for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) {
(*pExpr)[i] = pExprMsg; (*pExpr)[i] = pExprMsg;
pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx); pExprMsg->colInfo.colIdx = htons(pExprMsg->colInfo.colIdx);
@ -5457,7 +5455,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId
return TSDB_CODE_INVALID_QUERY_MSG; return TSDB_CODE_INVALID_QUERY_MSG;
} }
} else { } else {
if (!vnodeValidateExprColumnInfo(pQueryTableMsg, pExprMsg)) { if (!vnodeValidateExprColumnInfo(pQueryMsg, pExprMsg)) {
return TSDB_CODE_INVALID_QUERY_MSG; return TSDB_CODE_INVALID_QUERY_MSG;
} }
} }
@ -5465,55 +5463,59 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryTableMsg, SArray **pTableId
pExprMsg = (SSqlFuncExprMsg *)pMsg; pExprMsg = (SSqlFuncExprMsg *)pMsg;
} }
pQueryTableMsg->colNameLen = htonl(pQueryTableMsg->colNameLen); pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen);
if (hasArithmeticFunction) { // column name array if (hasArithmeticFunction) { // column name array
assert(pQueryTableMsg->colNameLen > 0); assert(pQueryMsg->colNameLen > 0);
pQueryTableMsg->colNameList = (int64_t)pMsg; pQueryMsg->colNameList = (int64_t)pMsg;
pMsg += pQueryTableMsg->colNameLen; pMsg += pQueryMsg->colNameLen;
} }
pMsg = createTableIdList(pQueryTableMsg, pMsg, pTableIdList); pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList);
if (pQueryTableMsg->numOfGroupCols > 0) { // group by tag columns if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns
// if (pQueryTableMsg->numOfGroupCols > 0) { // if (pQueryMsg->numOfGroupCols > 0) {
// pQueryTableMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryTableMsg->numOfTagsCols]); // pQueryMsg->groupbyTagIds = (uint64_t) & (pTagSchema[pQueryMsg->numOfTagsCols]);
// } else { // } else {
// pQueryTableMsg->groupbyTagIds = 0; // pQueryMsg->groupbyTagIds = 0;
// } // }
pQueryTableMsg->orderByIdx = htons(pQueryTableMsg->orderByIdx); pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx);
pQueryTableMsg->orderType = htons(pQueryTableMsg->orderType); pQueryMsg->orderType = htons(pQueryMsg->orderType);
pMsg += sizeof(SColIndexEx) * pQueryTableMsg->numOfGroupCols; pMsg += sizeof(SColIndexEx) * pQueryMsg->numOfGroupCols;
} else { } else {
pQueryTableMsg->groupbyTagIds = 0; pQueryMsg->groupbyTagIds = 0;
} }
pQueryTableMsg->interpoType = htons(pQueryTableMsg->interpoType); pQueryMsg->interpoType = htons(pQueryMsg->interpoType);
if (pQueryTableMsg->interpoType != TSDB_INTERPO_NONE) { if (pQueryMsg->interpoType != TSDB_INTERPO_NONE) {
pQueryTableMsg->defaultVal = (uint64_t)(pMsg); pQueryMsg->defaultVal = (uint64_t)(pMsg);
int64_t *v = (int64_t *)pMsg; int64_t *v = (int64_t *)pMsg;
for (int32_t i = 0; i < pQueryTableMsg->numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) {
v[i] = htobe64(v[i]); v[i] = htobe64(v[i]);
} }
pMsg += sizeof(int64_t) * pQueryTableMsg->numOfOutputCols; pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols;
} }
// the tag query condition expression string is located at the end of query msg // the tag query condition expression string is located at the end of query msg
if (pQueryTableMsg->tagCondLen > 0) { if (pQueryMsg->tagCondLen > 0) {
*tagCond = calloc(1, pQueryTableMsg->tagCondLen * TSDB_NCHAR_SIZE); *tagCond = calloc(1, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE);
memcpy(*tagCond, pMsg, pQueryTableMsg->tagCondLen * TSDB_NCHAR_SIZE); memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen * TSDB_NCHAR_SIZE);
}
if (pQueryMsg->nameCondLen > 0) {
*nameCond = strndup(pMsg, pQueryMsg->nameCondLen);
} }
dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, " dTrace("qmsg:%p query on %d meter(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, numOfTagCols:%d, "
"timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64 "timestamp order:%d, tags order:%d, tags order col:%d, numOfOutputCols:%d, numOfCols:%d, interval:%" PRId64
", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64, ", fillType:%d, comptslen:%d, limit:%" PRId64 ", offset:%" PRId64,
pQueryTableMsg, pQueryTableMsg->numOfTables, pQueryTableMsg->window.skey, pQueryTableMsg->window.ekey, pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey,
pQueryTableMsg->numOfGroupCols, pQueryTableMsg->order, pQueryTableMsg->orderType, pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->orderType,
pQueryTableMsg->orderByIdx, pQueryTableMsg->numOfOutputCols, pQueryMsg->orderByIdx, pQueryMsg->numOfOutputCols,
pQueryTableMsg->numOfCols, pQueryTableMsg->intervalTime, pQueryTableMsg->interpoType, pQueryTableMsg->tsLen, pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen,
pQueryTableMsg->limit, pQueryTableMsg->offset); pQueryMsg->limit, pQueryMsg->offset);
return 0; return 0;
} }
@ -6047,54 +6049,59 @@ _error:
return code; return code;
} }
int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryTableMsg, SQInfo **pQInfo) { int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, SQInfo **pQInfo) {
assert(pQueryTableMsg != NULL); assert(pQueryMsg != NULL);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SArray *pTableIdList = NULL; SArray *pTableIdList = NULL;
SSqlFuncExprMsg** pExprMsg = NULL; SSqlFuncExprMsg** pExprMsg = NULL;
wchar_t* tagCond = NULL; wchar_t* tagCond = NULL;
char* nameCond = NULL;
if ((code = convertQueryMsg(pQueryTableMsg, &pTableIdList, &pExprMsg, &tagCond)) != TSDB_CODE_SUCCESS) { if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &nameCond)) != TSDB_CODE_SUCCESS) {
return code; return code;
} }
if (pQueryTableMsg->numOfTables <= 0) { if (pQueryMsg->numOfTables <= 0) {
dError("Invalid number of tables to query, numOfTables:%d", pQueryTableMsg->numOfTables); dError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables);
code = TSDB_CODE_INVALID_QUERY_MSG; code = TSDB_CODE_INVALID_QUERY_MSG;
goto _query_over; goto _query_over;
} }
// todo check vnode status // todo check vnode status
if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) {
dError("qmsg:%p, SQueryTableMsg wrong format", pQueryTableMsg); dError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg);
code = TSDB_CODE_INVALID_QUERY_MSG; code = TSDB_CODE_INVALID_QUERY_MSG;
goto _query_over; goto _query_over;
} }
SSqlFunctionExpr *pExprs = NULL; SSqlFunctionExpr *pExprs = NULL;
if ((code = createSqlFunctionExprFromMsg(pQueryTableMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) {
goto _query_over; goto _query_over;
} }
SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryTableMsg, &code); SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, &code);
if ((pGroupbyExpr == NULL && pQueryTableMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
goto _query_over; goto _query_over;
} }
// super table query // super table query
if ((pQueryTableMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { SArray* res = NULL;
if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) {
STableId* id = taosArrayGet(pTableIdList, 0); STableId* id = taosArrayGet(pTableIdList, 0);
id->uid = -1;
SArray* res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryTableMsg->tagCondLen); res = tsdbQueryTableList(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen);
if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode if (taosArrayGetSize(res) == 0) { // no qualified table in stable query in this vnode
code = TSDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
goto _query_over; goto _query_over;
} }
} else {
res = pTableIdList;
} }
code = createQInfo(pQueryTableMsg, pGroupbyExpr, pExprs, pTableIdList, tsdb, pQInfo); code = createQInfo(pQueryMsg, pGroupbyExpr, pExprs, res, tsdb, pQInfo);
_query_over: _query_over:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -6103,16 +6110,16 @@ _query_over:
// if failed to add ref for all meters in this query, abort current query // if failed to add ref for all meters in this query, abort current query
// if (code != TSDB_CODE_SUCCESS) { // if (code != TSDB_CODE_SUCCESS) {
// vnodeDecQueryRefCount(pQueryTableMsg, pMeterObjList, incNumber); // vnodeDecQueryRefCount(pQueryMsg, pMeterObjList, incNumber);
// } // }
// //
// tfree(pQueryTableMsg->pSqlFuncExprs); // tfree(pQueryMsg->pSqlFuncExprs);
// tfree(pMeterObjList); // tfree(pMeterObjList);
// ret = vnodeSendQueryRspMsg(pObj, code, pObj->qhandle); // ret = vnodeSendQueryRspMsg(pObj, code, pObj->qhandle);
// //
// tfree(pQueryTableMsg->pSidExtInfo); // tfree(pQueryMsg->pSidExtInfo);
// for(int32_t i = 0; i < pQueryTableMsg->numOfCols; ++i) { // for(int32_t i = 0; i < pQueryMsg->numOfCols; ++i) {
// vnodeFreeColumnInfo(&pQueryTableMsg->colList[i]); // vnodeFreeColumnInfo(&pQueryMsg->colList[i]);
// } // }
// //
// atomic_fetch_add_32(&vnodeSelectReqNum, 1); // atomic_fetch_add_32(&vnodeSelectReqNum, 1);

83
src/vnode/tsdb/src/tsdbRead.c
View File

@ -73,14 +73,15 @@ typedef struct SQueryFilesInfo {
char dbFilePathPrefix[PATH_MAX]; char dbFilePathPrefix[PATH_MAX];
} SQueryFilesInfo; } SQueryFilesInfo;
typedef struct STableQueryRec { typedef struct STableQueryInfo {
STableId tableId;
TSKEY lastKey; TSKEY lastKey;
STable * pTableObj; STable * pTableObj;
int64_t offsetInHeaderFile; int64_t offsetInHeaderFile;
int32_t numOfBlocks; int32_t numOfBlocks;
int32_t start; int32_t start;
SCompBlock *pBlock; SCompBlock *pBlock;
} STableQueryRec; } STableQueryInfo;
typedef struct { typedef struct {
SCompBlock *compBlock; SCompBlock *compBlock;
@ -89,7 +90,7 @@ typedef struct {
typedef struct STableDataBlockInfoEx { typedef struct STableDataBlockInfoEx {
SCompBlockFields pBlock; SCompBlockFields pBlock;
STableQueryRec * pMeterDataInfo; STableQueryInfo* pMeterDataInfo;
int32_t blockIndex; int32_t blockIndex;
int32_t groupIdx; /* number of group is less than the total number of meters */ int32_t groupIdx; /* number of group is less than the total number of meters */
} STableDataBlockInfoEx; } STableDataBlockInfoEx;
@ -100,12 +101,11 @@ typedef struct STsdbQueryHandle {
SQueryHandlePos cur; // current position SQueryHandlePos cur; // current position
SQueryHandlePos start; // the start position, used for secondary/third iteration SQueryHandlePos start; // the start position, used for secondary/third iteration
int32_t unzipBufSize; int32_t unzipBufSize;
char *unzipBuffer; char *unzipBuffer;
char *secondaryUnzipBuffer; char *secondaryUnzipBuffer;
SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
SQueryFilesInfo vnodeFileInfo; SQueryFilesInfo vnodeFileInfo;
int16_t numOfRowsPerPage; int16_t numOfRowsPerPage;
@ -113,21 +113,22 @@ typedef struct STsdbQueryHandle {
int16_t order; int16_t order;
STimeWindow window; // the primary query time window that applies to all queries STimeWindow window; // the primary query time window that applies to all queries
int32_t blockBufferSize; int32_t blockBufferSize;
SCompBlock *pBlock; SCompBlock* pBlock;
int32_t numOfBlocks; int32_t numOfBlocks;
SField ** pFields; SField ** pFields;
SArray * pColumns; // column list, SColumnInfoEx array list SArray * pColumns; // column list, SColumnInfoEx array list
SArray * pTableIdList; // table id object list
bool locateStart; bool locateStart;
int32_t realNumOfRows; int32_t realNumOfRows;
bool loadDataAfterSeek; // load data after seek. bool loadDataAfterSeek; // load data after seek.
SArray* pTableQueryInfo;
STableDataBlockInfoEx *pDataBlockInfoEx; int32_t activeIndex;
STableQueryRec * pTableQueryInfo;
int32_t tableIndex; int32_t tableIndex;
bool isFirstSlot; bool isFirstSlot;
void * qinfo; // query info handle, for debug purpose void * qinfo; // query info handle, for debug purpose
SSkipListIterator* memIter; SSkipListIterator* memIter;
STableDataBlockInfoEx *pDataBlockInfoEx;
} STsdbQueryHandle; } STsdbQueryHandle;
int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) { int32_t doAllocateBuf(STsdbQueryHandle *pQueryHandle, int32_t rowsPerFileBlock) {
@ -263,25 +264,27 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond
pQueryHandle->window = pCond->twindow; pQueryHandle->window = pCond->twindow;
pQueryHandle->pTsdb = tsdb; pQueryHandle->pTsdb = tsdb;
pQueryHandle->pTableIdList = idList;
pQueryHandle->pColumns = pColumnInfo; pQueryHandle->pColumns = pColumnInfo;
pQueryHandle->loadDataAfterSeek = false; pQueryHandle->loadDataAfterSeek = false;
pQueryHandle->isFirstSlot = true; pQueryHandle->isFirstSlot = true;
// only support table query size_t size = taosArrayGetSize(idList);
assert(taosArrayGetSize(idList) == 1); assert(size >= 1);
pQueryHandle->pTableQueryInfo = taosArrayInit(size, sizeof(STableQueryInfo));
for(int32_t i = 0; i < size; ++i) {
STableId id = *(STableId*) taosArrayGet(idList, i);
STableQueryInfo info = {
.lastKey = pQueryHandle->window.skey,
.tableId = id,
.pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid), //todo this may be failed
};
taosArrayPush(pQueryHandle->pTableQueryInfo, &info);
}
pQueryHandle->pTableQueryInfo = calloc(1, sizeof(STableQueryRec)); pQueryHandle->activeIndex = 0;
STableQueryRec* pTableQRec = pQueryHandle->pTableQueryInfo;
pTableQRec->lastKey = pQueryHandle->window.skey;
STableIdInfo* idInfo = taosArrayGet(pQueryHandle->pTableIdList, 0);
STable *pTable = tsdbGetTableByUid(tsdbGetMeta(pQueryHandle->pTsdb), idInfo->uid);
assert(pTable != NULL);
pTableQRec->pTableObj = pTable;
// malloc buffer in order to load data from file // malloc buffer in order to load data from file
int32_t numOfCols = taosArrayGetSize(pColumnInfo); int32_t numOfCols = taosArrayGetSize(pColumnInfo);
@ -313,7 +316,9 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond
bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) { bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
STable *pTable = pHandle->pTableQueryInfo->pTableObj; STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex);
STable *pTable = pTableQInfo->pTableObj;
// no data in cache, abort // no data in cache, abort
if (pTable->mem == NULL && pTable->imem == NULL) { if (pTable->mem == NULL && pTable->imem == NULL) {
@ -321,7 +326,7 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
} }
// all data in mem are checked already. // all data in mem are checked already.
if (pHandle->pTableQueryInfo->lastKey > pTable->mem->keyLast) { if (pTableQInfo->lastKey > pTable->mem->keyLast) {
return false; return false;
} }
@ -364,9 +369,9 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max
// copy data from cache into data block // copy data from cache into data block
SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) { SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) {
STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
STableIdInfo* idInfo = taosArrayGet(pHandle->pTableIdList, 0);
STable *pTable = pHandle->pTableQueryInfo->pTableObj; STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex);
STable *pTable = pTableQInfo->pTableObj;
TSKEY skey = 0, ekey = 0; TSKEY skey = 0, ekey = 0;
int32_t rows = 0; int32_t rows = 0;
@ -382,14 +387,14 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) {
} }
SDataBlockInfo blockInfo = { SDataBlockInfo blockInfo = {
.uid = idInfo->uid, .uid = pTable->tableId.uid,
.sid = idInfo->sid, .sid = pTable->tableId.tid,
.size = rows, .size = rows,
.window = {.skey = skey, .ekey = ekey} .window = {.skey = skey, .ekey = ekey}
}; };
// update the last key value // update the last key value
pHandle->pTableQueryInfo->lastKey = ekey + 1; pTableQInfo->lastKey = ekey + 1;
return blockInfo; return blockInfo;
} }
@ -427,7 +432,9 @@ static SArray* createTableIdArrayList(struct STsdbRepo* tsdb, int64_t uid) {
SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex); SSkipListIterator* iter = tSkipListCreateIter(pTable->pIndex);
while(tSkipListIterNext(iter)) { while(tSkipListIterNext(iter)) {
STable* t = *(STable**) tSkipListIterGet(iter); SSkipListNode* pNode = tSkipListIterGet(iter);
STable* t = *(STable**) SL_GET_NODE_DATA(pNode);
taosArrayPush(pList, &t->tableId); taosArrayPush(pList, &t->tableId);
} }
@ -696,7 +703,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) {
return true; return true;
} }
static int32_t mgmtFilterMeterByIndex(STable* pSTable, SArray* pRes, const char* pCond) { static int32_t doQueryTableList(STable* pSTable, SArray* pRes, const char* pCond) {
STColumn* stcol = schemaColAt(pSTable->tagSchema, 0); STColumn* stcol = schemaColAt(pSTable->tagSchema, 0);
tSQLBinaryExpr* pExpr = NULL; tSQLBinaryExpr* pExpr = NULL;
@ -736,7 +743,7 @@ SArray *tsdbQueryTableList(struct STsdbRepo* tsdb, int64_t uid, const wchar_t *p
STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
assert(pSTable != NULL); assert(pSTable != NULL);
if (mgmtFilterMeterByIndex(pSTable, result, str) == TSDB_CODE_SUCCESS) { if (doQueryTableList(pSTable, result, str) == TSDB_CODE_SUCCESS) {
return result; return result;
} }
} }