diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b15286fe80..af91ac34f0 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -277,6 +277,9 @@ void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRo void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); int tscSetMgmtEpSetFromCfg(const char *first, const char *second); +bool tscSetSqlOwner(SSqlObj* pSql); +void tscClearSqlOwner(SSqlObj* pSql); + void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); char* strdup_throw(const char* str); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index b38e6377a9..6d02bc7fbd 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -80,8 +80,9 @@ typedef struct STableMetaInfo { * 2. keep the vgroup index for multi-vnode insertion */ int32_t vgroupIndex; - char name[TSDB_TABLE_ID_LEN]; // (super) table name - SArray* tagColList; // SArray, involved tag columns + char name[TSDB_TABLE_FNAME_LEN]; // (super) table name + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray* tagColList; // SArray, involved tag columns } STableMetaInfo; /* the structure for sql function in select clause */ @@ -106,7 +107,7 @@ typedef struct SColumnIndex { typedef struct SFieldSupInfo { bool visible; SExprInfo *pArithExprInfo; - SSqlExpr * pSqlExpr; + SSqlExpr *pSqlExpr; } SFieldSupInfo; typedef struct SFieldInfo { @@ -128,7 +129,7 @@ typedef struct SCond { } SCond; typedef struct SJoinNode { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; uint64_t uid; int16_t tagColId; } SJoinNode; @@ -162,7 +163,7 @@ typedef struct SParamInfo { } SParamInfo; typedef struct STableDataBlocks { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id @@ -255,6 +256,7 @@ typedef struct SResRec { typedef struct { int64_t numOfRows; // num of results in current retrieved + int64_t numOfRowsGroup; // num of results of current group int64_t numOfTotal; // num of total results int64_t numOfClauseTotal; // num of total result in current subclause char * pRsp; @@ -301,6 +303,7 @@ typedef struct STscObj { typedef struct SSqlObj { void *signature; + pthread_t owner; // owner of sql object, by which it is executed STscObj *pTscObj; void *pRpcCtx; void (*fp)(); @@ -419,7 +422,6 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); -//void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 759c6e5e1e..b05aef76eb 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -220,14 +220,13 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { tscFetchDatablockFromSubquery(pSql); } else if (pRes->completed) { - if(pCmd->command == TSDB_SQL_FETCH) { + if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) { if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode); - return; } else { /* - * all available virtual node has been checked already, now we need to check - * for the next subclause queries + * all available virtual nodes in current clause has been checked already, now try the + * next one in the following union subclause */ if (pCmd->clauseIndex < pCmd->numOfClause - 1) { tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); @@ -235,11 +234,12 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi } /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore + * 1. has reach the limitation + * 2. no remain virtual nodes to be retrieved anymore */ (*pSql->fetchFp)(param, pSql, 0); } + return; } else if (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE) { // in case of show command, return no data diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 9c0951cdd6..5c708dffee 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1942,11 +1942,12 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { - tValuePair **pList = pInfo->res; - tVariant val = {0}; tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); - + + tValuePair **pList = pInfo->res; + assert(pList != NULL); + if (pInfo->num < maxLen) { if (pInfo->num == 0) { valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index a56a01ba02..caaaa5bc18 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -293,7 +293,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { char db[TSDB_DB_NAME_LEN] = {0}; extractDBName(pSql->pTscObj->db, db); - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resType = TSDB_DATA_TYPE_BINARY; @@ -314,7 +314,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { static void tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resType = TSDB_DATA_TYPE_BINARY; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index b7d8a0b7b2..5955515bbd 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -68,7 +68,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); pCtx->aOutputBuf = - pReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pReducer->resColModel->capacity; + pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; pCtx->order = pQueryInfo->order.order; pCtx->functionId = pExpr->functionId; @@ -321,6 +321,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize; + assert(pReducer->finalRowSize > 0); if (pReducer->finalRowSize > 0) { pReducer->resColModel->capacity /= pReducer->finalRowSize; @@ -328,7 +329,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd assert(pReducer->finalRowSize <= pReducer->rowSize); pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity); -// pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize); if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL || /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { @@ -856,24 +856,6 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); } -// todo merge with following function -// static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) { -// -// for (int32_t i = 0; i < pQueryInfo->exprList.numOfExprs; ++i) { -// TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); -// -// int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); -// char * src = pFinalDataPage->data + (pRes->numOfRows - 1) * pField->bytes + pRes->numOfRows * offset; -// char * dst = pRes->data + pRes->numOfRows * offset; -// -// for (int32_t j = 0; j < pRes->numOfRows; ++j) { -// memcpy(dst, src, (size_t)pField->bytes); -// dst += pField->bytes; -// src -= pField->bytes; -// } -// } -//} - static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, SLocalReducer *pLocalReducer) { assert(0); @@ -907,20 +889,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO tFilePage * pFinalDataPage = pLocalReducer->pResultBuf; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); -// if (pRes->pLocalReducer != pLocalReducer) { -// /* -// * Release the SSqlObj is called, and it is int destroying function invoked by other thread. -// * However, the other thread will WAIT until current process fully completes. -// * Since the flag of release struct is set by doLocalReduce function -// */ -// assert(pRes->pLocalReducer == NULL); -// } - // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { pRes->data = pLocalReducer->pFinalRes; pRes->numOfRows = pFinalDataPage->num; - pRes->numOfClauseTotal += pRes->numOfRows; if (pQueryInfo->limit.offset > 0) { if (pQueryInfo->limit.offset < pRes->numOfRows) { @@ -931,22 +903,22 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO tColModelCompact(pLocalReducer->resColModel, pFinalDataPage, prevSize); pRes->numOfRows -= pQueryInfo->limit.offset; - pRes->numOfClauseTotal -= pQueryInfo->limit.offset; pQueryInfo->limit.offset = 0; } else { pQueryInfo->limit.offset -= pRes->numOfRows; pRes->numOfRows = 0; - pRes->numOfClauseTotal = 0; } } - if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) { + pRes->numOfRowsGroup += pRes->numOfRows; + + if (pQueryInfo->limit.limit >= 0 && pRes->numOfRowsGroup > pQueryInfo->limit.limit) { /* impose the limitation of output rows on the final result */ - int32_t prevSize = (int32_t)pFinalDataPage->num; - int32_t overflow = (int32_t)(pRes->numOfClauseTotal - pQueryInfo->limit.limit); + int32_t prevSize = pFinalDataPage->num; + int32_t overflow = pRes->numOfRowsGroup - pQueryInfo->limit.limit; assert(overflow < pRes->numOfRows); - pRes->numOfClauseTotal = pQueryInfo->limit.limit; + pRes->numOfRowsGroup = pQueryInfo->limit.limit; pRes->numOfRows -= overflow; pFinalDataPage->num -= overflow; @@ -957,6 +929,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * pLocalReducer->finalRowSize); + + pRes->numOfClauseTotal += pRes->numOfRows; pFinalDataPage->num = 0; return; } @@ -969,7 +943,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity); } - + while (1) { int64_t newRows = taosGenerateDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity); @@ -986,7 +960,6 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO pRes->data = pLocalReducer->pFinalRes; pRes->numOfRows = newRows; - pRes->numOfClauseTotal += newRows; pQueryInfo->limit.offset = 0; break; @@ -1010,15 +983,13 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } if (pRes->numOfRows > 0) { - if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) { - int32_t overflow = (int32_t)(pRes->numOfClauseTotal - pQueryInfo->limit.limit); + if (pQueryInfo->limit.limit >= 0 && pRes->numOfRows > pQueryInfo->limit.limit) { + int32_t overflow = pRes->numOfRows - pQueryInfo->limit.limit; pRes->numOfRows -= overflow; - - assert(pRes->numOfRows >= 0); - - pRes->numOfClauseTotal = pQueryInfo->limit.limit; pFinalDataPage->num -= overflow; + assert(pRes->numOfRows >= 0 && pFinalDataPage->num > 0); + /* set remain data to be discarded, and reset the interpolation information */ savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pFillInfo); } @@ -1032,6 +1003,9 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } else { // todo bug?? reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); } + + pRes->numOfRowsGroup += pRes->numOfRows; + pRes->numOfClauseTotal += pRes->numOfRows; } pFinalDataPage->num = 0; @@ -1227,7 +1201,10 @@ static bool saveGroupResultInfo(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->numOfGroups += 1; + + if (pRes->numOfRowsGroup > 0) { + pRes->numOfGroups += 1; + } // the output group is limited by the slimit clause if (reachGroupResultLimit(pQueryInfo, pRes)) { @@ -1266,7 +1243,12 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no pRes->numOfRows = 0; pQueryInfo->slimit.offset -= 1; pLocalReducer->discard = !noMoreCurrentGroupRes; - + + if (pLocalReducer->discard) { + SColumnModel *pInternModel = pLocalReducer->pDesc->pColumnModel; + tColModelAppend(pInternModel, pLocalReducer->discardData, pLocalReducer->pTempBuffer->data, 0, 1, 1); + } + return false; } @@ -1299,7 +1281,7 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { // In handling data in other groups, we need to reset the interpolation information for a new group data pRes->numOfRows = 0; - pRes->numOfClauseTotal = 0; + pRes->numOfRowsGroup = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1363,7 +1345,8 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || prevGroupCompleted) { // if fillType == TSDB_FILL_NONE, return directly - if (pQueryInfo->fillType != TSDB_FILL_NONE) { + if (pQueryInfo->fillType != TSDB_FILL_NONE && + ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; assert(pFillInfo->numOfRows == 0); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 33c8799624..e97d6c32e0 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -989,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { - tstrncpy(psTblToken->z, tblName, TSDB_TABLE_ID_LEN); + tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); psTblToken->n = len; psTblToken->type = TK_ID; @@ -1038,7 +1038,7 @@ int tsParseInsertSql(SSqlObj *pSql) { } if (NULL == pCmd->pTableList) { - pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); pCmd->pDataBlocks = taosArrayInit(4, POINTER_BYTES); if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1077,7 +1077,7 @@ int tsParseInsertSql(SSqlObj *pSql) { } pCmd->curSql = sToken.z; - char buf[TSDB_TABLE_ID_LEN]; + char buf[TSDB_TABLE_FNAME_LEN]; SSQLToken sTblToken; sTblToken.z = buf; // Check if the table name available or not diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index bc33aac174..1a97a9a997 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -86,7 +86,7 @@ static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQueryS static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString); +static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** exprString); static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); static int32_t validateEp(char* ep); @@ -1087,11 +1087,11 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL *xlen = totalLen; } - if (totalLen < TSDB_TABLE_ID_LEN) { + if (totalLen < TSDB_TABLE_FNAME_LEN) { fullName[totalLen] = 0; } - return (totalLen < TSDB_TABLE_ID_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; + return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } static void extractColumnNameFromString(tSQLExprItem* pItem) { @@ -1106,23 +1106,138 @@ static void extractColumnNameFromString(tSQLExprItem* pItem) { } } +static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { + const char* msg1 = "invalid column name, or illegal column type"; + const char* msg2 = "invalid arithmetic expression in select clause"; + const char* msg3 = "tag columns can not be used in arithmetic expression"; + const char* msg4 = "columns from different table mixed up in arithmetic expression"; + + // arithmetic function in select clause + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + + SColumnList columnList = {0}; + int32_t arithmeticType = NON_ARITHMEIC_EXPR; + + if (validateArithmeticSQLExpr(pCmd, pItem->pNode, pQueryInfo, &columnList, &arithmeticType) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + int32_t tableIndex = columnList.ids[0].tableIndex; + + // todo potential data overflow + char arithmeticExprStr[1024*12]; + char* p = arithmeticExprStr; + + if (arithmeticType == NORMAL_ARITHMETIC) { + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; + + // all columns in arithmetic expression must belong to the same table + for (int32_t f = 1; f < columnList.num; ++f) { + if (columnList.ids[f].tableIndex != tableIndex) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + } + + if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + // expr string is set as the parameter of function + SColumnIndex index = {.tableIndex = tableIndex}; + + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), + sizeof(double), false); + + char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; + tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); + + tExprNode* pNode = NULL; + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + + int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(&pNode, NULL); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + size_t numOfNode = taosArrayGetSize(colList); + for(int32_t k = 0; k < numOfNode; ++k) { + SColIndex* pIndex = taosArrayGet(colList, k); + if (pIndex->flag == 1) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + SBufferWriter bw = tbufInitWriter(NULL, false); + + TRY(0) { + exprTreeToBinary(&bw, pNode); + } CATCH(code) { + tbufCloseWriter(&bw); + UNUSED(code); + // TODO: other error handling + } END_TRY + + size_t len = tbufTell(&bw); + char* c = tbufGetData(&bw, true); + + // set the serialized binary string as the parameter of arithmetic expression + addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, len, index.tableIndex); + + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); + + taosArrayDestroy(colList); + tExprTreeDestroy(&pNode, NULL); + } else { + if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + columnList.num = 0; + columnList.ids[0] = (SColumnIndex) {0, 0}; + + char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, name, NULL); + + int32_t slot = tscNumOfFields(pQueryInfo) - 1; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); + + if (pInfo->pSqlExpr == NULL) { + SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo)); + + // arithmetic expression always return result in the format of double float + pArithExprInfo->bytes = sizeof(double); + pArithExprInfo->interBytes = sizeof(double); + pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE; + + int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); + } + + pInfo->pArithExprInfo = pArithExprInfo; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { assert(pSelection != NULL && pCmd != NULL); - const char* msg1 = "invalid column name, or illegal column type"; const char* msg2 = "functions can not be mixed up"; const char* msg3 = "not support query expression"; - const char* msg4 = "columns from different table mixed up in arithmetic expression"; const char* msg5 = "invalid function name"; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - + if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } - + for (int32_t i = 0; i < pSelection->nExpr; ++i) { - int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t outputIndex = tscSqlExprNumOfExprs(pQueryInfo); tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields @@ -1147,96 +1262,11 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } else if (pItem->pNode->nSQLOptr >= TK_PLUS && pItem->pNode->nSQLOptr <= TK_REM) { - // arithmetic function in select clause - SColumnList columnList = {0}; - int32_t arithmeticType = NON_ARITHMEIC_EXPR; - - if (validateArithmeticSQLExpr(pCmd, pItem->pNode, pQueryInfo, &columnList, &arithmeticType) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + int32_t code = handleArithmeticExpr(pCmd, clauseIndex, i, pItem); + if (code != TSDB_CODE_SUCCESS) { + return code; } - - int32_t tableIndex = columnList.ids[0].tableIndex; - char arithmeticExprStr[1024] = {0}; - char* p = arithmeticExprStr; - - if (arithmeticType == NORMAL_ARITHMETIC) { - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; - - // all columns in arithmetic expression must belong to the same table - for (int32_t f = 1; f < columnList.num; ++f) { - if (columnList.ids[f].tableIndex != tableIndex) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - - if (buildArithmeticExprString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - - // expr string is set as the parameter of function - SColumnIndex index = {.tableIndex = tableIndex}; - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, - sizeof(double), sizeof(double), false); - - /* todo alias name should use the original sql string */ - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); - - tExprNode* pNode = NULL; - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(&pNode, NULL); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid arithmetic expression in select clause"); - } - - SBufferWriter bw = tbufInitWriter(NULL, false); - TRY(0) { - exprTreeToBinary(&bw, pNode); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: other error handling - } END_TRY - - size_t len = tbufTell(&bw); - char* c = tbufGetData(&bw, true); - - // set the serialized binary string as the parameter of arithmetic expression - addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex); - - insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); - - taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); - } else { - columnList.num = 0; - columnList.ids[0] = (SColumnIndex) {0, 0}; - - insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, "dummy_column", NULL); - - int32_t slot = tscNumOfFields(pQueryInfo) - 1; - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); - - if (pInfo->pSqlExpr == NULL) { - SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo)); - - // arithmetic expression always return result in the format of double float - pArithExprInfo->bytes = sizeof(double); - pArithExprInfo->interBytes = sizeof(double); - pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE; - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); - } - - pInfo->pArithExprInfo = pArithExprInfo; - } - } } else { /* * not support such expression @@ -2106,13 +2136,10 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum } pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL; - char tableName[TSDB_TABLE_ID_LEN] = {0}; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - extractTableName(pTableMetaInfo->name, tableName); - - if (strncasecmp(tableName, pTableToken->z, pTableToken->n) == 0 && strlen(tableName) == pTableToken->n) { + char* name = pTableMetaInfo->aliasName; + if (strncasecmp(name, pTableToken->z, pTableToken->n) == 0 && strlen(name) == pTableToken->n) { pIndex->tableIndex = i; break; } @@ -3081,14 +3108,14 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* } // todo error handle / such as and /or mixed with +/-/*/ -int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString) { +int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { tSQLExpr* pLeft = pExpr->pLeft; tSQLExpr* pRight = pExpr->pRight; *(*exprString)++ = '('; if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { - buildArithmeticExprString(pLeft, exprString); + doArithmeticExprToString(pLeft, exprString); } else { int32_t ret = tSQLExprNodeToString(pLeft, exprString); if (ret != TSDB_CODE_SUCCESS) { @@ -3099,7 +3126,7 @@ int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString) { optrToString(pExpr, exprString); if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { - buildArithmeticExprString(pRight, exprString); + doArithmeticExprToString(pRight, exprString); } else { int32_t ret = tSQLExprNodeToString(pRight, exprString); if (ret != TSDB_CODE_SUCCESS) { @@ -3112,6 +3139,19 @@ int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } +static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { + char* start = *str; + + int32_t code = doArithmeticExprToString(pExpr, str); + if (code == TSDB_CODE_SUCCESS) { // remove out the parenthesis + int32_t len = strlen(start); + memmove(start, start + 1, len - 2); + start[len - 2] = 0; + } + + return code; +} + static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { if (pExpr->nSQLOptr == TK_ID) { if (*type == NON_ARITHMEIC_EXPR) { @@ -3615,7 +3655,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1)); taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - char db[TSDB_TABLE_ID_LEN] = {0}; + char db[TSDB_TABLE_FNAME_LEN] = {0}; // remove the duplicated input table names int32_t num = 0; @@ -3640,7 +3680,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1); } - char idBuf[TSDB_TABLE_ID_LEN] = {0}; + char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; int32_t xlen = (int32_t)strlen(segments[i]); SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; @@ -4345,7 +4385,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg16 = "only support one column"; const char* msg17 = "invalid column name"; const char* msg18 = "primary timestamp column cannot be dropped"; - + const char* msg19 = "invalid new tag name"; + SSqlCmd* pCmd = &pSql->cmd; SAlterTableSQL* pAlterSQL = pInfo->pAlterInfo; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); @@ -4446,12 +4487,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); } char name[TSDB_COL_NAME_LEN] = {0}; @@ -5872,15 +5913,16 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pQuerySql != NULL && (pQuerySql->from == NULL || pQuerySql->from->nExpr > 0)); - const char* msg0 = "invalid table name"; - const char* msg1 = "table name too long"; - const char* msg2 = "point interpolation query needs timestamp"; - const char* msg5 = "fill only available for interval query"; - const char* msg6 = "start(end) time of query range required or time range too large"; - const char* msg7 = "illegal number of tables in from clause"; - const char* msg8 = "too many columns in selection clause"; - const char* msg9 = "TWA query requires both the start and end time"; - const char* msg10= "too many tables in from clause"; + const char* msg0 = "invalid table name"; + const char* msg1 = "table name too long"; + const char* msg2 = "point interpolation query needs timestamp"; + const char* msg5 = "fill only available for interval query"; + const char* msg6 = "start(end) time of query range required or time range too large"; + const char* msg7 = "illegal number of tables in from clause"; + const char* msg8 = "too many columns in selection clause"; + const char* msg9 = "TWA query requires both the start and end time"; + const char* msg10 = "too many tables in from clause"; + const char* msg11 = "invalid table alias name"; int32_t code = TSDB_CODE_SUCCESS; @@ -5912,18 +5954,18 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySql); } - if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM) { + if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM * 2) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } pQueryInfo->command = TSDB_SQL_SELECT; - if (pQuerySql->from->nExpr > 2) { + if (pQuerySql->from->nExpr > 4) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); } // set all query tables, which are maybe more than one. - for (int32_t i = 0; i < pQuerySql->from->nExpr; ++i) { + for (int32_t i = 0; i < pQuerySql->from->nExpr; ) { tVariant* pTableItem = &pQuerySql->from->a[i].pVar; if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) { @@ -5937,24 +5979,34 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } - if (pQueryInfo->numOfTables <= i) { // more than one table + if (pQueryInfo->numOfTables <= i/2) { // more than one table tscAddEmptyMetaInfo(pQueryInfo); } - STableMetaInfo* pMeterInfo1 = tscGetMetaInfo(pQueryInfo, i); + STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; - if (tscSetTableFullName(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - code = tscGetTableMeta(pSql, pMeterInfo1); + tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar; + SSQLToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; + if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); + } + + tstrncpy(pTableMetaInfo1->aliasName, pTableItem1->pz, sizeof(pTableMetaInfo1->aliasName)); + + code = tscGetTableMeta(pSql, pTableMetaInfo1); if (code != TSDB_CODE_SUCCESS) { return code; } + + i += 2; } - assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr); + assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr / 2); bool isSTable = false; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -6122,12 +6174,14 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS if (ret != TSDB_CODE_SUCCESS) { return ret; } - + + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); + *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; @@ -6136,7 +6190,8 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name)); colIndex.colId = pSchema->colId; colIndex.colIndex = index.columnIndex; - + colIndex.flag = (index.columnIndex >= numOfColumns)? 1:0; + taosArrayPush(pCols, &colIndex); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3390710472..8d86f046ef 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -45,19 +45,27 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { + assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); + SRpcEpSet* pEpSet = &pSql->epSet; - pEpSet->inUse = 0; - if (pVgroupInfo == NULL) { - pEpSet->numOfEps = 0; - return; - } + pEpSet->inUse = 0; + + // apply the FQDN string length check here + bool hasFqdn = false; pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + + if (!hasFqdn) { + hasFqdn = (strlen(pEpSet->fqdn[i]) > 0); + } } + + assert(hasFqdn); } + static void tscDumpMgmtEpSet(SRpcEpSet *epSet) { taosCorBeginRead(&tscMgmtEpSet.version); *epSet = tscMgmtEpSet.epSet; @@ -127,21 +135,6 @@ void tscPrintMgmtEp() { } } -/* - * For each management node, try twice at least in case of poor network situation. - * If the client start to connect to a non-management node from the client, and the first retry may fail due to - * the poor network quality. And then, the second retry get the response with redirection command. - * The retry will not be executed since only *two* retry is allowed in case of single management node in the cluster. - * Therefore, we need to multiply the retry times by factor of 2 to fix this problem. - */ -UNUSED_FUNC -static int32_t tscGetMgmtConnMaxRetryTimes() { - int32_t factor = 2; - SRpcEpSet dump; - tscDumpMgmtEpSet(&dump); - return dump.numOfEps * factor; -} - void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; if (pObj == NULL) return; @@ -424,21 +417,18 @@ int doProcessSql(SSqlObj *pSql) { } int tscProcessSql(SSqlObj *pSql) { - char * name = NULL; + char *name = NULL; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = NULL; uint32_t type = 0; if (pQueryInfo != NULL) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - name = pTableMetaInfo->name; - } - + name = (pTableMetaInfo != NULL)? pTableMetaInfo->name:NULL; type = pQueryInfo->type; - + // while numOfTables equals to 0, it must be Heartbeat assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0); } @@ -450,7 +440,6 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - //pSql->epSet = tscMgmtEpSet; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); @@ -597,11 +586,11 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - tscSetDnodeEpSet(pSql, pVgroupInfo); - if (pVgroupInfo != NULL) { - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); - } + assert(pVgroupInfo != NULL); + + tscSetDnodeEpSet(pSql, pVgroupInfo); + pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->id.tid); @@ -1460,7 +1449,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows > 0) { - tscSetResultPointer(pQueryInfo, pRes); + tscCreateResPointerInfo(pRes, pQueryInfo); } pRes->row = 0; @@ -1562,7 +1551,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // fill head info SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize); - memset(pMgmt->db, 0, TSDB_TABLE_ID_LEN); // server don't need the db + memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN); // server don't need the db SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); pInfoMsg->numOfTables = htonl((int32_t)pCmd->count); @@ -1603,7 +1592,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { //// tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; //// } //// -//// int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; +//// int32_t joinCondLen = (TSDB_TABLE_FNAME_LEN + sizeof(int16_t)) * 2; //// int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; //// //// int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); @@ -1884,11 +1873,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port); - } - - pMsg += size; } + + pMsg += size; } return pSql->res.code; @@ -1966,7 +1954,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { } int tscProcessConnectRsp(SSqlObj *pSql) { - char temp[TSDB_TABLE_ID_LEN * 2]; + char temp[TSDB_TABLE_FNAME_LEN * 2]; STscObj *pObj = pSql->pTscObj; SSqlRes *pRes = &pSql->res; @@ -2114,21 +2102,6 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { return 0; } -int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) { - SSqlRes * pRes = &pSql->res; - SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; - - pRes->numOfRows = htonl(pRetrieve->numOfRows); - pRes->data = pRetrieve->data; - - tscSetResultPointer(pQueryInfo, pRes); - pRes->row = 0; - return 0; -} - void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 3e5280401f..6c7503ea4c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -233,35 +233,6 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { sem_post(&pSql->rspSem); } -TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - int32_t sqlLen = (int32_t)strlen(sqlstr); - if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_INVALID_SQL; - return NULL; - } - - taosNotePrintTsc(sqlstr); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("failed to malloc sqlObj"); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); - - // wait for the callback function to post the semaphore - tsem_wait(&pSql->rspSem); - return pSql; -} TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { @@ -274,7 +245,9 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { terrno = TSDB_CODE_TSC_INVALID_SQL; return NULL; } - + + taosNotePrintTsc(sqlstr); + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { tscError("failed to malloc sqlObj"); @@ -287,6 +260,11 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { tsem_wait(&pSql->rspSem); return pSql; } + +TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { + return taos_query_c(taos, sqlstr, strlen(sqlstr)); +} + int taos_result_precision(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; if (pSql == NULL || pSql->signature != pSql) return 0; @@ -422,7 +400,10 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pCmd->command == TSDB_SQL_INSERT) { return NULL; } - + + // set the sql object owner + tscSetSqlOwner(pSql); + // current data set are exhausted, fetch more data from node if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && (pCmd->command == TSDB_SQL_RETRIEVE || @@ -441,7 +422,10 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { sem_wait(&pSql->rspSem); } - return doSetResultRowData(pSql, true); + void* data = doSetResultRowData(pSql, true); + + tscClearSqlOwner(pSql); + return data; } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { @@ -509,7 +493,7 @@ int taos_select_db(TAOS *taos, const char *db) { } // send free message to vnode to free qhandle and corresponding resources in vnode -static bool tscFreeQhandleInVnode(SSqlObj* pSql) { +static bool tscKillQueryInVnode(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; @@ -557,16 +541,14 @@ void taos_free_result(TAOS_RES *res) { } pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - if (!tscFreeQhandleInVnode(pSql)) { + if (!tscKillQueryInVnode(pSql)) { tscFreeSqlObj(pSql); tscDebug("%p sqlObj is freed by app", pSql); } } -// todo should not be used in async query int taos_errno(TAOS_RES *tres) { SSqlObj *pSql = (SSqlObj *) tres; - if (pSql == NULL || pSql->signature != pSql) { return terrno; } @@ -812,7 +794,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t } char *nextStr; - char tblName[TSDB_TABLE_ID_LEN]; + char tblName[TSDB_TABLE_FNAME_LEN]; int payloadLen = 0; char *pMsg = pCmd->payload; while (1) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 8a596d8893..69a6d5db5b 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1026,9 +1026,11 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutput); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + int32_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * numOfExprs); + + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t tableIndexOfSub = -1; @@ -1045,8 +1047,8 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); - size_t numOfExprs = taosArrayGetSize(pSubQueryInfo->exprList); - for (int32_t k = 0; k < numOfExprs; ++k) { + size_t numOfSubExpr = taosArrayGetSize(pSubQueryInfo->exprList); + for (int32_t k = 0; k < numOfSubExpr; ++k) { SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; @@ -1054,6 +1056,10 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } } } + + // restore the offset value for super table query in case of final result. + tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); } void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { @@ -1079,7 +1085,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { assert(taos_errno(pSql) == code); - tscError("%p abort query, code:%d, global code:%d", pSql, code, pParentSql->res.code); + tscError("%p abort query, code:%s, global code:%s", pSql, tstrerror(code), tstrerror(pParentSql->res.code)); pParentSql->res.code = code; quitAllSubquery(pParentSql, pSupporter); @@ -2052,9 +2058,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { } while (1) { - if (pRes->row < pRes->numOfRows) { - assert(0); - } + assert (pRes->row >= pRes->numOfRows); doBuildResFromSubqueries(pSql); sem_post(&pSql->rspSem); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 1b6d18be0c..fcc752983c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -644,7 +644,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, 0); int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); size_t total = taosArrayGetSize(pTableDataBlockList); @@ -858,12 +858,13 @@ void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { } TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { + assert(index < pFieldInfo->numOfOutput); return TARRAY_GET_ELEM(pFieldInfo->pFields, index); } int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); - assert(pInfo != NULL); + assert(pInfo != NULL && pInfo->pSqlExpr != NULL); return pInfo->pSqlExpr->offset; } @@ -1680,6 +1681,77 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return pNew; } +// current sql function is not direct output result, so create a dummy output field +static void doSetNewFieldInfo(SQueryInfo* pNewQueryInfo, SSqlExpr* pExpr) { + TAOS_FIELD f = {.type = pExpr->resType, .bytes = pExpr->resBytes}; + tstrncpy(f.name, pExpr->aliasName, sizeof(f.name)); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); + + pInfo1->pSqlExpr = pExpr; + pInfo1->visible = false; +} + +static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) { + int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); + if (numOfOutput == 0) { + return; + } + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + // set the field info in pNewQueryInfo object + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + + if (pExpr->uid == uid) { + if (i < pFieldInfo->numOfOutput) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); + + if (pInfo->pSqlExpr != NULL) { + TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); + assert(strcmp(p->name, pExpr->aliasName) == 0); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); + *pInfo1 = *pInfo; + } else { + assert(pInfo->pArithExprInfo != NULL); + doSetNewFieldInfo(pNewQueryInfo, pExpr); + } + } else { // it is a arithmetic column, does not have actual field for sqlExpr, so build it + doSetNewFieldInfo(pNewQueryInfo, pExpr); + } + } + } + + // make sure the the sqlExpr for each fields is correct + numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); + + // update the pSqlExpr pointer in SFieldSupInfo according the field name + // make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo + for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { + TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); + + bool matched = false; + for (int32_t k1 = 0; k1 < numOfExprs; ++k1) { + SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); + + if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); + pInfo->pSqlExpr = pExpr1; + + matched = true; + break; + } + } + + assert(matched); + } + + tscFieldInfoUpdateOffset(pNewQueryInfo); +} + SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); @@ -1773,49 +1845,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true); - int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo); - - if (numOfOutput > 0) { // todo refactor to extract method - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - - for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - if (pExpr->uid == uid) { - TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); - *pInfo1 = *pInfo; - } - } - - // make sure the the sqlExpr for each fields is correct - // todo handle the agg arithmetic expression - numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); - - for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { - TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); - bool matched = false; - - for(int32_t k1 = 0; k1 < numOfExprs; ++k1) { - SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); - - if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); - pInfo->pSqlExpr = pExpr1; - - matched = true; - break; - } - } - - assert(matched); - } - - tscFieldInfoUpdateOffset(pNewQueryInfo); - } + doSetSqlExprAndResultFieldInfo(pQueryInfo, pNewQueryInfo, uid); pNew->fp = fp; pNew->fetchFp = fp; @@ -2013,6 +2043,10 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { } int32_t numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + if (pTableMetaInfo->pVgroupTables != NULL) { + numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } + return tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1); } @@ -2200,3 +2234,21 @@ int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { return 0; } + +bool tscSetSqlOwner(SSqlObj* pSql) { + SSqlRes* pRes = &pSql->res; + + // set the sql object owner + uint64_t threadId = taosGetPthreadId(); + if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { + pRes->code = TSDB_CODE_QRY_IN_EXEC; + return false; + } + + return true; +} + +void tscClearSqlOwner(SSqlObj* pSql) { + assert(pSql->owner != 0); + atomic_store_64(&pSql->owner, 0); +} \ No newline at end of file diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index cb53bb5e60..c99cf87b21 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -202,8 +202,9 @@ static void *dnodeProcessReadQueue(void *param) { break; } - dDebug("%p, msg:%s will be processed in vread queue, qtype:%d", pReadMsg->rpcMsg.ahandle, - taosMsg[pReadMsg->rpcMsg.msgType], type); + dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle, + taosMsg[pReadMsg->rpcMsg.msgType], type, pReadMsg); + int32_t code = vnodeProcessRead(pVnode, pReadMsg); if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { diff --git a/src/inc/query.h b/src/inc/query.h index ec1e458b62..0c18f85dc3 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -76,6 +76,9 @@ void* qGetResultRetrieveMsg(qinfo_t qinfo); */ int32_t qKillQuery(qinfo_t qinfo); +int32_t qQueryCompleted(qinfo_t qinfo); + + /** * destroy query info structure * @param qHandle diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ff05c840da..aa59418347 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -232,7 +232,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 33 -#define TSDB_TABLE_ID_LEN (TSDB_ACCT_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) +#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ece08ae173..9af4cee28a 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -219,6 +219,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "Duplicated TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag conditon too many") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 6d43568ecd..265bf47d6d 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -246,13 +246,13 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; + char superTableId[TSDB_TABLE_FNAME_LEN]; char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int8_t igExists; int8_t getMeta; @@ -265,12 +265,12 @@ typedef struct { } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int16_t type; /* operation type */ int16_t numOfCols; /* number of schema */ @@ -297,7 +297,7 @@ typedef struct { typedef struct { char clientVersion[TSDB_VERSION_LEN]; char msgVersion[TSDB_VERSION_LEN]; - char db[TSDB_TABLE_ID_LEN]; + char db[TSDB_TABLE_FNAME_LEN]; } SCMConnectMsg; typedef struct { @@ -347,14 +347,14 @@ typedef struct { int32_t vgId; int32_t sid; uint64_t uid; - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; } SMDDropTableMsg; typedef struct { int32_t contLen; int32_t vgId; uint64_t uid; - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; } SMDDropSTableMsg; typedef struct { @@ -527,7 +527,7 @@ typedef struct { } SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { - char db[TSDB_TABLE_ID_LEN]; + char db[TSDB_TABLE_FNAME_LEN]; uint8_t ignoreNotExists; } SCMDropDbMsg, SCMUseDbMsg; @@ -637,7 +637,7 @@ typedef struct { } SMDCreateVnodeMsg, SMDAlterVnodeMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; int16_t createFlag; char tags[]; } SCMTableInfoMsg; @@ -664,7 +664,7 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_ID_LEN]; // table id + char tableId[TSDB_TABLE_FNAME_LEN]; // table id uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -685,7 +685,7 @@ typedef struct SMultiTableMeta { typedef struct { int32_t dataLen; - char name[TSDB_TABLE_ID_LEN]; + char name[TSDB_TABLE_FNAME_LEN]; char data[TSDB_MAX_TAGS_LEN + TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * TSDB_MAX_TAGS]; } STagData; @@ -771,7 +771,7 @@ typedef struct { uint64_t uid; uint64_t stime; // stream starting time int32_t status; - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; } SMDAlterStreamMsg; typedef struct { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index bab17322f0..4776d1cda7 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -167,9 +167,14 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct { - size_t numOfTables; + void *pTable; + TSKEY lastKey; +} STableKeyInfo; + +typedef struct { + size_t numOfTables; SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo from STableId + SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; /** @@ -177,24 +182,24 @@ typedef struct { * * @param tsdb tsdb handle * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableqinfoGroupInfo tableId list in the form of set, seperated into different groups according to group by condition + * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the + * group by condition * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo); +TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. * Note that only one data block with only row will be returned while invoking retrieve data block function for * all tables in this group. * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each - * block - * @param tableqinfoGroupInfo tableId list. + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfo table list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo); +TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo); /** * get the queried table object list @@ -260,7 +265,7 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL * @param stableid. super table sid * @param pTagCond. tag query condition */ -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); @@ -278,7 +283,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); /** * diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 7f9be7eb54..f6249986fd 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -193,7 +193,7 @@ void mnodeDecDbRef(SDbObj *pDb) { } SDbObj *mnodeGetDbByTableId(char *tableId) { - char db[TSDB_TABLE_ID_LEN], *pos; + char db[TSDB_TABLE_FNAME_LEN], *pos; // tableId format should be : acct.db.table pos = strstr(tableId, TS_PATH_DELIMITER); @@ -1046,7 +1046,7 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pDrop->db); if (pMsg->pDb == NULL) { if (pDrop->ignoreNotExists) { - mDebug("db:%s, db is not exist, think drop success", pDrop->db); + mDebug("db:%s, db is not exist, treat as success", pDrop->db); return TSDB_CODE_SUCCESS; } else { mError("db:%s, failed to drop, invalid db", pDrop->db); diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 353dd59671..85457d7a26 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -73,13 +73,13 @@ int32_t mnodeInitProfile() { void mnodeCleanupProfile() { if (tsMnodeConnCache != NULL) { - mInfo("conn cache is cleanup"); taosCacheCleanup(tsMnodeConnCache); tsMnodeConnCache = NULL; } } SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { +#if 0 int32_t connSize = taosHashGetSize(tsMnodeConnCache->pHashTable); if (connSize > tsMaxShellConns) { mError("failed to create conn for user:%s ip:%s:%u, conns:%d larger than maxShellConns:%d, ", user, taosIpStr(ip), @@ -87,6 +87,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { terrno = TSDB_CODE_MND_TOO_MANY_SHELL_CONNS; return NULL; } +#endif int32_t connId = atomic_add_fetch_32(&tsConnIndex, 1); if (connId == 0) atomic_add_fetch_32(&tsConnIndex, 1); @@ -100,7 +101,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { tstrncpy(connObj.user, user, sizeof(connObj.user)); SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME); - + mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; } @@ -112,7 +113,7 @@ void mnodeReleaseConn(SConnObj *pConn) { SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port) { uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs(); - SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, &connId, sizeof(int32_t), expireTime); + SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t)); if (pConn == NULL) { mDebug("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return NULL; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index efca748e9f..3651aa8aad 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -874,7 +874,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); } - pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true); + pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, true); tsSdbObj.numOfTables++; tsSdbObj.tableList[pTable->tableId] = pTable; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 995bfbe840..9983c111f6 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -302,7 +302,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { SAcctObj *pAcct = pUser->pAcct; if (pConnectMsg->db[0]) { - char dbName[TSDB_TABLE_ID_LEN * 3] = {0}; + char dbName[TSDB_TABLE_FNAME_LEN * 3] = {0}; sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); SDbObj *pDb = mnodeGetDb(dbName); if (pDb == NULL) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 7b928fe885..dbc1bffa5c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -215,7 +215,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { assert(pTable != NULL && pOper->rowData != NULL); int32_t len = strlen(pTable->info.tableId); - if (len >= TSDB_TABLE_ID_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; + if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pOper->rowData, pTable->info.tableId, len); memset(pOper->rowData + len, 0, 1); @@ -246,7 +246,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) { if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); - if (len >= TSDB_TABLE_ID_LEN) { + if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; } @@ -348,7 +348,7 @@ static int32_t mnodeInitChildTables() { .tableId = SDB_TABLE_CTABLE, .tableName = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeChildTableActionInsert, @@ -387,7 +387,7 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { - pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); } if (pStable->vgHash != NULL) { @@ -479,7 +479,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t len = strlen(pStable->info.tableId); - if (len >= TSDB_TABLE_ID_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; + if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pOper->rowData, pStable->info.tableId, len); memset(pOper->rowData + len, 0, 1); @@ -503,7 +503,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) { if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); - if (len >= TSDB_TABLE_ID_LEN){ + if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; } @@ -539,7 +539,7 @@ static int32_t mnodeInitSuperTables() { .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, - .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN, + .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeSuperTableActionInsert, @@ -751,7 +751,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId); if (pMsg->pTable == NULL) { if (pDrop->igNotExists) { - mDebug("app:%p:%p, table:%s, table is not exist, think drop success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); + mDebug("app:%p:%p, table:%s, table is not exist, treat as success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); return TSDB_CODE_SUCCESS; } else { mError("app:%p:%p, table:%s, failed to drop table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); @@ -1464,7 +1464,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { // reserve space int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo); for (int32_t i = 0; i < numOfTable; ++i) { - char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i; + char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable != NULL && pTable->vgHash != NULL) { contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo)); @@ -1481,7 +1481,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN)*i; + char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName); @@ -1828,7 +1828,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); + tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); pDrop->vgId = htonl(pTable->vgId); pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); pDrop->sid = htonl(pTable->sid); @@ -2079,7 +2079,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->sid = htonl(pTable->sid); pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; - tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); + tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); if (pTable->info.type == TSDB_CHILD_TABLE) { pMeta->sversion = htons(pTable->superTable->sversion); @@ -2448,7 +2448,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { pMultiMeta->numOfTables = 0; for (int32_t t = 0; t < pInfo->numOfTables; ++t) { - char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN); + char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_FNAME_LEN); SChildTableObj *pTable = mnodeGetChildTable(tableId); if (pTable == NULL) continue; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index e367911695..f7694ded97 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -167,8 +167,8 @@ bool httpInitContext(HttpContext *pContext) { memset(pParser, 0, sizeof(HttpParser)); pParser->pCur = pParser->pLast = pParser->buffer; - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed); + httpDebug("context:%p, fd:%d, ip:%s, accessTimes:%d, parsed:%d", pContext, pContext->fd, pContext->ipstr, + pContext->accessTimes, pContext->parsed); return true; } diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 6c7ecbb951..32f5966e2d 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -192,7 +192,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.slowquery(ts timestamp, username " "binary(%d), created_time timestamp, time bigint, sql binary(%d))", - tsMonitorDbName, TSDB_TABLE_ID_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN); + tsMonitorDbName, TSDB_TABLE_FNAME_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN); } else if (cmd == MONITOR_CMD_CREATE_TB_LOG) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.log(ts timestamp, level tinyint, " diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 01b4c16ac1..ec568a6cdb 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -48,7 +48,7 @@ typedef struct tQueryInfo { SSchema sch; // schema of tags char* q; __compar_fn_t compare; // filter function - void* param; // STSchema + bool indexed; // indexed columns } tQueryInfo; typedef struct SExprTraverseSupp { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index bd2e0a4470..c7026b45c6 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -52,10 +52,10 @@ typedef struct SWindowStatus { typedef struct SWindowResult { uint16_t numOfRows; // number of rows of current time window + SWindowStatus status; // this result status: closed or opened SPosInfo pos; // Position of current result in disk-based output buffer SResultInfo* resultInfo; // For each result column, there is a resultInfo STimeWindow window; // The time window that current result covers. - SWindowStatus status; // this result status: closed or opened } SWindowResult; /** @@ -122,6 +122,7 @@ typedef struct SQueryCostInfo { uint32_t discardBlocks; uint64_t elapsedTime; uint64_t computTime; + uint64_t internalSupSize; } SQueryCostInfo; typedef struct SQuery { @@ -184,10 +185,8 @@ enum { typedef struct SQInfo { void* signature; - int32_t pointsInterpo; - int32_t code; // error code to returned to client -// sem_t dataReady; - + int32_t code; // error code to returned to client + pthread_t owner; // if it is in execution void* tsdb; int32_t vgId; STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index 36fc0c9820..0bdcf5c45e 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -29,7 +29,7 @@ extern "C" { #define MAX_TMPFILE_PATH_LENGTH PATH_MAX #define INITIAL_ALLOCATION_BUFFER_SIZE 64 -#define DEFAULT_PAGE_SIZE (4096L) // 16k larger than the SHistoInfo +#define DEFAULT_PAGE_SIZE (1024L) // 16k larger than the SHistoInfo typedef enum EXT_BUFFER_FLUSH_MODEL { /* diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 15cfeee6b2..2b2a967edf 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -26,7 +26,12 @@ #include #include #include +#include "qSqlparser.h" +#include "tcmdtype.h" +#include "tstoken.h" +#include "ttokendef.h" #include "tutil.h" +#include "tvariant.h" } %syntax_error { @@ -254,7 +259,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; } alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtod(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { @@ -422,8 +427,35 @@ as(X) ::= . { X.n = 0; } from(A) ::= FROM tablelist(X). {A = X;} %type tablelist {tVariantList*} -tablelist(A) ::= ids(X) cpxName(Y). { toTSDBType(X.type); X.n += Y.n; A = tVariantListAppendToken(NULL, &X, -1);} -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { toTSDBType(X.type); X.n += Z.n; A = tVariantListAppendToken(Y, &X, -1); } +tablelist(A) ::= ids(X) cpxName(Y). { + toTSDBType(X.type); + X.n += Y.n; + A = tVariantListAppendToken(NULL, &X, -1); + A = tVariantListAppendToken(A, &X, -1); // table alias name +} + +tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { + toTSDBType(X.type); + toTSDBType(Z.type); + X.n += Y.n; + A = tVariantListAppendToken(NULL, &X, -1); + A = tVariantListAppendToken(A, &Z, -1); +} + +tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { + toTSDBType(X.type); + X.n += Z.n; + A = tVariantListAppendToken(Y, &X, -1); + A = tVariantListAppendToken(A, &X, -1); +} + +tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { + toTSDBType(X.type); + toTSDBType(F.type); + X.n += Z.n; + A = tVariantListAppendToken(Y, &X, -1); + A = tVariantListAppendToken(A, &F, -1); +} // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" %type tmvar {SSQLToken} diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index e2d609a62c..afa3618a96 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -427,8 +427,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret != 0) { break; } - - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); } } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal bool comp = true; @@ -445,7 +446,8 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_GREATER) { continue; } else { - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); comp = false; } } @@ -458,8 +460,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (comp) { continue; } - - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); } tSkipListDestroyIter(iter); @@ -472,8 +475,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (comp) { continue; } - - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); } } else { @@ -496,12 +500,14 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_LESS) { continue; } else { - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); comp = false; // no need to compare anymore } } } } + free(cond.start); free(cond.end); tSkipListDestroyIter(iter); @@ -689,7 +695,8 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, } if (addToResult) { - taosArrayPush(res, pData); + STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(res, &info); } } @@ -716,7 +723,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S } tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->sch.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && pQueryInfo->optr != TSDB_RELATION_LIKE) { + if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3a31e5b9eb..7f6fac3631 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -130,6 +130,9 @@ static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv); (tw)->ekey = (tw)->skey + ((_q)->intervalTime - 1); \ } while (0) +#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (_q)->tableqinfoGroupInfo.numOfTables) +#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (_q)->tableqinfoGroupInfo.numOfTables) + // todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); @@ -400,8 +403,16 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin // more than the capacity, reallocate the resources if (pWindowResInfo->size >= pWindowResInfo->capacity) { - int64_t newCap = (int64_t)(pWindowResInfo->capacity * 1.5f); + int64_t newCap = 0; + if (pWindowResInfo->capacity > 10000) { + newCap = pWindowResInfo->capacity * 1.25; + } else { + newCap = pWindowResInfo->capacity * 1.5; + } + char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); + pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -411,6 +422,8 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin int32_t inc = (int32_t)newCap - pWindowResInfo->capacity; memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); + pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; + for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); } @@ -1823,10 +1836,14 @@ static void doExchangeTimeWindow(SQInfo* pQInfo) { for(int32_t i = 0; i < t; ++i) { SArray* p1 = GET_TABLEGROUP(pQInfo, i); + SArray* tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); size_t len = taosArrayGetSize(p1); for(int32_t j = 0; j < len; ++j) { STableQueryInfo* pTableQueryInfo = (STableQueryInfo*) taosArrayGetP(p1, j); SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + + STableKeyInfo* pInfo = taosArrayGet(tableKeyGroup, j); + pInfo->lastKey = pTableQueryInfo->win.skey; } } } @@ -2225,7 +2242,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv)) { + if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { @@ -2662,6 +2679,10 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } + if (pQInfo->groupIndex == numOfGroups && pQInfo->offset == pQInfo->numOfGroupResultPages) { + SET_STABLE_QUERY_OVER(pQInfo); + } + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo, pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); @@ -2680,7 +2701,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { - pQInfo->tableIndex = (int32_t)pQInfo->tableqinfoGroupInfo.numOfTables; // set query completed + SET_STABLE_QUERY_OVER(pQInfo); return; } } @@ -2704,8 +2725,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num); } -// rows += pData->num; - offset += (int32_t)pData->num; + offset += pData->num; } assert(pQuery->rec.rows == 0); @@ -2790,14 +2810,20 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + // todo add windowRes iterator int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); while (1) { + if (IS_QUERY_KILLED(pQInfo)) { + qDebug("QInfo:%p it is already killed, abort", pQInfo); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + int32_t pos = pTree->pNode[0].index; SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); @@ -2834,6 +2860,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { lastTimestamp = ts; + // move to the next element of current entry + int32_t currentPageId = pWindowRes->pos.pageId; + cs.position[pos] += 1; if (cs.position[pos] >= pWindowResInfo->size) { cs.position[pos] = -1; @@ -2842,6 +2871,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { if (--numOfTables == 0) { break; } + } else { + // current page is not needed anymore + SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + if (pNextWindowRes->pos.pageId != currentPageId) { + releaseResBufPage(pRuntimeEnv->pResultBuf, page); + } } } @@ -2935,7 +2970,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * return; } - // order has change already! + // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // TODO validate the assertion @@ -2944,9 +2979,13 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * // } else { // assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step); // } - - pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; - + + if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { + // do nothing, no results + } else { + pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; + } + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; @@ -3008,16 +3047,26 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { } } } - - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - +} + +static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + int32_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < numOfGroups; ++i) { SArray *group = GET_TABLEGROUP(pQInfo, i); - + SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); + + // update the last key in tableKeyInfo list + STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); + pTableKeyInfo->lastKey = pCheckInfo->lastKey; + + assert(pCheckInfo->pTable == pTableKeyInfo->pTable); } } } @@ -3266,20 +3315,20 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI TIME_WINDOW_COPY(cond.twindow, pQuery->window); + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + switchCtxOrder(pRuntimeEnv); + disableFuncInReverseScan(pQInfo); + setupQueryRangeForReverseScan(pQInfo); + // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - // add ref for table pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); } static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { @@ -3304,6 +3353,13 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus pQuery->window = pTableQueryInfo->win; } +static void restoreTimeWindow(STableGroupInfo* pTableGroupInfo, STsdbQueryCond* pCond) { + assert(pTableGroupInfo->numOfTables == 1); + SArray* pTableKeyGroup = taosArrayGetP(pTableGroupInfo->pGroupList, 0); + STableKeyInfo* pKeyInfo = taosArrayGet(pTableKeyGroup, 0); + pKeyInfo->lastKey = pCond->twindow.skey; +} + void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; @@ -3352,6 +3408,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } + restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); @@ -3800,7 +3857,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc } } -bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) { +bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo *pFillInfo = pRuntimeEnv->pFillInfo; @@ -3809,8 +3866,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) { } if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { - // There are results not returned to client yet, so filling operation applied to the remain result is required - // in the first place. + // There are results not returned to client yet, so filling applied to the remain result is required firstly. int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { return true; @@ -3864,14 +3920,14 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data data += sizeof(STableIdInfo); } - // all data returned, set query over + // Check if query is completed or not for stable query or normal table query respectively. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (pQInfo->runtimeEnv.stableQuery) { - if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { + if (IS_STASBLE_QUERY_OVER(pQInfo)) { setQueryStatus(pQuery, QUERY_OVER); } } else { - if (!queryHasRemainResults(&pQInfo->runtimeEnv)) { + if (!queryHasRemainResForTableQuery(&pQInfo->runtimeEnv)) { setQueryStatus(pQuery, QUERY_OVER); } } @@ -3917,7 +3973,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int ret = 0; } - if (!queryHasRemainResults(pRuntimeEnv)) { + if (!queryHasRemainResForTableQuery(pRuntimeEnv)) { return ret; } } @@ -3931,6 +3987,8 @@ static void queryCostStatis(SQInfo *pQInfo) { " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); + + qDebug("QInfo:%p :cost summary: internal size:%"PRId64, pQInfo, pSummary->internalSupSize); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { @@ -4425,9 +4483,11 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { // todo refactor SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayInit(1, POINTER_BYTES); + SArray *tx = taosArrayInit(1, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = pCheckInfo->pTable, .lastKey = pCheckInfo->lastKey}; + taosArrayPush(tx, &info); - taosArrayPush(tx, &pCheckInfo->pTable); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; @@ -4579,7 +4639,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pRuntimeEnv->pQueryHandle = NULL; } + // no need to update the lastkey for each table pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo); + taosArrayDestroy(g1); taosArrayDestroy(tx); if (pRuntimeEnv->pQueryHandle == NULL) { @@ -4680,7 +4742,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // the limitation of output result is reached, set the query completed if (limitResults(pRuntimeEnv)) { - pQInfo->tableIndex = (int32_t)pQInfo->tableqinfoGroupInfo.numOfTables; + SET_STABLE_QUERY_OVER(pQInfo); break; } @@ -4705,8 +4767,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); // if the buffer is full or group by each table, we need to jump out of the loop - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| - isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { break; } @@ -4770,7 +4831,7 @@ static void doSaveContext(SQInfo *pQInfo) { .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + TIME_WINDOW_COPY(cond.twindow, pQuery->window); // clean unused handle @@ -4778,15 +4839,16 @@ static void doSaveContext(SQInfo *pQInfo) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + switchCtxOrder(pRuntimeEnv); + disableFuncInReverseScan(pQInfo); + setupQueryRangeForReverseScan(pQInfo); + pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); } static void doRestoreContext(SQInfo *pQInfo) { @@ -5070,15 +5132,13 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); } - - pQInfo->pointsInterpo += numOfFilled; } static void tableQueryImpl(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - if (queryHasRemainResults(pRuntimeEnv)) { + if (queryHasRemainResForTableQuery(pRuntimeEnv)) { if (pQuery->fillType != TSDB_FILL_NONE) { /* @@ -5859,7 +5919,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); } int tableIndex = 0; @@ -5880,8 +5940,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, } for(int32_t j = 0; j < s; ++j) { - void* pTable = taosArrayGetP(pa, j); - STableId* id = TSDB_TABLEID(pTable); + STableKeyInfo* info = taosArrayGet(pa, j); + STableId* id = TSDB_TABLEID(info->pTable); STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); if (pTableId != NULL ) { @@ -5891,10 +5951,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, } void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, pTable, window, buf); + STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; } + item->groupIndex = i; taosArrayPush(p1, &item); taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); @@ -5923,6 +5984,7 @@ _cleanup_query: taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); } + taosTFree(pTagCols); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; @@ -5930,6 +5992,7 @@ _cleanup_query: tExprTreeDestroy(&pExprInfo->pExpr, NULL); } } + taosTFree(pExprs); _cleanup: @@ -6011,8 +6074,6 @@ static void freeQInfo(SQInfo *pQInfo) { } SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - setQueryKilled(pQInfo); - qDebug("QInfo:%p start to free QInfo", pQInfo); for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { taosTFree(pQuery->sdata[col]); @@ -6217,7 +6278,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi STableIdInfo *id = taosArrayGet(pTableIdList, 0); qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { goto _over; } } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { @@ -6234,8 +6295,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi } qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, - numOfGroupByCols); + code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, tagCond, pQueryMsg->tagCondLen, + pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, numOfGroupByCols); + if (code != TSDB_CODE_SUCCESS) { qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); goto _over; @@ -6315,14 +6377,22 @@ static bool doBuildResCheck(SQInfo* pQInfo) { pthread_mutex_unlock(&pQInfo->lock); + // clear qhandle owner + assert(pQInfo->owner == pthread_self()); + pQInfo->owner = 0; + return buildRes; } bool qTableQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; + assert(pQInfo && pQInfo->signature == pQInfo); + int64_t threadId = pthread_self(); - if (pQInfo == NULL || pQInfo->signature != pQInfo) { - qDebug("QInfo:%p has been freed, no need to execute", pQInfo); + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { + qError("QInfo:%p qhandle is now executed by thread:%p", pQInfo, (void*) curOwner); + pQInfo->code = TSDB_CODE_QRY_IN_EXEC; return false; } @@ -6377,6 +6447,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex return TSDB_CODE_QRY_INVALID_QHANDLE; } + *buildRes = false; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); @@ -6400,34 +6471,6 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex return code; } -bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (!isValidQInfo(pQInfo) || pQInfo->code != TSDB_CODE_SUCCESS) { - qDebug("QInfo:%p invalid qhandle or error occurs, abort query, code:%x", pQInfo, pQInfo->code); - return false; - } - - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - bool ret = false; - if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - ret = false; - } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - ret = true; - } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - ret = true; - } else { - assert(0); - } - - if (ret) { - qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); - } - - return ret; -} - int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { SQInfo *pQInfo = (SQInfo *)qinfo; @@ -6455,11 +6498,11 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { - (*pRsp)->offset = htobe64(pQuery->limit.offset); + (*pRsp)->offset = htobe64(pQuery->limit.offset); (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } else { - (*pRsp)->useconds = 0; - (*pRsp)->offset = 0; + (*pRsp)->offset = 0; + (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } (*pRsp)->precision = htons(pQuery->precision); @@ -6471,22 +6514,30 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co } pQInfo->rspContext = NULL; - pQInfo->dataReady = QUERY_RESULT_NOT_READY; + pQInfo->dataReady = QUERY_RESULT_NOT_READY; if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - (*pRsp)->completed = 1; // notify no more result to client - } - - if (qHasMoreResultsToRetrieve(pQInfo)) { - *continueExec = true; - } else { // failed to dump result, free qhandle immediately *continueExec = false; - qKillQuery(pQInfo); + (*pRsp)->completed = 1; // notify no more result to client + } else { + *continueExec = true; + qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); } return code; } +int32_t qQueryCompleted(qinfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + return IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER); +} + int32_t qKillQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; @@ -6495,6 +6546,13 @@ int32_t qKillQuery(qinfo_t qinfo) { } setQueryKilled(pQInfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while(pQInfo->owner != 0) { + taosMsleep(100); + } + return TSDB_CODE_SUCCESS; } @@ -6589,7 +6647,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { *(int64_t*) pQuery->sdata[0]->data = num; count = 1; - pQInfo->tableIndex = (int32_t)num; //set query completed + SET_STABLE_QUERY_OVER(pQInfo); qDebug("QInfo:%p create count(tbname) query, res:%d rows:1", pQInfo, count); } else { // return only the tags|table name etc. count = 0; @@ -6661,7 +6719,7 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 60; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); @@ -6691,9 +6749,9 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) { SQueryMgmt* pQueryMgmt = pQMgmt; qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); - pthread_mutex_lock(&pQueryMgmt->lock); +// pthread_mutex_lock(&pQueryMgmt->lock); pQueryMgmt->closed = true; - pthread_mutex_unlock(&pQueryMgmt->lock); +// pthread_mutex_unlock(&pQueryMgmt->lock); taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); } @@ -6723,7 +6781,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { return NULL; } - const int32_t DEFAULT_QHANDLE_LIFE_SPAN = tsShellActivityTimer * 2; + const int32_t DEFAULT_QHANDLE_LIFE_SPAN = tsShellActivityTimer * 2 * 1000; SQueryMgmt *pQueryMgmt = pMgmt; if (pQueryMgmt->qinfoPool == NULL) { @@ -6731,16 +6789,16 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { return NULL; } - pthread_mutex_lock(&pQueryMgmt->lock); +// pthread_mutex_lock(&pQueryMgmt->lock); if (pQueryMgmt->closed) { - pthread_mutex_unlock(&pQueryMgmt->lock); +// pthread_mutex_unlock(&pQueryMgmt->lock); qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); return NULL; } else { uint64_t handleVal = (uint64_t) qInfo; void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(int64_t), &qInfo, POINTER_BYTES, DEFAULT_QHANDLE_LIFE_SPAN); - pthread_mutex_unlock(&pQueryMgmt->lock); +// pthread_mutex_unlock(&pQueryMgmt->lock); return handle; } @@ -6763,7 +6821,6 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key) { void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { return NULL; } diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index fcc17a0c9e..a147780144 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,15 +12,14 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "qExtbuffer.h" #include "os.h" +#include "qExtbuffer.h" #include "queryLog.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" #include "tulog.h" -#include "tutil.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index a300cc6382..7c40c33933 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -6,7 +6,8 @@ #include "queryLog.h" #include "taoserror.h" -#define GET_DATA_PAYLOAD(_p) ((tFilePage*)(((char*)(_p)->pData) + POINTER_BYTES)) +#define GET_DATA_PAYLOAD(_p) ((_p)->pData + POINTER_BYTES) +#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, int32_t inMemBufSize, const void* handle) { @@ -25,7 +26,7 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro pResBuf->comp = true; pResBuf->file = NULL; pResBuf->handle = handle; - pResBuf->fileSize = 0; + pResBuf->fileSize = 0; // at least more than 2 pages must be in memory assert(inMemBufSize >= pagesize * 2); @@ -34,9 +35,9 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro pResBuf->lruList = tdListNew(POINTER_BYTES); // init id hash table - pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES - pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); char path[PATH_MAX] = {0}; taosGetTmpfilePath("qbuf", path); @@ -186,8 +187,6 @@ static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { return (char*)GET_DATA_PAYLOAD(pg); } -#define NO_AVAILABLE_PAGES(_b) ((_b)->numOfPages >= (_b)->inMemPages) - static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); @@ -211,11 +210,12 @@ static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, pResultBuf->numOfPages += 1; SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; - ppi->info = PAGE_INFO_INITIALIZER; - ppi->pageId = pageId; - ppi->pData = NULL; - ppi->pn = NULL; - ppi->used = true; + + ppi->pageId = pageId; + ppi->pData = NULL; + ppi->info = PAGE_INFO_INITIALIZER; + ppi->used = true; + ppi->pn = NULL; return *(SPageInfo**) taosArrayPush(list, &ppi); } @@ -246,7 +246,9 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { // all pages are referenced by user, try to allocate new space if (pn == NULL) { int32_t prev = pResultBuf->inMemPages; - pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f); + + // increase by 50% of previous mem pages + pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5f; qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev, pResultBuf->inMemPages, pResultBuf->pageSize); @@ -281,7 +283,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 pResultBuf->statis.getPages += 1; char* availablePage = NULL; - if (NO_AVAILABLE_PAGES(pResultBuf)) { + if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { availablePage = evicOneDataPage(pResultBuf); } @@ -340,7 +342,7 @@ tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0); char* availablePage = NULL; - if (NO_AVAILABLE_PAGES(pResultBuf)) { + if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { availablePage = evicOneDataPage(pResultBuf); } @@ -353,6 +355,8 @@ tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { ((void**)((*pi)->pData))[0] = (*pi); lruListPushFront(pResultBuf->lruList, *pi); + (*pi)->used = true; + loadPageFromDisk(pResultBuf, *pi); return GET_DATA_PAYLOAD(*pi); } @@ -396,12 +400,13 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file size:%"PRId64" bytes", - pResultBuf->handle, pResultBuf->totalBufSize, pResultBuf->fileSize); + qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes", + pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, + pResultBuf->fileSize); fclose(pResultBuf->file); } else { - qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, + qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, pResultBuf->totalBufSize); } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index d730b3cab0..227aded5f1 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -25,7 +25,7 @@ // All the keywords of the SQL language are stored in a hash table typedef struct SKeyword { const char* name; // The keyword name - uint16_t type; // type + uint16_t type; // type uint8_t len; // length } SKeyword; @@ -257,7 +257,7 @@ static void* KeywordHashTable = NULL; static void doInitKeywordsTable(void) { int numOfEntries = tListLen(keywordTable); - KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, false); + KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false); for (int32_t i = 0; i < numOfEntries; i++) { keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name); void* ptr = &keywordTable[i]; diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index ab7678a22b..25eb33ff7d 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -72,6 +72,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // invalid file if (header.magic != TS_COMP_FILE_MAGIC) { + tsBufDestroy(pTSBuf); return NULL; } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 7175262270..a01eb33ec7 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -37,7 +37,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->type = type; _hash_fn_t fn = taosGetDefaultHashFunction(type); - pWindowResInfo->hashList = taosHashInit(threshold, fn, false); + pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false); if (pWindowResInfo->hashList == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -46,12 +46,17 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); if (pWindowResInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + pRuntimeEnv->summary.internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); if (code != TSDB_CODE_SUCCESS) { @@ -104,7 +109,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR pWindowResInfo->size = 0; _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); - pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, false); + pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false); pWindowResInfo->startTime = TSKEY_INITIAL_VAL; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index fe26f61725..75ef2f3218 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -126,17 +126,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 243 -#define YYNRULE 226 +#define YYNSTATE 245 +#define YYNRULE 228 #define YYNTOKEN 207 -#define YY_MAX_SHIFT 242 -#define YY_MIN_SHIFTREDUCE 405 -#define YY_MAX_SHIFTREDUCE 630 -#define YY_ERROR_ACTION 631 -#define YY_ACCEPT_ACTION 632 -#define YY_NO_ACTION 633 -#define YY_MIN_REDUCE 634 -#define YY_MAX_REDUCE 859 +#define YY_MAX_SHIFT 244 +#define YY_MIN_SHIFTREDUCE 407 +#define YY_MAX_SHIFTREDUCE 634 +#define YY_ERROR_ACTION 635 +#define YY_ACCEPT_ACTION 636 +#define YY_NO_ACTION 637 +#define YY_MIN_REDUCE 638 +#define YY_MAX_REDUCE 865 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -202,64 +202,64 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (552) +#define YY_ACTTAB_COUNT (554) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 103, 446, 135, 673, 632, 242, 126, 515, 135, 447, - /* 10 */ 135, 158, 847, 41, 43, 11, 35, 36, 846, 157, - /* 20 */ 847, 29, 134, 446, 197, 39, 37, 40, 38, 155, - /* 30 */ 103, 447, 139, 34, 33, 217, 216, 32, 31, 30, - /* 40 */ 41, 43, 767, 35, 36, 32, 31, 30, 29, 756, - /* 50 */ 446, 197, 39, 37, 40, 38, 182, 802, 447, 192, - /* 60 */ 34, 33, 21, 21, 32, 31, 30, 406, 407, 408, - /* 70 */ 409, 410, 411, 412, 413, 414, 415, 416, 417, 241, - /* 80 */ 41, 43, 228, 35, 36, 194, 843, 58, 29, 21, - /* 90 */ 842, 197, 39, 37, 40, 38, 166, 167, 753, 753, - /* 100 */ 34, 33, 168, 56, 32, 31, 30, 778, 841, 16, - /* 110 */ 235, 208, 234, 233, 207, 206, 205, 232, 204, 231, - /* 120 */ 230, 229, 203, 215, 151, 753, 732, 586, 719, 720, - /* 130 */ 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, - /* 140 */ 731, 43, 8, 35, 36, 61, 113, 21, 29, 153, - /* 150 */ 240, 197, 39, 37, 40, 38, 239, 238, 95, 775, - /* 160 */ 34, 33, 165, 99, 32, 31, 30, 169, 35, 36, - /* 170 */ 214, 213, 592, 29, 595, 103, 197, 39, 37, 40, - /* 180 */ 38, 220, 756, 753, 236, 34, 33, 175, 12, 32, - /* 190 */ 31, 30, 162, 599, 179, 178, 590, 767, 593, 103, - /* 200 */ 596, 161, 162, 599, 756, 17, 590, 148, 593, 152, - /* 210 */ 596, 154, 26, 88, 87, 142, 185, 567, 568, 16, - /* 220 */ 235, 147, 234, 233, 159, 160, 219, 232, 196, 231, - /* 230 */ 230, 229, 801, 76, 159, 160, 162, 599, 547, 228, - /* 240 */ 590, 3, 593, 17, 596, 74, 78, 83, 86, 77, - /* 250 */ 26, 39, 37, 40, 38, 80, 59, 754, 21, 34, - /* 260 */ 33, 544, 60, 32, 31, 30, 18, 140, 159, 160, - /* 270 */ 181, 737, 539, 736, 27, 734, 735, 150, 682, 184, - /* 280 */ 738, 126, 740, 741, 739, 674, 531, 141, 126, 528, - /* 290 */ 42, 529, 558, 530, 752, 591, 46, 594, 34, 33, - /* 300 */ 42, 598, 32, 31, 30, 116, 117, 68, 64, 67, - /* 310 */ 588, 598, 143, 50, 73, 72, 597, 170, 171, 130, - /* 320 */ 128, 91, 90, 89, 98, 47, 597, 144, 559, 616, - /* 330 */ 51, 26, 14, 13, 42, 145, 600, 521, 520, 201, - /* 340 */ 13, 46, 22, 22, 48, 598, 589, 10, 9, 535, - /* 350 */ 533, 536, 534, 85, 84, 146, 137, 133, 856, 138, - /* 360 */ 597, 136, 755, 812, 811, 163, 808, 807, 164, 777, - /* 370 */ 747, 218, 794, 100, 793, 769, 114, 115, 26, 684, - /* 380 */ 112, 202, 131, 183, 24, 211, 681, 212, 855, 532, - /* 390 */ 70, 854, 93, 852, 118, 702, 554, 25, 23, 132, - /* 400 */ 671, 79, 52, 186, 669, 81, 82, 667, 666, 172, - /* 410 */ 127, 664, 190, 663, 662, 661, 660, 652, 129, 658, - /* 420 */ 49, 656, 654, 766, 781, 782, 795, 104, 195, 44, - /* 430 */ 193, 191, 189, 187, 210, 105, 75, 28, 221, 199, - /* 440 */ 222, 223, 53, 225, 224, 149, 226, 62, 65, 703, - /* 450 */ 227, 237, 630, 173, 174, 629, 177, 665, 628, 119, - /* 460 */ 176, 92, 121, 125, 120, 751, 122, 123, 659, 124, - /* 470 */ 108, 106, 107, 109, 110, 111, 94, 1, 2, 621, - /* 480 */ 180, 184, 541, 55, 57, 555, 101, 156, 188, 198, - /* 490 */ 19, 63, 5, 560, 102, 4, 6, 601, 20, 15, - /* 500 */ 7, 488, 484, 200, 482, 481, 480, 477, 450, 209, - /* 510 */ 66, 45, 22, 69, 71, 517, 516, 514, 471, 54, - /* 520 */ 469, 461, 467, 463, 465, 459, 457, 487, 486, 485, - /* 530 */ 483, 479, 478, 476, 46, 448, 421, 419, 634, 633, - /* 540 */ 633, 633, 633, 633, 633, 633, 633, 633, 633, 633, - /* 550 */ 96, 97, + /* 0 */ 105, 448, 137, 677, 636, 244, 128, 517, 137, 449, + /* 10 */ 137, 160, 853, 41, 43, 11, 35, 36, 852, 159, + /* 20 */ 853, 29, 136, 448, 199, 39, 37, 40, 38, 157, + /* 30 */ 105, 449, 141, 34, 33, 219, 218, 32, 31, 30, + /* 40 */ 41, 43, 771, 35, 36, 32, 31, 30, 29, 760, + /* 50 */ 448, 199, 39, 37, 40, 38, 184, 808, 449, 194, + /* 60 */ 34, 33, 21, 21, 32, 31, 30, 408, 409, 410, + /* 70 */ 411, 412, 413, 414, 415, 416, 417, 418, 419, 243, + /* 80 */ 41, 43, 230, 35, 36, 196, 849, 60, 29, 21, + /* 90 */ 848, 199, 39, 37, 40, 38, 168, 169, 757, 757, + /* 100 */ 34, 33, 170, 56, 32, 31, 30, 782, 847, 16, + /* 110 */ 237, 210, 236, 235, 209, 208, 207, 234, 206, 233, + /* 120 */ 232, 231, 205, 217, 153, 757, 736, 590, 723, 724, + /* 130 */ 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, + /* 140 */ 735, 43, 8, 35, 36, 63, 115, 21, 29, 155, + /* 150 */ 242, 199, 39, 37, 40, 38, 241, 240, 97, 779, + /* 160 */ 34, 33, 167, 101, 32, 31, 30, 171, 35, 36, + /* 170 */ 216, 215, 596, 29, 599, 105, 199, 39, 37, 40, + /* 180 */ 38, 222, 760, 757, 238, 34, 33, 177, 12, 32, + /* 190 */ 31, 30, 164, 603, 181, 180, 594, 771, 597, 105, + /* 200 */ 600, 163, 164, 603, 760, 17, 594, 150, 597, 154, + /* 210 */ 600, 156, 26, 90, 89, 144, 187, 571, 572, 16, + /* 220 */ 237, 149, 236, 235, 161, 162, 221, 234, 198, 233, + /* 230 */ 232, 231, 807, 78, 161, 162, 164, 603, 549, 230, + /* 240 */ 594, 3, 597, 17, 600, 76, 80, 85, 88, 79, + /* 250 */ 26, 39, 37, 40, 38, 82, 61, 758, 21, 34, + /* 260 */ 33, 546, 62, 32, 31, 30, 18, 142, 161, 162, + /* 270 */ 183, 741, 541, 740, 27, 738, 739, 152, 686, 186, + /* 280 */ 742, 128, 744, 745, 743, 678, 533, 143, 128, 530, + /* 290 */ 42, 531, 562, 532, 756, 595, 46, 598, 34, 33, + /* 300 */ 42, 602, 32, 31, 30, 118, 119, 70, 66, 69, + /* 310 */ 592, 602, 145, 50, 75, 74, 601, 172, 173, 132, + /* 320 */ 130, 93, 92, 91, 100, 47, 601, 146, 563, 620, + /* 330 */ 51, 26, 14, 13, 42, 147, 604, 523, 522, 203, + /* 340 */ 13, 46, 22, 22, 48, 602, 593, 10, 9, 537, + /* 350 */ 535, 538, 536, 87, 86, 148, 139, 135, 862, 140, + /* 360 */ 601, 138, 759, 818, 817, 165, 814, 813, 166, 781, + /* 370 */ 751, 220, 800, 786, 788, 773, 102, 799, 26, 116, + /* 380 */ 114, 117, 688, 185, 204, 133, 24, 213, 685, 534, + /* 390 */ 214, 861, 95, 72, 860, 858, 558, 120, 706, 25, + /* 400 */ 23, 134, 52, 188, 675, 81, 673, 83, 84, 671, + /* 410 */ 670, 174, 192, 129, 668, 667, 666, 665, 664, 656, + /* 420 */ 49, 131, 662, 660, 658, 770, 57, 58, 801, 44, + /* 430 */ 197, 195, 193, 191, 189, 28, 106, 212, 77, 223, + /* 440 */ 224, 225, 226, 201, 53, 227, 228, 229, 239, 64, + /* 450 */ 67, 634, 151, 175, 176, 633, 178, 179, 632, 669, + /* 460 */ 186, 625, 94, 96, 123, 127, 2, 122, 707, 755, + /* 470 */ 121, 124, 125, 111, 107, 108, 126, 109, 110, 112, + /* 480 */ 663, 113, 182, 1, 543, 55, 59, 559, 103, 158, + /* 490 */ 564, 5, 190, 104, 6, 65, 490, 605, 4, 19, + /* 500 */ 20, 15, 200, 7, 202, 486, 484, 483, 482, 479, + /* 510 */ 452, 211, 68, 45, 22, 71, 73, 519, 518, 516, + /* 520 */ 54, 473, 471, 463, 469, 465, 467, 461, 459, 489, + /* 530 */ 488, 487, 485, 481, 480, 478, 46, 450, 423, 421, + /* 540 */ 638, 637, 637, 637, 637, 637, 637, 637, 637, 637, + /* 550 */ 637, 637, 98, 99, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 211, 1, 262, 215, 208, 209, 218, 5, 262, 9, @@ -299,25 +299,25 @@ static const YYCODETYPE yy_lookahead[] = { /* 340 */ 104, 104, 104, 104, 123, 110, 37, 129, 130, 5, /* 350 */ 5, 7, 7, 72, 73, 262, 262, 262, 248, 262, /* 360 */ 125, 262, 248, 243, 243, 243, 243, 243, 243, 211, - /* 370 */ 244, 243, 269, 211, 269, 246, 211, 211, 106, 211, + /* 370 */ 244, 243, 269, 211, 211, 246, 211, 269, 106, 211, /* 380 */ 250, 211, 211, 246, 211, 211, 211, 211, 211, 105, /* 390 */ 211, 211, 59, 211, 211, 211, 110, 211, 211, 211, /* 400 */ 211, 211, 120, 265, 211, 211, 211, 211, 211, 211, /* 410 */ 211, 211, 265, 211, 211, 211, 211, 211, 211, 211, - /* 420 */ 122, 211, 211, 259, 212, 212, 212, 258, 114, 119, - /* 430 */ 118, 113, 112, 111, 75, 257, 84, 124, 83, 212, - /* 440 */ 49, 80, 212, 53, 82, 212, 81, 216, 216, 226, - /* 450 */ 79, 75, 5, 136, 5, 5, 5, 212, 5, 225, - /* 460 */ 136, 213, 220, 219, 224, 246, 223, 221, 212, 222, - /* 470 */ 254, 256, 255, 253, 252, 251, 213, 217, 214, 87, - /* 480 */ 127, 107, 100, 108, 104, 100, 99, 1, 99, 101, - /* 490 */ 104, 72, 115, 100, 99, 99, 115, 100, 104, 99, - /* 500 */ 99, 9, 5, 101, 5, 5, 5, 5, 76, 15, - /* 510 */ 72, 16, 104, 130, 130, 5, 5, 100, 5, 99, - /* 520 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 530 */ 5, 5, 5, 5, 104, 76, 59, 58, 0, 273, - /* 540 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 550 */ 21, 21, 273, 273, 273, 273, 273, 273, 273, 273, + /* 420 */ 122, 211, 211, 211, 211, 259, 212, 212, 212, 119, + /* 430 */ 114, 118, 113, 112, 111, 124, 258, 75, 84, 83, + /* 440 */ 49, 80, 82, 212, 212, 53, 81, 79, 75, 216, + /* 450 */ 216, 5, 212, 136, 5, 5, 136, 5, 5, 212, + /* 460 */ 107, 87, 213, 213, 220, 219, 214, 224, 226, 246, + /* 470 */ 225, 223, 221, 253, 257, 256, 222, 255, 254, 252, + /* 480 */ 212, 251, 127, 217, 100, 108, 104, 100, 99, 1, + /* 490 */ 100, 115, 99, 99, 115, 72, 9, 100, 99, 104, + /* 500 */ 104, 99, 101, 99, 101, 5, 5, 5, 5, 5, + /* 510 */ 76, 15, 72, 16, 104, 130, 130, 5, 5, 100, + /* 520 */ 99, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 530 */ 5, 5, 5, 5, 5, 5, 104, 76, 59, 58, + /* 540 */ 0, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 550 */ 273, 273, 21, 21, 273, 273, 273, 273, 273, 273, /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 570 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 580 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, @@ -337,83 +337,84 @@ static const YYCODETYPE yy_lookahead[] = { /* 720 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 730 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 740 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 760 */ 273, }; -#define YY_SHIFT_COUNT (242) +#define YY_SHIFT_COUNT (244) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (538) +#define YY_SHIFT_MAX (540) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 144, 24, 134, 191, 235, 49, 49, 49, 49, 49, /* 10 */ 49, 0, 22, 235, 284, 284, 284, 106, 49, 49, - /* 20 */ 49, 49, 49, 161, 4, 4, 552, 201, 235, 235, + /* 20 */ 49, 49, 49, 161, 4, 4, 554, 201, 235, 235, /* 30 */ 235, 235, 235, 235, 235, 235, 235, 235, 235, 235, /* 40 */ 235, 235, 235, 235, 235, 284, 284, 2, 2, 2, - /* 50 */ 2, 2, 2, 43, 2, 225, 49, 49, 101, 101, - /* 60 */ 157, 49, 49, 49, 49, 49, 49, 49, 49, 49, + /* 50 */ 2, 2, 2, 43, 2, 225, 49, 49, 49, 49, + /* 60 */ 101, 101, 157, 49, 49, 49, 49, 49, 49, 49, /* 70 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, /* 80 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - /* 90 */ 49, 49, 49, 49, 49, 49, 49, 49, 272, 333, - /* 100 */ 333, 286, 286, 333, 282, 298, 310, 314, 312, 318, - /* 110 */ 320, 322, 313, 272, 333, 333, 359, 359, 333, 352, - /* 120 */ 355, 391, 361, 362, 390, 365, 371, 333, 376, 333, - /* 130 */ 376, 552, 552, 27, 67, 67, 67, 127, 152, 226, - /* 140 */ 226, 226, 181, 265, 265, 265, 265, 241, 255, 39, - /* 150 */ 60, 8, 8, 96, 172, 192, 228, 229, 236, 167, - /* 160 */ 290, 309, 142, 221, 209, 237, 238, 239, 185, 218, - /* 170 */ 344, 345, 281, 447, 317, 449, 450, 324, 451, 453, - /* 180 */ 392, 353, 374, 382, 375, 380, 385, 387, 486, 389, - /* 190 */ 393, 395, 386, 377, 394, 381, 397, 396, 400, 388, - /* 200 */ 401, 402, 419, 492, 497, 499, 500, 501, 502, 432, - /* 210 */ 494, 438, 495, 383, 384, 408, 510, 511, 417, 420, - /* 220 */ 408, 513, 515, 516, 517, 518, 519, 520, 521, 522, - /* 230 */ 523, 524, 525, 526, 527, 528, 430, 459, 529, 530, - /* 240 */ 477, 479, 538, + /* 90 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + /* 100 */ 272, 333, 333, 286, 286, 333, 282, 298, 310, 316, + /* 110 */ 313, 319, 321, 323, 311, 272, 333, 333, 362, 362, + /* 120 */ 333, 354, 356, 391, 361, 360, 392, 365, 368, 333, + /* 130 */ 373, 333, 373, 554, 554, 27, 67, 67, 67, 127, + /* 140 */ 152, 226, 226, 226, 181, 265, 265, 265, 265, 241, + /* 150 */ 255, 39, 60, 8, 8, 96, 172, 192, 228, 229, + /* 160 */ 236, 167, 290, 309, 142, 221, 209, 237, 238, 239, + /* 170 */ 185, 218, 344, 345, 281, 446, 317, 449, 450, 320, + /* 180 */ 452, 453, 374, 355, 353, 384, 377, 382, 387, 389, + /* 190 */ 488, 393, 390, 394, 395, 376, 396, 379, 397, 399, + /* 200 */ 402, 401, 404, 403, 423, 487, 500, 501, 502, 503, + /* 210 */ 504, 434, 496, 440, 497, 385, 386, 410, 512, 513, + /* 220 */ 419, 421, 410, 516, 517, 518, 519, 520, 521, 522, + /* 230 */ 523, 524, 525, 526, 527, 528, 529, 530, 432, 461, + /* 240 */ 531, 532, 479, 481, 540, }; -#define YY_REDUCE_COUNT (132) +#define YY_REDUCE_COUNT (134) #define YY_REDUCE_MIN (-260) -#define YY_REDUCE_MAX (264) +#define YY_REDUCE_MAX (268) static const short yy_reduce_ofst[] = { /* 0 */ -204, -101, 44, -260, -252, -211, -181, -149, -148, -122, /* 10 */ -64, -104, -61, -254, -199, -66, -44, -49, -48, -36, /* 20 */ -12, 15, 47, -212, 63, 70, 13, -247, -240, -230, /* 30 */ -176, -172, -154, -138, -53, 5, 25, 50, 65, 73, /* 40 */ 93, 94, 95, 97, 99, 110, 114, 120, 121, 122, - /* 50 */ 123, 124, 125, 126, 128, 129, 158, 162, 103, 105, - /* 60 */ 130, 165, 166, 168, 170, 171, 173, 174, 175, 176, + /* 50 */ 123, 124, 125, 126, 128, 129, 158, 162, 163, 165, + /* 60 */ 103, 108, 130, 168, 170, 171, 173, 174, 175, 176, /* 70 */ 177, 179, 180, 182, 183, 184, 186, 187, 188, 189, /* 80 */ 190, 193, 194, 195, 196, 197, 198, 199, 200, 202, - /* 90 */ 203, 204, 205, 206, 207, 208, 210, 211, 137, 212, - /* 100 */ 213, 138, 147, 214, 164, 169, 178, 215, 217, 216, - /* 110 */ 220, 222, 224, 219, 227, 230, 231, 232, 233, 223, - /* 120 */ 234, 240, 242, 243, 246, 247, 244, 245, 248, 256, - /* 130 */ 263, 260, 264, + /* 90 */ 203, 204, 205, 206, 207, 208, 210, 211, 212, 213, + /* 100 */ 137, 214, 215, 138, 147, 216, 166, 178, 217, 219, + /* 110 */ 222, 224, 220, 227, 230, 223, 231, 232, 233, 234, + /* 120 */ 240, 242, 245, 243, 244, 248, 251, 254, 246, 247, + /* 130 */ 249, 268, 250, 266, 252, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 631, 683, 672, 849, 849, 631, 631, 631, 631, 631, - /* 10 */ 631, 779, 649, 849, 631, 631, 631, 631, 631, 631, - /* 20 */ 631, 631, 631, 685, 685, 685, 774, 631, 631, 631, - /* 30 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 40 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 50 */ 631, 631, 631, 631, 631, 631, 631, 631, 798, 798, - /* 60 */ 772, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 70 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 670, - /* 80 */ 631, 668, 631, 631, 631, 631, 631, 631, 631, 631, - /* 90 */ 631, 631, 631, 631, 631, 657, 631, 631, 631, 651, - /* 100 */ 651, 631, 631, 651, 805, 809, 803, 791, 799, 790, - /* 110 */ 786, 785, 813, 631, 651, 651, 680, 680, 651, 701, - /* 120 */ 699, 697, 689, 695, 691, 693, 687, 651, 678, 651, - /* 130 */ 678, 718, 733, 631, 814, 848, 804, 832, 831, 844, - /* 140 */ 838, 837, 631, 836, 835, 834, 833, 631, 631, 631, - /* 150 */ 631, 840, 839, 631, 631, 631, 631, 631, 631, 631, - /* 160 */ 631, 631, 816, 810, 806, 631, 631, 631, 631, 631, - /* 170 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 180 */ 631, 631, 771, 631, 631, 780, 631, 631, 631, 631, - /* 190 */ 631, 631, 800, 631, 792, 631, 631, 631, 631, 631, - /* 200 */ 631, 748, 631, 631, 631, 631, 631, 631, 631, 631, - /* 210 */ 631, 631, 631, 631, 631, 853, 631, 631, 631, 742, - /* 220 */ 851, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 230 */ 631, 631, 631, 631, 631, 631, 704, 631, 655, 653, - /* 240 */ 631, 647, 631, + /* 0 */ 635, 687, 676, 855, 855, 635, 635, 635, 635, 635, + /* 10 */ 635, 783, 653, 855, 635, 635, 635, 635, 635, 635, + /* 20 */ 635, 635, 635, 689, 689, 689, 778, 635, 635, 635, + /* 30 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 40 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 50 */ 635, 635, 635, 635, 635, 635, 635, 785, 787, 635, + /* 60 */ 804, 804, 776, 635, 635, 635, 635, 635, 635, 635, + /* 70 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 80 */ 635, 674, 635, 672, 635, 635, 635, 635, 635, 635, + /* 90 */ 635, 635, 635, 635, 635, 635, 635, 661, 635, 635, + /* 100 */ 635, 655, 655, 635, 635, 655, 811, 815, 809, 797, + /* 110 */ 805, 796, 792, 791, 819, 635, 655, 655, 684, 684, + /* 120 */ 655, 705, 703, 701, 693, 699, 695, 697, 691, 655, + /* 130 */ 682, 655, 682, 722, 737, 635, 820, 854, 810, 838, + /* 140 */ 837, 850, 844, 843, 635, 842, 841, 840, 839, 635, + /* 150 */ 635, 635, 635, 846, 845, 635, 635, 635, 635, 635, + /* 160 */ 635, 635, 635, 635, 822, 816, 812, 635, 635, 635, + /* 170 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 180 */ 635, 635, 635, 635, 775, 635, 635, 784, 635, 635, + /* 190 */ 635, 635, 635, 635, 806, 635, 798, 635, 635, 635, + /* 200 */ 635, 635, 635, 752, 635, 635, 635, 635, 635, 635, + /* 210 */ 635, 635, 635, 635, 635, 635, 635, 859, 635, 635, + /* 220 */ 635, 746, 857, 635, 635, 635, 635, 635, 635, 635, + /* 230 */ 635, 635, 635, 635, 635, 635, 635, 635, 708, 635, + /* 240 */ 659, 657, 635, 651, 635, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1154,84 +1155,86 @@ static const char *const yyRuleName[] = { /* 145 */ "as ::=", /* 146 */ "from ::= FROM tablelist", /* 147 */ "tablelist ::= ids cpxName", - /* 148 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 149 */ "tmvar ::= VARIABLE", - /* 150 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 151 */ "interval_opt ::=", - /* 152 */ "fill_opt ::=", - /* 153 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 154 */ "fill_opt ::= FILL LP ID RP", - /* 155 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 156 */ "sliding_opt ::=", - /* 157 */ "orderby_opt ::=", - /* 158 */ "orderby_opt ::= ORDER BY sortlist", - /* 159 */ "sortlist ::= sortlist COMMA item sortorder", - /* 160 */ "sortlist ::= item sortorder", - /* 161 */ "item ::= ids cpxName", - /* 162 */ "sortorder ::= ASC", - /* 163 */ "sortorder ::= DESC", - /* 164 */ "sortorder ::=", - /* 165 */ "groupby_opt ::=", - /* 166 */ "groupby_opt ::= GROUP BY grouplist", - /* 167 */ "grouplist ::= grouplist COMMA item", - /* 168 */ "grouplist ::= item", - /* 169 */ "having_opt ::=", - /* 170 */ "having_opt ::= HAVING expr", - /* 171 */ "limit_opt ::=", - /* 172 */ "limit_opt ::= LIMIT signed", - /* 173 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 174 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 175 */ "slimit_opt ::=", - /* 176 */ "slimit_opt ::= SLIMIT signed", - /* 177 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 178 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 179 */ "where_opt ::=", - /* 180 */ "where_opt ::= WHERE expr", - /* 181 */ "expr ::= LP expr RP", - /* 182 */ "expr ::= ID", - /* 183 */ "expr ::= ID DOT ID", - /* 184 */ "expr ::= ID DOT STAR", - /* 185 */ "expr ::= INTEGER", - /* 186 */ "expr ::= MINUS INTEGER", - /* 187 */ "expr ::= PLUS INTEGER", - /* 188 */ "expr ::= FLOAT", - /* 189 */ "expr ::= MINUS FLOAT", - /* 190 */ "expr ::= PLUS FLOAT", - /* 191 */ "expr ::= STRING", - /* 192 */ "expr ::= NOW", - /* 193 */ "expr ::= VARIABLE", - /* 194 */ "expr ::= BOOL", - /* 195 */ "expr ::= ID LP exprlist RP", - /* 196 */ "expr ::= ID LP STAR RP", - /* 197 */ "expr ::= expr AND expr", - /* 198 */ "expr ::= expr OR expr", - /* 199 */ "expr ::= expr LT expr", - /* 200 */ "expr ::= expr GT expr", - /* 201 */ "expr ::= expr LE expr", - /* 202 */ "expr ::= expr GE expr", - /* 203 */ "expr ::= expr NE expr", - /* 204 */ "expr ::= expr EQ expr", - /* 205 */ "expr ::= expr PLUS expr", - /* 206 */ "expr ::= expr MINUS expr", - /* 207 */ "expr ::= expr STAR expr", - /* 208 */ "expr ::= expr SLASH expr", - /* 209 */ "expr ::= expr REM expr", - /* 210 */ "expr ::= expr LIKE expr", - /* 211 */ "expr ::= expr IN LP exprlist RP", - /* 212 */ "exprlist ::= exprlist COMMA expritem", - /* 213 */ "exprlist ::= expritem", - /* 214 */ "expritem ::= expr", - /* 215 */ "expritem ::=", - /* 216 */ "cmd ::= RESET QUERY CACHE", - /* 217 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 218 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 222 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 223 */ "cmd ::= KILL CONNECTION INTEGER", - /* 224 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 225 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 148 */ "tablelist ::= ids cpxName ids", + /* 149 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 150 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 151 */ "tmvar ::= VARIABLE", + /* 152 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 153 */ "interval_opt ::=", + /* 154 */ "fill_opt ::=", + /* 155 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 156 */ "fill_opt ::= FILL LP ID RP", + /* 157 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 158 */ "sliding_opt ::=", + /* 159 */ "orderby_opt ::=", + /* 160 */ "orderby_opt ::= ORDER BY sortlist", + /* 161 */ "sortlist ::= sortlist COMMA item sortorder", + /* 162 */ "sortlist ::= item sortorder", + /* 163 */ "item ::= ids cpxName", + /* 164 */ "sortorder ::= ASC", + /* 165 */ "sortorder ::= DESC", + /* 166 */ "sortorder ::=", + /* 167 */ "groupby_opt ::=", + /* 168 */ "groupby_opt ::= GROUP BY grouplist", + /* 169 */ "grouplist ::= grouplist COMMA item", + /* 170 */ "grouplist ::= item", + /* 171 */ "having_opt ::=", + /* 172 */ "having_opt ::= HAVING expr", + /* 173 */ "limit_opt ::=", + /* 174 */ "limit_opt ::= LIMIT signed", + /* 175 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 176 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 177 */ "slimit_opt ::=", + /* 178 */ "slimit_opt ::= SLIMIT signed", + /* 179 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 180 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 181 */ "where_opt ::=", + /* 182 */ "where_opt ::= WHERE expr", + /* 183 */ "expr ::= LP expr RP", + /* 184 */ "expr ::= ID", + /* 185 */ "expr ::= ID DOT ID", + /* 186 */ "expr ::= ID DOT STAR", + /* 187 */ "expr ::= INTEGER", + /* 188 */ "expr ::= MINUS INTEGER", + /* 189 */ "expr ::= PLUS INTEGER", + /* 190 */ "expr ::= FLOAT", + /* 191 */ "expr ::= MINUS FLOAT", + /* 192 */ "expr ::= PLUS FLOAT", + /* 193 */ "expr ::= STRING", + /* 194 */ "expr ::= NOW", + /* 195 */ "expr ::= VARIABLE", + /* 196 */ "expr ::= BOOL", + /* 197 */ "expr ::= ID LP exprlist RP", + /* 198 */ "expr ::= ID LP STAR RP", + /* 199 */ "expr ::= expr AND expr", + /* 200 */ "expr ::= expr OR expr", + /* 201 */ "expr ::= expr LT expr", + /* 202 */ "expr ::= expr GT expr", + /* 203 */ "expr ::= expr LE expr", + /* 204 */ "expr ::= expr GE expr", + /* 205 */ "expr ::= expr NE expr", + /* 206 */ "expr ::= expr EQ expr", + /* 207 */ "expr ::= expr PLUS expr", + /* 208 */ "expr ::= expr MINUS expr", + /* 209 */ "expr ::= expr STAR expr", + /* 210 */ "expr ::= expr SLASH expr", + /* 211 */ "expr ::= expr REM expr", + /* 212 */ "expr ::= expr LIKE expr", + /* 213 */ "expr ::= expr IN LP exprlist RP", + /* 214 */ "exprlist ::= exprlist COMMA expritem", + /* 215 */ "exprlist ::= expritem", + /* 216 */ "expritem ::= expr", + /* 217 */ "expritem ::=", + /* 218 */ "cmd ::= RESET QUERY CACHE", + /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 223 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 224 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 225 */ "cmd ::= KILL CONNECTION INTEGER", + /* 226 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 227 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1837,84 +1840,86 @@ static const struct { { 263, 0 }, /* (145) as ::= */ { 250, -2 }, /* (146) from ::= FROM tablelist */ { 264, -2 }, /* (147) tablelist ::= ids cpxName */ - { 264, -4 }, /* (148) tablelist ::= tablelist COMMA ids cpxName */ - { 265, -1 }, /* (149) tmvar ::= VARIABLE */ - { 252, -4 }, /* (150) interval_opt ::= INTERVAL LP tmvar RP */ - { 252, 0 }, /* (151) interval_opt ::= */ - { 253, 0 }, /* (152) fill_opt ::= */ - { 253, -6 }, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 253, -4 }, /* (154) fill_opt ::= FILL LP ID RP */ - { 254, -4 }, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ - { 254, 0 }, /* (156) sliding_opt ::= */ - { 256, 0 }, /* (157) orderby_opt ::= */ - { 256, -3 }, /* (158) orderby_opt ::= ORDER BY sortlist */ - { 266, -4 }, /* (159) sortlist ::= sortlist COMMA item sortorder */ - { 266, -2 }, /* (160) sortlist ::= item sortorder */ - { 268, -2 }, /* (161) item ::= ids cpxName */ - { 269, -1 }, /* (162) sortorder ::= ASC */ - { 269, -1 }, /* (163) sortorder ::= DESC */ - { 269, 0 }, /* (164) sortorder ::= */ - { 255, 0 }, /* (165) groupby_opt ::= */ - { 255, -3 }, /* (166) groupby_opt ::= GROUP BY grouplist */ - { 270, -3 }, /* (167) grouplist ::= grouplist COMMA item */ - { 270, -1 }, /* (168) grouplist ::= item */ - { 257, 0 }, /* (169) having_opt ::= */ - { 257, -2 }, /* (170) having_opt ::= HAVING expr */ - { 259, 0 }, /* (171) limit_opt ::= */ - { 259, -2 }, /* (172) limit_opt ::= LIMIT signed */ - { 259, -4 }, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ - { 259, -4 }, /* (174) limit_opt ::= LIMIT signed COMMA signed */ - { 258, 0 }, /* (175) slimit_opt ::= */ - { 258, -2 }, /* (176) slimit_opt ::= SLIMIT signed */ - { 258, -4 }, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 258, -4 }, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ - { 251, 0 }, /* (179) where_opt ::= */ - { 251, -2 }, /* (180) where_opt ::= WHERE expr */ - { 262, -3 }, /* (181) expr ::= LP expr RP */ - { 262, -1 }, /* (182) expr ::= ID */ - { 262, -3 }, /* (183) expr ::= ID DOT ID */ - { 262, -3 }, /* (184) expr ::= ID DOT STAR */ - { 262, -1 }, /* (185) expr ::= INTEGER */ - { 262, -2 }, /* (186) expr ::= MINUS INTEGER */ - { 262, -2 }, /* (187) expr ::= PLUS INTEGER */ - { 262, -1 }, /* (188) expr ::= FLOAT */ - { 262, -2 }, /* (189) expr ::= MINUS FLOAT */ - { 262, -2 }, /* (190) expr ::= PLUS FLOAT */ - { 262, -1 }, /* (191) expr ::= STRING */ - { 262, -1 }, /* (192) expr ::= NOW */ - { 262, -1 }, /* (193) expr ::= VARIABLE */ - { 262, -1 }, /* (194) expr ::= BOOL */ - { 262, -4 }, /* (195) expr ::= ID LP exprlist RP */ - { 262, -4 }, /* (196) expr ::= ID LP STAR RP */ - { 262, -3 }, /* (197) expr ::= expr AND expr */ - { 262, -3 }, /* (198) expr ::= expr OR expr */ - { 262, -3 }, /* (199) expr ::= expr LT expr */ - { 262, -3 }, /* (200) expr ::= expr GT expr */ - { 262, -3 }, /* (201) expr ::= expr LE expr */ - { 262, -3 }, /* (202) expr ::= expr GE expr */ - { 262, -3 }, /* (203) expr ::= expr NE expr */ - { 262, -3 }, /* (204) expr ::= expr EQ expr */ - { 262, -3 }, /* (205) expr ::= expr PLUS expr */ - { 262, -3 }, /* (206) expr ::= expr MINUS expr */ - { 262, -3 }, /* (207) expr ::= expr STAR expr */ - { 262, -3 }, /* (208) expr ::= expr SLASH expr */ - { 262, -3 }, /* (209) expr ::= expr REM expr */ - { 262, -3 }, /* (210) expr ::= expr LIKE expr */ - { 262, -5 }, /* (211) expr ::= expr IN LP exprlist RP */ - { 271, -3 }, /* (212) exprlist ::= exprlist COMMA expritem */ - { 271, -1 }, /* (213) exprlist ::= expritem */ - { 272, -1 }, /* (214) expritem ::= expr */ - { 272, 0 }, /* (215) expritem ::= */ - { 209, -3 }, /* (216) cmd ::= RESET QUERY CACHE */ - { 209, -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 209, -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 209, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 209, -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 209, -8 }, /* (221) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 209, -9 }, /* (222) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 209, -3 }, /* (223) cmd ::= KILL CONNECTION INTEGER */ - { 209, -5 }, /* (224) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 209, -5 }, /* (225) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 264, -3 }, /* (148) tablelist ::= ids cpxName ids */ + { 264, -4 }, /* (149) tablelist ::= tablelist COMMA ids cpxName */ + { 264, -5 }, /* (150) tablelist ::= tablelist COMMA ids cpxName ids */ + { 265, -1 }, /* (151) tmvar ::= VARIABLE */ + { 252, -4 }, /* (152) interval_opt ::= INTERVAL LP tmvar RP */ + { 252, 0 }, /* (153) interval_opt ::= */ + { 253, 0 }, /* (154) fill_opt ::= */ + { 253, -6 }, /* (155) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 253, -4 }, /* (156) fill_opt ::= FILL LP ID RP */ + { 254, -4 }, /* (157) sliding_opt ::= SLIDING LP tmvar RP */ + { 254, 0 }, /* (158) sliding_opt ::= */ + { 256, 0 }, /* (159) orderby_opt ::= */ + { 256, -3 }, /* (160) orderby_opt ::= ORDER BY sortlist */ + { 266, -4 }, /* (161) sortlist ::= sortlist COMMA item sortorder */ + { 266, -2 }, /* (162) sortlist ::= item sortorder */ + { 268, -2 }, /* (163) item ::= ids cpxName */ + { 269, -1 }, /* (164) sortorder ::= ASC */ + { 269, -1 }, /* (165) sortorder ::= DESC */ + { 269, 0 }, /* (166) sortorder ::= */ + { 255, 0 }, /* (167) groupby_opt ::= */ + { 255, -3 }, /* (168) groupby_opt ::= GROUP BY grouplist */ + { 270, -3 }, /* (169) grouplist ::= grouplist COMMA item */ + { 270, -1 }, /* (170) grouplist ::= item */ + { 257, 0 }, /* (171) having_opt ::= */ + { 257, -2 }, /* (172) having_opt ::= HAVING expr */ + { 259, 0 }, /* (173) limit_opt ::= */ + { 259, -2 }, /* (174) limit_opt ::= LIMIT signed */ + { 259, -4 }, /* (175) limit_opt ::= LIMIT signed OFFSET signed */ + { 259, -4 }, /* (176) limit_opt ::= LIMIT signed COMMA signed */ + { 258, 0 }, /* (177) slimit_opt ::= */ + { 258, -2 }, /* (178) slimit_opt ::= SLIMIT signed */ + { 258, -4 }, /* (179) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 258, -4 }, /* (180) slimit_opt ::= SLIMIT signed COMMA signed */ + { 251, 0 }, /* (181) where_opt ::= */ + { 251, -2 }, /* (182) where_opt ::= WHERE expr */ + { 262, -3 }, /* (183) expr ::= LP expr RP */ + { 262, -1 }, /* (184) expr ::= ID */ + { 262, -3 }, /* (185) expr ::= ID DOT ID */ + { 262, -3 }, /* (186) expr ::= ID DOT STAR */ + { 262, -1 }, /* (187) expr ::= INTEGER */ + { 262, -2 }, /* (188) expr ::= MINUS INTEGER */ + { 262, -2 }, /* (189) expr ::= PLUS INTEGER */ + { 262, -1 }, /* (190) expr ::= FLOAT */ + { 262, -2 }, /* (191) expr ::= MINUS FLOAT */ + { 262, -2 }, /* (192) expr ::= PLUS FLOAT */ + { 262, -1 }, /* (193) expr ::= STRING */ + { 262, -1 }, /* (194) expr ::= NOW */ + { 262, -1 }, /* (195) expr ::= VARIABLE */ + { 262, -1 }, /* (196) expr ::= BOOL */ + { 262, -4 }, /* (197) expr ::= ID LP exprlist RP */ + { 262, -4 }, /* (198) expr ::= ID LP STAR RP */ + { 262, -3 }, /* (199) expr ::= expr AND expr */ + { 262, -3 }, /* (200) expr ::= expr OR expr */ + { 262, -3 }, /* (201) expr ::= expr LT expr */ + { 262, -3 }, /* (202) expr ::= expr GT expr */ + { 262, -3 }, /* (203) expr ::= expr LE expr */ + { 262, -3 }, /* (204) expr ::= expr GE expr */ + { 262, -3 }, /* (205) expr ::= expr NE expr */ + { 262, -3 }, /* (206) expr ::= expr EQ expr */ + { 262, -3 }, /* (207) expr ::= expr PLUS expr */ + { 262, -3 }, /* (208) expr ::= expr MINUS expr */ + { 262, -3 }, /* (209) expr ::= expr STAR expr */ + { 262, -3 }, /* (210) expr ::= expr SLASH expr */ + { 262, -3 }, /* (211) expr ::= expr REM expr */ + { 262, -3 }, /* (212) expr ::= expr LIKE expr */ + { 262, -5 }, /* (213) expr ::= expr IN LP exprlist RP */ + { 271, -3 }, /* (214) exprlist ::= exprlist COMMA expritem */ + { 271, -1 }, /* (215) exprlist ::= expritem */ + { 272, -1 }, /* (216) expritem ::= expr */ + { 272, 0 }, /* (217) expritem ::= */ + { 209, -3 }, /* (218) cmd ::= RESET QUERY CACHE */ + { 209, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 209, -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 209, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 209, -7 }, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 209, -8 }, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 209, -9 }, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 209, -3 }, /* (225) cmd ::= KILL CONNECTION INTEGER */ + { 209, -5 }, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 209, -5 }, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2465,29 +2470,59 @@ static void yy_reduce( {yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;} break; case 147: /* tablelist ::= ids cpxName */ -{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);} +{ + toTSDBType(yymsp[-1].minor.yy0.type); + yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); // table alias name +} yymsp[-1].minor.yy498 = yylhsminor.yy498; break; - case 148: /* tablelist ::= tablelist COMMA ids cpxName */ -{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1); } + case 148: /* tablelist ::= ids cpxName ids */ +{ + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1); +} + yymsp[-2].minor.yy498 = yylhsminor.yy498; + break; + case 149: /* tablelist ::= tablelist COMMA ids cpxName */ +{ + toTSDBType(yymsp[-1].minor.yy0.type); + yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); +} yymsp[-3].minor.yy498 = yylhsminor.yy498; break; - case 149: /* tmvar ::= VARIABLE */ + case 150: /* tablelist ::= tablelist COMMA ids cpxName ids */ +{ + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1); +} + yymsp[-4].minor.yy498 = yylhsminor.yy498; + break; + case 151: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 150: /* interval_opt ::= INTERVAL LP tmvar RP */ - case 155: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==155); + case 152: /* interval_opt ::= INTERVAL LP tmvar RP */ + case 157: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==157); {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 151: /* interval_opt ::= */ - case 156: /* sliding_opt ::= */ yytestcase(yyruleno==156); + case 153: /* interval_opt ::= */ + case 158: /* sliding_opt ::= */ yytestcase(yyruleno==158); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 152: /* fill_opt ::= */ + case 154: /* fill_opt ::= */ {yymsp[1].minor.yy498 = 0; } break; - case 153: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 155: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); @@ -2497,33 +2532,33 @@ static void yy_reduce( yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498; } break; - case 154: /* fill_opt ::= FILL LP ID RP */ + case 156: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 157: /* orderby_opt ::= */ - case 165: /* groupby_opt ::= */ yytestcase(yyruleno==165); + case 159: /* orderby_opt ::= */ + case 167: /* groupby_opt ::= */ yytestcase(yyruleno==167); {yymsp[1].minor.yy498 = 0;} break; - case 158: /* orderby_opt ::= ORDER BY sortlist */ - case 166: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==166); + case 160: /* orderby_opt ::= ORDER BY sortlist */ + case 168: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==168); {yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;} break; - case 159: /* sortlist ::= sortlist COMMA item sortorder */ + case 161: /* sortlist ::= sortlist COMMA item sortorder */ { yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } yymsp[-3].minor.yy498 = yylhsminor.yy498; break; - case 160: /* sortlist ::= item sortorder */ + case 162: /* sortlist ::= item sortorder */ { yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } yymsp[-1].minor.yy498 = yylhsminor.yy498; break; - case 161: /* item ::= ids cpxName */ + case 163: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; @@ -2532,196 +2567,196 @@ static void yy_reduce( } yymsp[-1].minor.yy134 = yylhsminor.yy134; break; - case 162: /* sortorder ::= ASC */ + case 164: /* sortorder ::= ASC */ {yymsp[0].minor.yy46 = TSDB_ORDER_ASC; } break; - case 163: /* sortorder ::= DESC */ + case 165: /* sortorder ::= DESC */ {yymsp[0].minor.yy46 = TSDB_ORDER_DESC;} break; - case 164: /* sortorder ::= */ + case 166: /* sortorder ::= */ {yymsp[1].minor.yy46 = TSDB_ORDER_ASC;} break; - case 167: /* grouplist ::= grouplist COMMA item */ + case 169: /* grouplist ::= grouplist COMMA item */ { yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } yymsp[-2].minor.yy498 = yylhsminor.yy498; break; - case 168: /* grouplist ::= item */ + case 170: /* grouplist ::= item */ { yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } yymsp[0].minor.yy498 = yylhsminor.yy498; break; - case 169: /* having_opt ::= */ - case 179: /* where_opt ::= */ yytestcase(yyruleno==179); - case 215: /* expritem ::= */ yytestcase(yyruleno==215); + case 171: /* having_opt ::= */ + case 181: /* where_opt ::= */ yytestcase(yyruleno==181); + case 217: /* expritem ::= */ yytestcase(yyruleno==217); {yymsp[1].minor.yy64 = 0;} break; - case 170: /* having_opt ::= HAVING expr */ - case 180: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==180); + case 172: /* having_opt ::= HAVING expr */ + case 182: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==182); {yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;} break; - case 171: /* limit_opt ::= */ - case 175: /* slimit_opt ::= */ yytestcase(yyruleno==175); + case 173: /* limit_opt ::= */ + case 177: /* slimit_opt ::= */ yytestcase(yyruleno==177); {yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;} break; - case 172: /* limit_opt ::= LIMIT signed */ - case 176: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==176); + case 174: /* limit_opt ::= LIMIT signed */ + case 178: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==178); {yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} break; - case 173: /* limit_opt ::= LIMIT signed OFFSET signed */ - case 177: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==177); + case 175: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 179: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==179); {yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} break; - case 174: /* limit_opt ::= LIMIT signed COMMA signed */ - case 178: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==178); + case 176: /* limit_opt ::= LIMIT signed COMMA signed */ + case 180: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==180); {yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} break; - case 181: /* expr ::= LP expr RP */ + case 183: /* expr ::= LP expr RP */ {yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; } break; - case 182: /* expr ::= ID */ + case 184: /* expr ::= ID */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 183: /* expr ::= ID DOT ID */ + case 185: /* expr ::= ID DOT ID */ {yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 184: /* expr ::= ID DOT STAR */ + case 186: /* expr ::= ID DOT STAR */ {yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 185: /* expr ::= INTEGER */ + case 187: /* expr ::= INTEGER */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 186: /* expr ::= MINUS INTEGER */ - case 187: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==187); + case 188: /* expr ::= MINUS INTEGER */ + case 189: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==189); {yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} yymsp[-1].minor.yy64 = yylhsminor.yy64; break; - case 188: /* expr ::= FLOAT */ + case 190: /* expr ::= FLOAT */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 189: /* expr ::= MINUS FLOAT */ - case 190: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==190); + case 191: /* expr ::= MINUS FLOAT */ + case 192: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==192); {yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} yymsp[-1].minor.yy64 = yylhsminor.yy64; break; - case 191: /* expr ::= STRING */ + case 193: /* expr ::= STRING */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 192: /* expr ::= NOW */ + case 194: /* expr ::= NOW */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 193: /* expr ::= VARIABLE */ + case 195: /* expr ::= VARIABLE */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 194: /* expr ::= BOOL */ + case 196: /* expr ::= BOOL */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 195: /* expr ::= ID LP exprlist RP */ + case 197: /* expr ::= ID LP exprlist RP */ { yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy64 = yylhsminor.yy64; break; - case 196: /* expr ::= ID LP STAR RP */ + case 198: /* expr ::= ID LP STAR RP */ { yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy64 = yylhsminor.yy64; break; - case 197: /* expr ::= expr AND expr */ + case 199: /* expr ::= expr AND expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 198: /* expr ::= expr OR expr */ + case 200: /* expr ::= expr OR expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 199: /* expr ::= expr LT expr */ + case 201: /* expr ::= expr LT expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 200: /* expr ::= expr GT expr */ + case 202: /* expr ::= expr GT expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 201: /* expr ::= expr LE expr */ + case 203: /* expr ::= expr LE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 202: /* expr ::= expr GE expr */ + case 204: /* expr ::= expr GE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 203: /* expr ::= expr NE expr */ + case 205: /* expr ::= expr NE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 204: /* expr ::= expr EQ expr */ + case 206: /* expr ::= expr EQ expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 205: /* expr ::= expr PLUS expr */ + case 207: /* expr ::= expr PLUS expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 206: /* expr ::= expr MINUS expr */ + case 208: /* expr ::= expr MINUS expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 207: /* expr ::= expr STAR expr */ + case 209: /* expr ::= expr STAR expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 208: /* expr ::= expr SLASH expr */ + case 210: /* expr ::= expr SLASH expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 209: /* expr ::= expr REM expr */ + case 211: /* expr ::= expr REM expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 210: /* expr ::= expr LIKE expr */ + case 212: /* expr ::= expr LIKE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 211: /* expr ::= expr IN LP exprlist RP */ + case 213: /* expr ::= expr IN LP exprlist RP */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); } yymsp[-4].minor.yy64 = yylhsminor.yy64; break; - case 212: /* exprlist ::= exprlist COMMA expritem */ + case 214: /* exprlist ::= exprlist COMMA expritem */ {yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);} yymsp[-2].minor.yy290 = yylhsminor.yy290; break; - case 213: /* exprlist ::= expritem */ + case 215: /* exprlist ::= expritem */ {yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);} yymsp[0].minor.yy290 = yylhsminor.yy290; break; - case 214: /* expritem ::= expr */ + case 216: /* expritem ::= expr */ {yylhsminor.yy64 = yymsp[0].minor.yy64;} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 216: /* cmd ::= RESET QUERY CACHE */ + case 218: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 217: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 219: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 218: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 220: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2732,14 +2767,14 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 219: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 221: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 220: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 222: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2750,7 +2785,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 221: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 223: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2764,7 +2799,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 224: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -2776,13 +2811,13 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 223: /* cmd ::= KILL CONNECTION INTEGER */ + case 225: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 224: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 226: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 225: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 227: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index e9611a3232..7b946d8589 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -130,7 +130,6 @@ void recyclePageTest() { ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); releaseResBufPage(pResultBuf, t4); - releaseResBufPage(pResultBuf, t4); tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); tFilePage* t5 = getResBufPage(pResultBuf, pageId); diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 28b1d9cefe..f8738eec9c 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -47,6 +47,8 @@ void simpleTest() { EXPECT_EQ(pTSBuf->block.numOfElem, num); tsBufDestroy(pTSBuf); + + free(list); } // one large list of ts, the ts list need to be split into several small blocks @@ -71,6 +73,7 @@ void largeTSTest() { EXPECT_EQ(pTSBuf->block.numOfElem, num); tsBufDestroy(pTSBuf); + free(list); } void multiTagsTest() { @@ -208,6 +211,8 @@ void loadDataTest() { int64_t e = taosGetTimestampUs(); printf("end:%" PRIu64 ", elapsed:%" PRIu64 ", total obj:%d\n", e, e - s, x); + tsBufDestroy(pTSBuf); + tsBufDestroy(pNewBuf); } void randomIncTsTest() {} @@ -338,6 +343,8 @@ void TSTraverse() { } } } + + tsBufDestroy(pTSBuf); } void performanceTest() {} @@ -352,9 +359,12 @@ void invalidFileTest() { STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); EXPECT_TRUE(pNewBuf == NULL); + tsBufDestroy(pNewBuf); pNewBuf = tsBufCreateFromFile("/tmp/911", true); EXPECT_TRUE(pNewBuf == NULL); + + tsBufDestroy(pNewBuf); } void mergeDiffVnodeBufferTest() { diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 70e5713ee6..13f44f20b7 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -260,7 +260,7 @@ void *rpcOpen(const SRpcInit *pInit) { } if (pRpc->connType == TAOS_CONN_SERVER) { - pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); + pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); if (pRpc->hash == NULL) { tError("%s failed to init string hash", pRpc->label); rpcClose(pRpc); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 7c4b57350a..f2abded3b6 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -96,7 +96,7 @@ static void syncModuleInitFunc() { return; } - vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); + vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); if (vgIdHash == NULL) { taosTmrCleanUp(syncTmrCtrl); taosCloseTcpThreadPool(tsTcpPool); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index eb1f1fc340..a84bb69777 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -443,7 +443,7 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { goto _err; } - pMeta->uidMap = taosHashInit((size_t)(TSDB_INIT_NTABLES * 1.1), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pMeta->uidMap = taosHashInit((size_t)(TSDB_INIT_NTABLES * 1.1), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); if (pMeta->uidMap == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 010955388b..07b50301d3 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -208,6 +208,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab if (pQueryHandle->statis == NULL) { goto out_of_memory; } + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array? if (pQueryHandle->pColumns == NULL) { goto out_of_memory; @@ -229,8 +230,9 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab if (pQueryHandle->pTableCheckInfo == NULL) { goto out_of_memory; } + STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); + assert(pMeta != NULL && sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i); @@ -239,18 +241,26 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab assert(gsize > 0); for (int32_t j = 0; j < gsize; ++j) { - STable* pTable = (STable*) taosArrayGetP(group, j); + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); STableCheckInfo info = { - .lastKey = pQueryHandle->window.skey, - //.tableId = pTable->tableId, - .pTableObj = pTable, + .lastKey = pKeyInfo->lastKey, + .tableId = ((STable*)(pKeyInfo->pTable))->tableId, + .pTableObj = pKeyInfo->pTable, }; - info.tableId = pTable->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); + info.tableId.tid = info.pTableObj->tableId.tid; + info.tableId.uid = info.pTableObj->tableId.uid; + + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + assert(info.lastKey >= pQueryHandle->window.skey); + } else { + assert(info.lastKey <= pQueryHandle->window.skey); + } + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %p", pQueryHandle, info.tableId.uid, info.tableId.tid, info.lastKey, qinfo); @@ -326,17 +336,23 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - // TODO: add uid check - if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables && - pHandle->mem->tData[pCheckInfo->tableId.tid] != NULL) { - pCheckInfo->iter = tSkipListCreateIterFromVal(pHandle->mem->tData[pCheckInfo->tableId.tid]->pData, - (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + STableData* pMem = NULL; + STableData* pIMem = NULL; + + if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) { + pMem = pHandle->mem->tData[pCheckInfo->tableId.tid]; + if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid + pCheckInfo->iter = + tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + } } - if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables && - pHandle->imem->tData[pCheckInfo->tableId.tid] != NULL) { - pCheckInfo->iiter = tSkipListCreateIterFromVal(pHandle->imem->tData[pCheckInfo->tableId.tid]->pData, - (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) { + pIMem = pHandle->imem->tData[pCheckInfo->tableId.tid]; + if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid + pCheckInfo->iiter = + tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + } } // both iterators are NULL, no data in buffer right now @@ -356,8 +372,17 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 ", lastKey:%" PRId64 ", %p", + pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, + pCheckInfo->lastKey, pHandle->qinfo); + + if (ASCENDING_TRAVERSE(order)) { + assert(pCheckInfo->lastKey <= key); + } else { + assert(pCheckInfo->lastKey >= key); + } + } else { tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pHandle->qinfo); @@ -369,8 +394,16 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 ", lastKey:%" PRId64 ", %p", + pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, + pCheckInfo->lastKey, pHandle->qinfo); + + if (ASCENDING_TRAVERSE(order)) { + assert(pCheckInfo->lastKey <= key); + } else { + assert(pCheckInfo->lastKey >= key); + } } else { tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pHandle->qinfo); @@ -2051,7 +2084,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) { SSkipListNode* pNode = tSkipListIterGet(iter); STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); - taosArrayPush(list, pTable); + + STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(list, &info); } tSkipListDestroyIter(iter); @@ -2089,7 +2124,7 @@ void filterPrepare(void* expr, void* param) { pInfo->sch = *pSchema; pInfo->optr = pExpr->_node.optr; pInfo->compare = getComparFunc(pSchema->type, pInfo->optr); - pInfo->param = pTSSchema; + pInfo->indexed = pTSSchema->columns->colId == pInfo->sch.colId; if (pInfo->optr == TSDB_RELATION_IN) { pInfo->q = (char*) pCond->arr; @@ -2107,8 +2142,8 @@ typedef struct STableGroupSupporter { int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; - STable* pTable1 = *(STable**) p1; - STable* pTable2 = *(STable**) p2; + STable* pTable1 = ((STableKeyInfo*) p1)->pTable; + STable* pTable2 = ((STableKeyInfo*) p2)->pTable; for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) { SColIndex* pColIndex = &pTableGroupSupp->pCols[i]; @@ -2158,12 +2193,14 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp, +void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); - SArray* g = taosArrayInit(16, POINTER_BYTES); - taosArrayPush(g, &pTable); + SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; + taosArrayPush(g, &info); tsdbRefTable(pTable); for (int32_t i = 1; i < numOfTables; ++i) { @@ -2177,18 +2214,21 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable assert((*p)->type == TSDB_CHILD_TABLE); if (ret == 0) { - taosArrayPush(g, p); + STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + taosArrayPush(g, &info1); } else { taosArrayPush(pGroups, &g); // current group is ended, start a new group - g = taosArrayInit(16, POINTER_BYTES); - taosArrayPush(g, p); + g = taosArrayInit(16, sizeof(STableKeyInfo)); + + STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + taosArrayPush(g, &info1); } } taosArrayPush(pGroups, &g); } -SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) { +SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -2199,13 +2239,16 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - SArray* sa = taosArrayInit(size, POINTER_BYTES); - for(int32_t i = 0; i < size; ++i) { - STable** pTable = taosArrayGet(pTableList, i); - assert((*pTable)->type == TSDB_CHILD_TABLE); + SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); - tsdbRefTable(*pTable); - taosArrayPush(sa, pTable); + for(int32_t i = 0; i < size; ++i) { + STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); + assert(((STable*)pKeyInfo->pTable)->type == TSDB_CHILD_TABLE); + + tsdbRefTable(pKeyInfo->pTable); + + STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; + taosArrayPush(sa, &info); } taosArrayPush(pTableGroup, &sa); @@ -2216,8 +2259,8 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC pSupp->pTagSchema = pTagSchema; pSupp->pCols = pCols; - taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), pSupp, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, pSupp, tableGroupComparFn); taosTFree(pSupp); } @@ -2290,7 +2333,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -2314,7 +2357,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT } //NOTE: not add ref count for super table - SArray* res = taosArrayInit(8, POINTER_BYTES); + SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); STSchema* pTagSchema = tsdbGetTableTagSchema(pTable); // no tags and tbname condition, all child tables of this stable are involved @@ -2326,7 +2369,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT } pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables); taosArrayDestroy(res); @@ -2369,7 +2412,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT doQueryTableList(pTable, res, expr); pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%zu, belong to %zu groups", tsdb, pTable->tableId.tid, pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); @@ -2383,7 +2426,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT return terrno; } -int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -2400,9 +2443,11 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* p pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); - SArray* group = taosArrayInit(1, POINTER_BYTES); + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = pTable, .lastKey = startKey}; + taosArrayPush(group, &info); - taosArrayPush(group, &pTable); taosArrayPush(pGroupInfo->pGroupList, &group); return TSDB_CODE_SUCCESS; @@ -2419,7 +2464,7 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa assert(pTableIdList != NULL); size_t size = taosArrayGetSize(pTableIdList); pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); - SArray* group = taosArrayInit(1, POINTER_BYTES); + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); int32_t i = 0; for(; i < size; ++i) { @@ -2437,7 +2482,9 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa } tsdbRefTable(pTable); - taosArrayPush(group, &pTable); + + STableKeyInfo info = {.pTable = pTable, .lastKey = id->key}; + taosArrayPush(group, &info); } if (tsdbUnlockRepoMeta(tsdb) < 0) { diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 137777f3cb..f289a4e8c3 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -20,7 +20,9 @@ extern "C" { #endif +#include "tarray.h" #include "hashfunc.h" +#include "tlockfree.h" #define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_DEFAULT_LOAD_FACTOR (0.75) @@ -29,38 +31,45 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { - char *key; -// union { - struct SHashNode * prev; -// struct SHashEntry *prev1; -// }; -// + char *key; +// struct SHashNode *prev; struct SHashNode *next; uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash uint32_t keyLen; // length of the key - char data[]; + char *data; } SHashNode; -typedef struct SHashObj { - SHashNode **hashList; - size_t capacity; // number of slots - size_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _hash_free_fn_t freeFp; // hash node free callback function +typedef enum SHashLockTypeE { + HASH_NO_LOCK = 0, + HASH_ENTRY_LOCK = 1, +} SHashLockTypeE; -#if defined(LINUX) - pthread_rwlock_t *lock; -#else - pthread_mutex_t *lock; -#endif +typedef struct SHashEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; +} SHashEntry; + +typedef struct SHashObj { + SHashEntry **hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _hash_free_fn_t freeFp; // hash node free callback function + + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; typedef struct SHashMutableIterator { - SHashObj * pHashObj; + SHashObj *pHashObj; int32_t entryIndex; SHashNode *pCur; - SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current - int32_t num; // already check number of elements in hash table + SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current + size_t numOfChecked; // already check number of elements in hash table + size_t numOfEntries; // number of entries while the iterator is created } SHashMutableIterator; /** @@ -71,7 +80,7 @@ typedef struct SHashMutableIterator { * @param threadsafe thread safe or not * @return */ -SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe); +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type); /** * return the size of hash table @@ -101,13 +110,19 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da */ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); +void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void(*fp)(void*)); + /** * remove item with the specified key * @param pHashObj * @param key * @param keyLen */ -void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); + +int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize); + +int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); /** * clean up hash table @@ -115,13 +130,6 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashCleanup(SHashObj *pHashObj); -/** - * Set the free callback function - * This function if set will be invoked right before freeing each hash node - * @param pHashObj - */ -void taosHashSetFreecb(SHashObj *pHashObj, _hash_free_fn_t freeFp); - /** * * @param pHashObj diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 1e2aeae394..11121fcf3b 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -121,7 +121,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen * @param expireTime new expire time of data * @return */ -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); +//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); /** * Add one reference count for the exist data, and assign this data for a new owner. diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 705c52af47..c1c6f2de7a 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -24,8 +24,6 @@ extern "C" { #include "tutil.h" #include "ttokendef.h" - - #define TSQL_TBNAME "TBNAME" #define TSQL_TBNAME_L "tbname" diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 37c9146a49..8df3e3d4d3 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -18,64 +18,52 @@ #include "tulog.h" #include "taosdef.h" -static FORCE_INLINE void __wr_lock(void *lock) { - if (lock == NULL) { +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) + +#define DO_FREE_HASH_NODE(_n) \ + do { \ + taosTFree((_n)->data); \ + taosTFree(_n); \ + } while (0) + +#define FREE_HASH_NODE(_h, _n) \ + do { \ + if ((_h)->freeFp) { \ + (_h)->freeFp((_n)->data); \ + } \ + \ + DO_FREE_HASH_NODE(_n); \ + } while (0); + +static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { + return; + } + taosWLockLatch(lock); +} + +static FORCE_INLINE void __rd_lock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { return; } -#if defined(LINUX) - pthread_rwlock_wrlock(lock); -#else - pthread_mutex_lock(lock); -#endif + taosRLockLatch(lock); } -static FORCE_INLINE void __rd_lock(void *lock) { - if (lock == NULL) { +static FORCE_INLINE void __rd_unlock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { return; } -#if defined(LINUX) - pthread_rwlock_rdlock(lock); -#else - pthread_mutex_lock(lock); -#endif + taosRUnLockLatch(lock); } -static FORCE_INLINE void __unlock(void *lock) { - if (lock == NULL) { +static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { return; } -#if defined(LINUX) - pthread_rwlock_unlock(lock); -#else - pthread_mutex_unlock(lock); -#endif -} - -static FORCE_INLINE int32_t __lock_init(void *lock) { - if (lock == NULL) { - return 0; - } - -#if defined(LINUX) - return pthread_rwlock_init(lock, NULL); -#else - return pthread_mutex_init(lock, NULL); -#endif -} - -static FORCE_INLINE void __lock_destroy(void *lock) { - if (lock == NULL) { - return; - } - -#if defined(LINUX) - pthread_rwlock_destroy(lock); -#else - pthread_mutex_destroy(lock); -#endif + taosWUnLockLatch(lock); } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -86,37 +74,17 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -/** - * Get SHashNode from hashlist, nodes from trash are not included. - * @param pHashObj Cache objection - * @param key key for hash - * @param keyLen key length - * @param hashVal hash value by hash function - * @return - */ -FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) { - uint32_t hash = (*pHashObj->hashFp)(key, keyLen); - - int32_t slot = HASH_INDEX(hash, pHashObj->capacity); - SHashNode *pNode = pHashObj->hashList[slot]; - +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { + SHashNode *pNode = pe->next; while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + assert(pNode->hashVal == hashVal); break; } - + pNode = pNode->next; } - - if (pNode) { - assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot); - } - - // return the calculated hash value, to avoid calculating it again in other functions - if (hashVal != NULL) { - *hashVal = hash; - } - + return pNode; } @@ -147,7 +115,13 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize); +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) { + assert(pNode->keyLen == pNewNode->keyLen); + SWAP(pNode->key, pNewNode->key, void *); + SWAP(pNode->data, pNewNode->data, void *); + + return pNewNode; +} /** * insert the hash node at the front of the linked list @@ -155,7 +129,7 @@ static SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t key * @param pHashObj * @param pNode */ -static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode); +static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); /** * Get the next element in hash table for iterator @@ -164,7 +138,7 @@ static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode); */ static SHashNode *getNextHashNode(SHashMutableIterator *pIter); -SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (capacity == 0 || fn == NULL) { return NULL; } @@ -180,151 +154,360 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->hashFp = fn; + pHashObj->type = type; + pHashObj->enableUpdate = update; - pHashObj->hashList = (SHashNode **)calloc(pHashObj->capacity, POINTER_BYTES); + pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { free(pHashObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; - } + } else { + pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); - if (threadsafe) { -#if defined(LINUX) - pHashObj->lock = calloc(1, sizeof(pthread_rwlock_t)); -#else - pHashObj->lock = calloc(1, sizeof(pthread_mutex_t)); -#endif - } + void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + pHashObj->hashList[i] = p + i * sizeof(SHashEntry); + } - if (__lock_init(pHashObj->lock) != 0) { - free(pHashObj->hashList); - free(pHashObj); - - uError("failed to init lock, reason:%s", strerror(errno)); - return NULL; + taosArrayPush(pHashObj->pMemBlock, &p); } return pHashObj; } -size_t taosHashGetSize(const SHashObj *pHashObj) { - if (pHashObj == NULL) { - return 0; - } - - return pHashObj->size; -} +size_t taosHashGetSize(const SHashObj *pHashObj) { return (pHashObj == NULL) ? 0 : pHashObj->size; } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { - __wr_lock(pHashObj->lock); - - uint32_t hashVal = 0; - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, (uint32_t)keyLen, &hashVal); - - if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table - taosHashTableResize(pHashObj); - - SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); - if (pNewNode == NULL) { - __unlock(pHashObj->lock); - - return -1; - } - - doAddToHashTable(pHashObj, pNewNode); - } else { - SHashNode *pNewNode = doUpdateHashNode(pNode, key, keyLen, data, size); - if (pNewNode == NULL) { - __unlock(pHashObj->lock); - return -1; - } - - if (pNewNode->prev) { - pNewNode->prev->next = pNewNode; - } else { - int32_t slot = HASH_INDEX(pNewNode->hashVal, pHashObj->capacity); - - assert(pHashObj->hashList[slot] == pNode); - pHashObj->hashList[slot] = pNewNode; - } - - if (pNewNode->next) { - (pNewNode->next)->prev = pNewNode; - } + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); + SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); + if (pNewNode == NULL) { + return -1; } - __unlock(pHashObj->lock); - return 0; + // need the resize process, write lock applied + if (HASH_NEED_RESIZE(pHashObj)) { + __wr_lock(&pHashObj->lock, pHashObj->type); + taosHashTableResize(pHashObj); + __wr_unlock(&pHashObj->lock, pHashObj->type); + } + + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode *pNode = pe->next; + if (pe->num > 0) { + assert(pNode != NULL); + } else { + assert(pNode == NULL); + } + + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + assert(pNode->hashVal == hashVal); + break; + } + + pNode = pNode->next; + } + + if (pNode == NULL) { + // no data in hash table with the specified key, add it into hash table + pushfrontNodeInEntryList(pe, pNewNode); + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + // enable resize + __rd_unlock(&pHashObj->lock, pHashObj->type); + atomic_add_fetch_64(&pHashObj->size, 1); + + return 0; + } else { + // not support the update operation, return error + if (pHashObj->enableUpdate) { + doUpdateHashNode(pNode, pNewNode); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + // enable resize + __rd_unlock(&pHashObj->lock, pHashObj->type); + + DO_FREE_HASH_NODE(pNewNode); + return pHashObj->enableUpdate ? 0 : -1; + } } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - __rd_lock(pHashObj->lock); + return taosHashGetCB(pHashObj, key, keyLen, NULL); +} - uint32_t hashVal = 0; - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, (int32_t)keyLen, &hashVal); - - __unlock(pHashObj->lock); - - if (pNode != NULL) { - assert(pNode->hashVal == hashVal); - - return pNode->data; - } else { +void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *)) { + if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { return NULL; } + + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (atomic_load_32(&pe->num) == 0) { + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return NULL; + } + + char *data = NULL; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + if (pe->num > 0) { + assert(pe->next != NULL); + } else { + assert(pe->next == NULL); + } + + SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); + if (pNode != NULL) { + if (fp != NULL) { + fp(pNode->data); + } + + data = pNode->data; + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; +} + +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { + return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); +} + +int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { + if (pHashObj == NULL || pHashObj->size <= 0) { + return -1; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); + + // disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (pe->num == 0) { + assert(pe->next == NULL); + __rd_unlock(&pHashObj->lock, pHashObj->type); + return -1; + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + // double check after locked + if (pe->num == 0) { + assert(pe->next == NULL); + taosWUnLockLatch(&pe->latch); + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return -1; + } + + SHashNode *pNode = pe->next; + SHashNode *pRes = NULL; + + // remove it + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + pe->num -= 1; + pRes = pNode; + pe->next = pNode->next; + } else { + while (pNode->next != NULL) { + if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) { + assert((pNode->next)->hashVal == hashVal); + break; + } + + pNode = pNode->next; + } + + + if (pNode->next != NULL) { + pe->num -= 1; + pRes = pNode->next; + pNode->next = pNode->next->next; + } + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + + if (data != NULL && pRes != NULL) { + memcpy(data, pRes->data, dsize); + } + + if (pRes != NULL) { + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pRes); + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + return 0; + } else { + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + return -1; + } } -void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - __wr_lock(pHashObj->lock); - - uint32_t val = 0; - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, (uint32_t)keyLen, &val); - if (pNode == NULL) { - __unlock(pHashObj->lock); - return; +int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { + if (pHashObj == NULL || pHashObj->size == 0) { + return 0; } - SHashNode *pNext = pNode->next; - if (pNode->prev == NULL) { - int32_t slot = HASH_INDEX(val, pHashObj->capacity); - assert(pHashObj->hashList[slot] == pNode); - - pHashObj->hashList[slot] = pNext; - } else { - pNode->prev->next = pNext; - } - - if (pNext != NULL) { - pNext->prev = pNode->prev; + // disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t numOfEntries = pHashObj->capacity; + for (int32_t i = 0; i < numOfEntries; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + continue; + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pEntry->latch); + } + + // todo remove the first node + SHashNode *pNode = NULL; + while((pNode = pEntry->next) != NULL) { + if (fp && (!fp(param, pNode->data))) { + pEntry->num -= 1; + atomic_sub_fetch_64(&pHashObj->size, 1); + + pEntry->next = pNode->next; + + if (pEntry->num == 0) { + assert(pEntry->next == NULL); + } else { + assert(pEntry->next != NULL); + } + + FREE_HASH_NODE(pHashObj, pNode); + } else { + break; + } + } + + // handle the following node + if (pNode != NULL) { + assert(pNode == pEntry->next); + SHashNode *pNext = NULL; + + while ((pNext = pNode->next) != NULL) { + // not qualified, remove it + if (fp && (!fp(param, pNext->data))) { + pNode->next = pNext->next; + pEntry->num -= 1; + atomic_sub_fetch_64(&pHashObj->size, 1); + + if (pEntry->num == 0) { + assert(pEntry->next == NULL); + } else { + assert(pEntry->next != NULL); + } + + FREE_HASH_NODE(pHashObj, pNext); + } else { + pNode = pNext; + } + } + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pEntry->latch); + } } - pHashObj->size--; - - pNode->next = NULL; - pNode->prev = NULL; - - taosTFree(pNode); - __unlock(pHashObj->lock); + __rd_unlock(&pHashObj->lock, pHashObj->type); + return 0; } void taosHashCleanup(SHashObj *pHashObj) { - if (pHashObj == NULL) return; + if (pHashObj == NULL) { + return; + } SHashNode *pNode, *pNext; - __wr_lock(pHashObj->lock); + __wr_lock(&pHashObj->lock, pHashObj->type); if (pHashObj->hashList) { for (int32_t i = 0; i < pHashObj->capacity; ++i) { - pNode = pHashObj->hashList[i]; + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + assert(pEntry->next == 0); + continue; + } + + pNode = pEntry->next; + assert(pNode != NULL); while (pNode) { pNext = pNode->next; - if (pHashObj->freeFp) { - pHashObj->freeFp(pNode->data); - } + FREE_HASH_NODE(pHashObj, pNode); - free(pNode); pNode = pNext; } } @@ -332,20 +515,19 @@ void taosHashCleanup(SHashObj *pHashObj) { free(pHashObj->hashList); } - __unlock(pHashObj->lock); - __lock_destroy(pHashObj->lock); + __wr_unlock(&pHashObj->lock, pHashObj->type); - taosTFree(pHashObj->lock); - memset(pHashObj, 0, sizeof(SHashObj)); - free(pHashObj); -} - -void taosHashSetFreecb(SHashObj *pHashObj, _hash_free_fn_t freeFp) { - if (pHashObj == NULL || freeFp == NULL) { - return; + // destroy mem block + size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); + for (int32_t i = 0; i < memBlock; ++i) { + void *p = taosArrayGetP(pHashObj->pMemBlock, i); + taosTFree(p); } - pHashObj->freeFp = freeFp; + taosArrayDestroy(pHashObj->pMemBlock); + + memset(pHashObj, 0, sizeof(SHashObj)); + free(pHashObj); } SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { @@ -355,6 +537,9 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { } pIter->pHashObj = pHashObj; + + // keep it in local variable, in case the resize operation expand the size + pIter->numOfEntries = pHashObj->capacity; return pIter; } @@ -369,28 +554,42 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { } // check the first one - if (pIter->num == 0) { + if (pIter->numOfChecked == 0) { assert(pIter->pCur == NULL && pIter->pNext == NULL); while (1) { - SHashNode *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry == NULL) { + SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; + if (pEntry->num == 0) { + assert(pEntry->next == NULL); + pIter->entryIndex++; continue; } - pIter->pCur = pEntry; + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pEntry->latch); + } + + pIter->pCur = pEntry->next; if (pIter->pCur->next) { pIter->pNext = pIter->pCur->next; + + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pEntry->latch); + } } else { + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pEntry->latch); + } + pIter->pNext = getNextHashNode(pIter); } break; } - pIter->num++; + pIter->numOfChecked++; return true; } else { assert(pIter->pCur != NULL); @@ -400,7 +599,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { return false; } - pIter->num++; + pIter->numOfChecked++; if (pIter->pCur->next) { pIter->pNext = pIter->pCur->next; @@ -432,19 +631,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { int32_t num = 0; for (int32_t i = 0; i < pHashObj->size; ++i) { - SHashNode *pEntry = pHashObj->hashList[i]; - if (pEntry == NULL) { - continue; - } - - int32_t j = 0; - while(pEntry != NULL) { - pEntry = pEntry->next; - j++; - } - - if (num < j) { - num = j; + SHashEntry *pEntry = pHashObj->hashList[i]; + if (num < pEntry->num) { + num = pEntry->num; } } @@ -452,146 +641,166 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { } void taosHashTableResize(SHashObj *pHashObj) { - if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) { + if (!HASH_NEED_RESIZE(pHashObj)) { return; } - + // double the original capacity SHashNode *pNode = NULL; SHashNode *pNext = NULL; - - int32_t newSize = (int32_t)(pHashObj->capacity) << 1u; + + int32_t newSize = pHashObj->capacity << 1u; if (newSize > HASH_MAX_CAPACITY) { -// uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); + // uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); return; } - int32_t pointerSize = POINTER_BYTES; - void *pNewEntry = realloc(pHashObj->hashList, pointerSize * newSize); - if (pNewEntry == NULL) {// todo handle error -// uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); + int64_t st = taosGetTimestampUs(); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); + if (pNewEntryList == NULL) { // todo handle error + // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; } - - pHashObj->hashList = pNewEntry; - memset(&pHashObj->hashList[pHashObj->capacity], 0, POINTER_BYTES * (newSize - pHashObj->capacity)); - + + pHashObj->hashList = pNewEntryList; + + size_t inc = newSize - pHashObj->capacity; + void * p = calloc(inc, sizeof(SHashEntry)); + + for (int32_t i = 0; i < inc; ++i) { + pHashObj->hashList[i + pHashObj->capacity] = p + i * sizeof(SHashEntry); + } + + taosArrayPush(pHashObj->pMemBlock, &p); + pHashObj->capacity = newSize; - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - pNode = pHashObj->hashList[i]; - if (pNode != NULL) { - assert(pNode->prev == NULL); + SHashEntry *pe = pHashObj->hashList[i]; + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); } - - while (pNode) { + + if (pe->num == 0) { + assert(pe->next == NULL); + continue; + } + + while ((pNode = pe->next) != NULL) { int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - if (j == i) { // this key locates in the same slot, no need to relocate it - pNode = pNode->next; - } else { - pNext = pNode->next; - - if (pNode->prev == NULL) { // first node of the overflow linked list - pHashObj->hashList[i] = pNext; + if (j != i) { + pe->num -= 1; + pe->next = pNode->next; + + if (pe->num == 0) { + assert(pe->next == NULL); } else { - pNode->prev->next = pNext; + assert(pe->next != NULL); } - - if (pNext != NULL) { - pNext->prev = pNode->prev; - } - - // clear pointer - pNode->next = NULL; - pNode->prev = NULL; - - // added into new slot - SHashNode *pNew = pHashObj->hashList[j]; - if (pNew != NULL) { - assert(pNew->prev == NULL); - pNew->prev = pNode; - } - - pNode->next = pNew; - pHashObj->hashList[j] = pNode; - - // continue - pNode = pNext; + + SHashEntry *pNewEntry = pHashObj->hashList[j]; + pushfrontNodeInEntryList(pNewEntry, pNode); + } else { + break; } } + + if (pNode != NULL) { + while ((pNext = pNode->next) != NULL) { + int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity); + if (j != i) { + pe->num -= 1; + + pNode->next = pNext->next; + pNext->next = NULL; + + // added into new slot + SHashEntry *pNewEntry = pHashObj->hashList[j]; + + if (pNewEntry->num == 0) { + assert(pNewEntry->next == NULL); + } else { + assert(pNewEntry->next != NULL); + } + + pushfrontNodeInEntryList(pNewEntry, pNext); + } else { + pNode = pNext; + } + } + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + } + } -// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + int64_t et = taosGetTimestampUs(); + + uDebug("hash table resize completed, new capacity:%"PRId64", load factor:%f, elapsed time:%fms", pHashObj->capacity, + ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - size_t totalSize = dsize + sizeof(SHashNode) + keyLen; - - SHashNode *pNewNode = calloc(1, totalSize); + SHashNode *pNewNode = calloc(1, sizeof(SHashNode)); + if (pNewNode == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - + + pNewNode->data = malloc(dsize + keyLen); memcpy(pNewNode->data, pData, dsize); - + pNewNode->key = pNewNode->data + dsize; memcpy(pNewNode->key, key, keyLen); - pNewNode->keyLen = (uint32_t)keyLen; - + + pNewNode->keyLen = keyLen; pNewNode->hashVal = hashVal; return pNewNode; } -SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize) { - size_t size = dsize + sizeof(SHashNode) + keyLen; - - SHashNode *pNewNode = (SHashNode *)realloc(pNode, size); - if (pNewNode == NULL) { - return NULL; - } - - memcpy(pNewNode->data, pData, dsize); - - pNewNode->key = pNewNode->data + dsize; - assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen); - - memcpy(pNewNode->key, key, keyLen); - return pNewNode; -} +void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { + assert(pNode != NULL && pEntry != NULL); -void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode) { - assert(pNode != NULL); - - int32_t index = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - - SHashNode* pEntry = pHashObj->hashList[index]; - if (pEntry != NULL) { - pEntry->prev = pNode; - - pNode->next = pEntry; - pNode->prev = NULL; - } - - pHashObj->hashList[index] = pNode; - pHashObj->size++; + pNode->next = pEntry->next; + pEntry->next = pNode; + + pEntry->num += 1; } SHashNode *getNextHashNode(SHashMutableIterator *pIter) { assert(pIter != NULL); - + pIter->entryIndex++; - while (pIter->entryIndex < pIter->pHashObj->capacity) { - SHashNode *pNode = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pNode == NULL) { + SHashNode *p = NULL; + + while (pIter->entryIndex < pIter->numOfEntries) { + SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; + if (pEntry->num == 0) { pIter->entryIndex++; continue; } - - return pNode; + + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pEntry->latch); + } + + p = pEntry->next; + + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pEntry->latch); + } + + return p; } - + return NULL; } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 7eca3b2637..54a4b289b7 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -30,14 +30,6 @@ static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE void __cache_rd_lock(SCacheObj *pCacheObj) { -#if defined(LINUX) - pthread_rwlock_rdlock(&pCacheObj->lock); -#else - pthread_mutex_lock(&pCacheObj->lock); -#endif -} - static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_unlock(&pCacheObj->lock); @@ -62,13 +54,6 @@ static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) { #endif } -#if 0 -static FORCE_INLINE void taosFreeNode(void *data) { - SCacheDataNode *pNode = *(SCacheDataNode **)data; - free(pNode); -} -#endif - /** * @param key key of object for hash, usually a null-terminated string * @param keyLen length of key @@ -88,13 +73,6 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const */ static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode); -/** - * remove node in trash can - * @param pCacheObj - * @param pElem - */ -static void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem); - /** * remove nodes in trash with refCount == 0 in cache * @param pNode @@ -107,111 +85,54 @@ static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force); /** * release node * @param pCacheObj cache object - * @param pNode data node + * @param pNode data node */ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->signature != (uint64_t)pNode) { uError("key:%s, %p data is invalid, or has been released", pNode->key, pNode); + assert(0); return; } - - taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); pCacheObj->totalSize -= pNode->size; - uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", - pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, - pNode->size); + int32_t size = taosHashGetSize(pCacheObj->pHashTable); + assert(size > 0); + + uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes", + pCacheObj->name, pNode->key, pNode->data, pNode->size, size - 1, pCacheObj->totalSize); + + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pNode->data); + } - if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); free(pNode); } -/** - * move the old node into trash - * @param pCacheObj - * @param pNode - */ -static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { - taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - taosAddToTrash(pCacheObj, pNode); +static FORCE_INLINE void doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem *pElem) { + if (pElem->pData->signature != (uint64_t) pElem->pData) { + uError("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); + return; + } + + pCacheObj->numOfElemsInTrash--; + if (pElem->prev) { + pElem->prev->next = pElem->next; + } else { // pnode is the header, update header + pCacheObj->pTrash = pElem->next; + } + + if (pElem->next) { + pElem->next->prev = pElem->prev; + } } -/** - * update data in cache - * @param pCacheObj - * @param pNode - * @param key - * @param keyLen - * @param pData - * @param dataSize - * @return - */ -static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode *pNode, const char *key, int32_t keyLen, - const void *pData, uint32_t dataSize, uint64_t duration) { - SCacheDataNode *pNewNode = NULL; - - // only a node is not referenced by any other object, in-place update it - if (T_REF_VAL_GET(pNode) == 0) { - size_t newSize = sizeof(SCacheDataNode) + dataSize + keyLen + 1; - - pNewNode = (SCacheDataNode *)realloc(pNode, newSize); - if (pNewNode == NULL) { - return NULL; - } - - memset(pNewNode, 0, newSize); - pNewNode->signature = (uint64_t)pNewNode; - memcpy(pNewNode->data, pData, dataSize); - - pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + dataSize; - pNewNode->keySize = keyLen; - memcpy(pNewNode->key, key, keyLen); - - // update the timestamp information for updated key/value - pNewNode->addedTime = taosGetTimestampMs(); - pNewNode->lifespan = duration; - - T_REF_INC(pNewNode); - - // the address of this node may be changed, so the prev and next element should update the corresponding pointer - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); - } else { - taosCacheMoveToTrash(pCacheObj, pNode); - - pNewNode = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); - if (pNewNode == NULL) { - return NULL; - } - - T_REF_INC(pNewNode); - - // addedTime new element to hashtable - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); +static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem *pElem) { + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pElem->pData->data); } - - return pNewNode; -} -/** - * addedTime data into hash table - * @param key - * @param pData - * @param size - * @param pCacheObj - * @param keyLen - * @param pNode - * @return - */ -static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pCacheObj, const char *key, size_t keyLen, const void *pData, - size_t dataSize, uint64_t duration) { - SCacheDataNode *pNode = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); - if (pNode == NULL) { - return NULL; - } - - T_REF_INC(pNode); - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode, sizeof(void *)); - return pNode; + free(pElem->pData); + free(pElem); } /** @@ -237,7 +158,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false); + pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); @@ -269,36 +190,74 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext } void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { - SCacheDataNode *pNode; - if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) { return NULL; } - __cache_wr_lock(pCacheObj); - SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); - SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; - - if (pOld == NULL) { // do addedTime to cache - pNode = taosAddToCacheImpl(pCacheObj, key, keyLen, pData, dataSize, duration * 1000L); - if (NULL != pNode) { - pCacheObj->totalSize += pNode->size; - - uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64 - "bytes size:%" PRId64 "bytes", - pCacheObj->name, key, pNode->data, pNode->addedTime, pNode->expireTime, - (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); - } else { - uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); - } - } else { // old data exists, update the node - pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, (int32_t)keyLen, pData, (uint32_t)dataSize, duration * 1000L); - uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode->data, pOld); + SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); + if (pNode1 == NULL) { + uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); + return NULL; } - __cache_unlock(pCacheObj); + T_REF_INC(pNode1); - return (pNode != NULL) ? pNode->data : NULL; + int32_t succ = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); + if (succ == 0) { + atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); + uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 + ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, + (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); + } else { // duplicated key exists + while (1) { + SCacheDataNode* p = NULL; + int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); + + // add to trashcan + if (ret == 0) { + if (T_REF_VAL_GET(p) == 0) { + if (pCacheObj->freeFp) { + pCacheObj->freeFp(p->data); + } + + taosTFree(p); + } else { + taosAddToTrash(pCacheObj, p); + uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p); + } + } + + assert(T_REF_VAL_GET(pNode1) == 1); + + ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); + if (ret == 0) { + atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); + + uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 + ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, + (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); + + return pNode1->data; + + } else { + // failed, try again + } + } + } + + return pNode1->data; +} + +static void incRefFn(void* ptNode) { + assert(ptNode != NULL); + + SCacheDataNode** p = (SCacheDataNode**) ptNode; + assert(T_REF_VAL_GET(*p) >= 0); + + int32_t ret = T_REF_INC(*p); + assert(ret > 0); } void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { @@ -306,23 +265,12 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return NULL; } - void *pData = NULL; - - __cache_rd_lock(pCacheObj); - - SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); - - int32_t ref = 0; - if (ptNode != NULL) { - ref = T_REF_INC(*ptNode); - pData = (*ptNode)->data; - } - - __cache_unlock(pCacheObj); + SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGetCB(pCacheObj->pHashTable, key, keyLen, incRefFn); + void* pData = (ptNode != NULL)? (*ptNode)->data:NULL; if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, ref); + uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, T_REF_VAL_GET(*ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); @@ -332,34 +280,6 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return pData; } -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime) { - if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { - return NULL; - } - - __cache_rd_lock(pCacheObj); - - SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); - if (ptNode != NULL) { - T_REF_INC(*ptNode); - (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan; - } - - __cache_unlock(pCacheObj); - - if (ptNode != NULL) { - atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("cache:%s, key:%p, %p expireTime is updated in cache, refcnt:%d", pCacheObj->name, key, - (*ptNode)->data, T_REF_VAL_GET(*ptNode)); - } else { - atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); - } - - atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); - return (ptNode != NULL) ? (*ptNode)->data : NULL; -} - void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; @@ -420,15 +340,16 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (pCacheObj->extendLifespan && (!inTrashCan) && (!_remove)) { atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); - uDebug("cache:%s data:%p extend life time to %"PRId64 " before release", pCacheObj->name, pNode->data, pNode->expireTime); + uDebug("cache:%s data:%p extend expire time: %"PRId64, pCacheObj->name, pNode->data, pNode->expireTime); } if (_remove) { - __cache_wr_lock(pCacheObj); - // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. - int32_t ref = T_REF_DEC(pNode); - uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); + char* key = pNode->key; + char* d = pNode->data; + + int32_t ref = T_REF_VAL_GET(pNode); + uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, key, d, ref - 1); /* * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users @@ -437,50 +358,85 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread * that tries to do the same thing. */ - if (pNode->inTrashCan) { + if (inTrashCan) { + ref = T_REF_DEC(pNode); + if (ref == 0) { assert(pNode->pTNodeHeader->pData == pNode); - taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); + + __cache_wr_lock(pCacheObj); + doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); + __cache_unlock(pCacheObj); + + doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); } } else { - if (ref > 0) { - assert(pNode->pTNodeHeader == NULL); - taosCacheMoveToTrash(pCacheObj, pNode); - } else { - taosCacheReleaseNode(pCacheObj, pNode); + // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread + // when reaches here. + int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + ref = T_REF_DEC(pNode); + + // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. + // note that the remove operation can be executed only once. + if (ret == 0) { + if (ref > 0) { + assert(pNode->pTNodeHeader == NULL); + + __cache_wr_lock(pCacheObj); + taosAddToTrash(pCacheObj, pNode); + __cache_unlock(pCacheObj); + } else { // ref == 0 + atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); + + int32_t size = taosHashGetSize(pCacheObj->pHashTable); + uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes", + pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize); + + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pNode->data); + } + + free(pNode); + } } } - __cache_unlock(pCacheObj); - } else { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. - int32_t ref = T_REF_DEC(pNode); + char* key = pNode->key; + char* p = pNode->data; - uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, pNode->key, pNode->data, - ref, inTrashCan); + int32_t ref = T_REF_DEC(pNode); + uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, key, p, ref, inTrashCan); } } -void taosCacheEmpty(SCacheObj *pCacheObj) { - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); - - __cache_wr_lock(pCacheObj); - while (taosHashIterNext(pIter)) { - if (pCacheObj->deleting == 1) { - break; - } - - SCacheDataNode *pNode = *(SCacheDataNode **) taosHashIterGet(pIter); - if (T_REF_VAL_GET(pNode) == 0) { - taosCacheReleaseNode(pCacheObj, pNode); - } else { - taosCacheMoveToTrash(pCacheObj, pNode); - } +typedef struct SHashTravSupp { + SCacheObj* pCacheObj; + int64_t time; + __cache_free_fn_t fp; +} SHashTravSupp; + +static bool travHashTableEmptyFn(void* param, void* data) { + SHashTravSupp* ps = (SHashTravSupp*) param; + SCacheObj* pCacheObj= ps->pCacheObj; + + SCacheDataNode *pNode = *(SCacheDataNode **) data; + + if (T_REF_VAL_GET(pNode) == 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } else { // do add to trashcan + taosAddToTrash(pCacheObj, pNode); } - __cache_unlock(pCacheObj); - - taosHashDestroyIter(pIter); + + // this node should be remove from hash table + return false; +} + +void taosCacheEmpty(SCacheObj *pCacheObj) { + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + + taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); taosTrashCanEmpty(pCacheObj, false); } @@ -496,8 +452,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { doCleanupDataCache(pCacheObj); } -SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, - uint64_t duration) { +SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; SCacheDataNode *pNewNode = calloc(1, totalSize); @@ -543,36 +498,10 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pNode->pTNodeHeader = pElem; pCacheObj->numOfElemsInTrash++; - uDebug("key:%p, %p move to trash, numOfElem in trash:%d", pNode->key, pNode->data, pCacheObj->numOfElemsInTrash); + uDebug("%s key:%p, %p move to trash, numOfElem in trash:%d", pCacheObj->name, pNode->key, pNode->data, + pCacheObj->numOfElemsInTrash); } -void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { - if (pElem->pData->signature != (uint64_t)pElem->pData) { - uError("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); - return; - } - - pCacheObj->numOfElemsInTrash--; - if (pElem->prev) { - pElem->prev->next = pElem->next; - } else { /* pnode is the header, update header */ - pCacheObj->pTrash = pElem->next; - } - - if (pElem->next) { - pElem->next->prev = pElem->prev; - } - - pElem->pData->signature = 0; - if (pCacheObj->freeFp) { - pCacheObj->freeFp(pElem->pData->data); - } - - free(pElem->pData); - free(pElem); -} - -// TODO add another lock when scanning trashcan void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { __cache_wr_lock(pCacheObj); @@ -580,8 +509,8 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { if (pCacheObj->pTrash != NULL) { uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash); } - pCacheObj->pTrash = NULL; + pCacheObj->pTrash = NULL; __cache_unlock(pCacheObj); return; } @@ -597,10 +526,12 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { uDebug("key:%p, %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData->data, pCacheObj->numOfElemsInTrash - 1); - STrashElem *p = pElem; + STrashElem *p = pElem; pElem = pElem->next; - taosRemoveFromTrashCan(pCacheObj, p); + + doRemoveElemInTrashcan(pCacheObj, p); + doDestroyTrashcanElem(pCacheObj, p); } else { pElem = pElem->next; } @@ -610,27 +541,13 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { } void doCleanupDataCache(SCacheObj *pCacheObj) { - __cache_wr_lock(pCacheObj); - - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); - while (taosHashIterNext(pIter)) { - SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); - - int32_t c = T_REF_VAL_GET(pNode); - if (c <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - } else { - uDebug("cache:%s key:%p, %p will not remove from cache, refcnt:%d", pCacheObj->name, pNode->key, - pNode->data, T_REF_VAL_GET(pNode)); - } - } - taosHashDestroyIter(pIter); + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); - __cache_unlock(pCacheObj); - taosTrashCanEmpty(pCacheObj, true); + __cache_lock_destroy(pCacheObj); taosTFree(pCacheObj->name); @@ -638,26 +555,31 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { free(pCacheObj); } -static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t fp) { - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); +bool travHashTableFn(void* param, void* data) { + SHashTravSupp* ps = (SHashTravSupp*) param; + SCacheObj* pCacheObj= ps->pCacheObj; - __cache_wr_lock(pCacheObj); - while (taosHashIterNext(pIter)) { - SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); + SCacheDataNode* pNode = *(SCacheDataNode **) data; + if (pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); - if (pNode->expireTime < (uint64_t)time && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - continue; - } - - if (fp) { - fp(pNode->data); - } + // this node should be remove from hash table + return false; } - __cache_unlock(pCacheObj); + if (ps->fp) { + (ps->fp)(pNode->data); + } - taosHashDestroyIter(pIter); + // do not remove element in hash table + return true; +} + +static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t fp) { + assert(pCacheObj != NULL); + + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time}; + taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); } void* taosCacheTimedRefresh(void *handle) { diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 9657d82773..c026aed5f2 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -433,7 +433,7 @@ static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void pStore->iFunc = iFunc; pStore->aFunc = aFunc; pStore->appH = appH; - pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); if (pStore->map == NULL) { terrno = TSDB_CODE_COM_OUT_OF_MEMORY; goto _err; diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 8687a8005d..09523cbfb4 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -10,6 +10,6 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) + ADD_EXECUTABLE(utilTest ./cacheTest.cpp ./hashTest.cpp) TARGET_LINK_LIBRARIES(utilTest tutil common osdetail gtest pthread gcov) ENDIF() diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp index f9f636db90..bc3fed74c4 100644 --- a/src/util/tests/hashTest.cpp +++ b/src/util/tests/hashTest.cpp @@ -10,7 +10,7 @@ namespace { // the simple test code for basic operations void simpleTest() { - auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + SHashObj* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); // put 400 elements in the hash table @@ -47,7 +47,7 @@ void simpleTest() { } void stringKeyTest() { - auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; @@ -97,7 +97,7 @@ void functionTest() { * a single threads situation */ void noLockPerformanceTest() { - auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 9695f90c30..a576d43cda 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -50,7 +50,7 @@ int32_t vnodeInitResources() { vnodeInitWriteFp(); vnodeInitReadFp(); - tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); + tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); if (tsDnodeVnodesHash == NULL) { vError("failed to init vnode list"); return TSDB_CODE_VND_OUT_OF_MEMORY; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 066770e1bb..8ca76ef22d 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +//#include #include "os.h" #include "tglobal.h" @@ -66,7 +66,7 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) { +static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; pRead->pCont = qhandle; @@ -74,22 +74,23 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) { pRead->rpcMsg.handle = NULL; atomic_add_fetch_32(&pVnode->refCount, 1); + + vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead); taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); } -static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, bool* freeHandle) { +static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle, bool* freeHandle) { bool continueExec = false; int32_t code = TSDB_CODE_SUCCESS; - if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { + if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { if (continueExec) { - vDebug("QInfo:%p add to query task queue for exec", handle); - vnodePutItemIntoReadQueue(pVnode, handle); - pRet->qhandle = handle; *freeHandle = false; + vnodePutItemIntoReadQueue(pVnode, handle); + pRet->qhandle = *handle; } else { - vDebug("QInfo:%p exec completed", handle); *freeHandle = true; + vDebug("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle); } } else { pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -158,7 +159,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { // current connect is broken if (code == TSDB_CODE_SUCCESS) { handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo); - if (handle == NULL) { // failed to register qhandle + if (handle == NULL) { // failed to register qhandle, todo add error test case vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo, tstrerror(pRsp->code)); pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE; @@ -179,41 +180,40 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } if (handle != NULL) { - vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, register qhandle and return to app", vgId, *handle); - - vnodePutItemIntoReadQueue(pVnode, *handle); - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); + vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); + vnodePutItemIntoReadQueue(pVnode, handle); } - } else { assert(pCont != NULL); + void** qhandle = (void**) pCont; - handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); - if (handle == NULL) { - vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); - code = TSDB_CODE_QRY_INVALID_QHANDLE; - } else { - vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont); + vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); - bool freehandle = false; - bool buildRes = qTableQuery(*handle); // do execute query + bool freehandle = false; + bool buildRes = qTableQuery(*qhandle); // do execute query - // build query rsp - if (buildRes) { - // update the connection info according to the retrieve connection - pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*handle); - assert(pReadMsg->rpcMsg.handle != NULL); + // build query rsp, the retrieve request has reached here already + if (buildRes) { + // update the connection info according to the retrieve connection + pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); + assert(pReadMsg->rpcMsg.handle != NULL); - vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); - code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle); + vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, + pReadMsg->rpcMsg.handle); + code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle); - // todo test the error code case - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_QRY_HAS_RSP; - } + // todo test the error code case + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_HAS_RSP; } + } else { + freehandle = qQueryCompleted(*qhandle); + } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle); + // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. + // if not build result, free it not by forced. + if (freehandle || (!buildRes)) { + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); } } @@ -225,8 +225,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { SRspRet *pRet = &pReadMsg->rspRet; SRetrieveTableMsg *pRetrieve = pCont; - pRetrieve->qhandle = htobe64(pRetrieve->qhandle); pRetrieve->free = htons(pRetrieve->free); + pRetrieve->qhandle = htobe64(pRetrieve->qhandle); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle); @@ -236,18 +236,27 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle); if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) { code = TSDB_CODE_QRY_INVALID_QHANDLE; - vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle); + vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle); vnodeBuildNoResultQueryRsp(pRet); return code; } if (pRetrieve->free == 1) { - vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); + vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); vnodeBuildNoResultQueryRsp(pRet); + code = TSDB_CODE_TSC_QUERY_CANCELLED; + return code; + } + + // register the qhandle to connect to quit query immediate if connection is broken + if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); + code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); return code; } @@ -259,16 +268,22 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { //TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); + freeHandle = true; } else { // result is not ready, return immediately if (!buildRes) { qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); return TSDB_CODE_QRY_NOT_READY; } - code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle); + } + + // If qhandle is not added into vread queue, the query should be completed already or paused with error. + // Here free qhandle immediately + if (freeHandle) { + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); return code; } diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 07f2cd3f77..882f561ae1 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -257,6 +257,21 @@ if $data01 != $val then return -1 endi +sql select count(join_tb1.*) + count(join_tb0.*) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts >= 100000 and join_tb0.c7 = false;; +if $rows != 1 then + return -1 +endi + +if $data00 != 20.000000000 then + print expect 20.000000000 actual $data00 + return -1 +endi + +sql select count(join_tb1.*)/10 from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts >= 100000 and join_tb0.c7 = false;; +if $data00 != 1.000000000 then + return -1 +endi + print 3 #agg + where condition sql select count(join_tb1.c3), count(join_tb0.ts) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts <= 100002 and join_tb0.c7 = true; @@ -381,6 +396,15 @@ if $data00 != $val then return -1 endi +sql select sum(join_mt0.c1)+sum(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.ts = join_mt1.ts and join_mt0.t1=join_mt1.t1 and join_mt0.c2=99 and join_mt1.ts=100999; +if $rows != 1 then + return -1 +endi + +if $data00 != 396.000000000 then + return -1 +endi + # first/last sql select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts and join_mt0.t1=1 interval(10a) order by join_mt0.ts asc; diff --git a/tests/script/general/parser/sliding.sim b/tests/script/general/parser/sliding.sim index 177c95651f..f85211beb8 100644 --- a/tests/script/general/parser/sliding.sim +++ b/tests/script/general/parser/sliding.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c debugFlag -v 135 system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start @@ -28,7 +28,7 @@ $mt = $mtPrefix . $i sql drop database if exits $db -x step1 step1: -sql create database if not exists $db tables 4 keep 36500 +sql create database if not exists $db maxtables 4 keep 36500 sql use $db sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12)) diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 1e73893793..dd4faee959 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,27 +1,27 @@ -sleep 2000 -run general/parser/alter.sim -sleep 2000 -run general/parser/alter1.sim -sleep 2000 -run general/parser/alter_stable.sim -sleep 2000 -run general/parser/auto_create_tb.sim -sleep 2000 -run general/parser/auto_create_tb_drop_tb.sim -sleep 2000 -run general/parser/col_arithmetic_operation.sim -sleep 2000 -run general/parser/columnValue.sim -sleep 2000 -run general/parser/commit.sim -sleep 2000 -run general/parser/create_db.sim -sleep 2000 -run general/parser/create_mt.sim -sleep 2000 -run general/parser/create_tb.sim -sleep 2000 -run general/parser/dbtbnameValidate.sim +#sleep 2000 +#run general/parser/alter.sim +#sleep 2000 +#run general/parser/alter1.sim +#sleep 2000 +#run general/parser/alter_stable.sim +#sleep 2000 +#run general/parser/auto_create_tb.sim +#sleep 2000 +#run general/parser/auto_create_tb_drop_tb.sim +#sleep 2000 +#run general/parser/col_arithmetic_operation.sim +#sleep 2000 +#run general/parser/columnValue.sim +#sleep 2000 +#run general/parser/commit.sim +#sleep 2000 +#run general/parser/create_db.sim +#sleep 2000 +#run general/parser/create_mt.sim +#sleep 2000 +#run general/parser/create_tb.sim +#sleep 2000 +#run general/parser/dbtbnameValidate.sim sleep 2000 run general/parser/fill.sim sleep 2000 diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index a0c46dbc65..fdda79451d 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -66,9 +66,56 @@ if $row != 100 then return -1 endi -sql select last(c2) from tb_tb9 +sql select bottom(c1, 100) from tb_stb0 +if $row != 100 then + return -1 +endi + +sql select last(*) from tb_tb9 if $row != 1 then return -1 endi +sql select last(c2) from tb_tb9 +if $row != 0 then + return -1 +endi + +sql select first(c2), last(c2) from tb_tb9 +if $row != 0 then + return -1 +endi + +sql create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20)); +sql create table test1 using test tags('beijing'); +sql insert into test1 values(1537146000000, 1, 1, 1, 1, 0.100000, 0.100000, 0, 'taosdata1', '涛思数据1'); +sql insert into test1 values(1537146000001, 2, 2, 2, 2, 1.100000, 1.100000, 1, 'taosdata2', '涛思数据2'); +sql insert into test1 values(1537146000002, 3, 3, 3, 3, 2.100000, 2.100000, 0, 'taosdata3', '涛思数据3'); +sql insert into test1 values(1537146000003, 4, 4, 4, 4, 3.100000, 3.100000, 1, 'taosdata4', '涛思数据4'); +sql insert into test1 values(1537146000004, 5, 5, 5, 5, 4.100000, 4.100000, 0, 'taosdata5', '涛思数据5'); +sql insert into test1 values(1537146000005, 6, 6, 6, 6, 5.100000, 5.100000, 1, 'taosdata6', '涛思数据6'); +sql insert into test1 values(1537146000006, 7, 7, 7, 7, 6.100000, 6.100000, 0, 'taosdata7', '涛思数据7'); +sql insert into test1 values(1537146000007, 8, 8, 8, 8, 7.100000, 7.100000, 1, 'taosdata8', '涛思数据8'); +sql insert into test1 values(1537146000008, 9, 9, 9, 9, 8.100000, 8.100000, 0, 'taosdata9', '涛思数据9'); +sql insert into test1 values(1537146000009, 10, 10, 10, 10, 9.100000, 9.100000, 1, 'taosdata10', '涛思数据10'); +sql select bottom(col5, 10) from test +if $rows != 10 then + return -1 +endi + +if $data01 != 0.10000 then + print expect 0.10000 actual: $data01 + return -1 +endi + +if $data11 != 1.10000 then + print expect 1.10000 actual: $data11 + return -1 +endi + +if $data21 != 2.10000 then + print expect 2.10000 actual: $data21 + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim index 358bcb8a40..fbd1c211b9 100644 --- a/tests/script/general/parser/union.sim +++ b/tests/script/general/parser/union.sim @@ -1,10 +1,10 @@ -#system sh/stop_dnodes.sh -# -#system sh/deploy.sh -n dnode1 -i 1 -#system sh/cfg.sh -n dnode1 -c walLevel -v 0 -#system sh/cfg.sh -n dnode1 -c debugFlag -v 135 -#system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -#system sh/exec.sh -n dnode1 -s start +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c debugFlag -v 135 +system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 +system sh/exec.sh -n dnode1 -s start sleep 1000 sql connect @@ -24,77 +24,77 @@ $mt = $mtPrefix . $i $j = 1 $mt1 = $mtPrefix . $j -# -#sql drop database if exits $db -x step1 -#step1: -#sql create database if not exists $db maxtables 4 + +sql drop database if exits $db -x step1 +step1: +sql create database if not exists $db maxtables 4 sql use $db -#sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) -# -#$i = 0 -#$t = 1578203484000 -# -#while $i < $tbNum -# $tb = $tbPrefix . $i -# sql create table $tb using $mt tags( $i ) -# -# $x = 0 -# while $x < $rowNum -# $ms = $x * 1000 -# $ms = $ms * 60 -# -# $c = $x / 100 -# $c = $c * 100 -# $c = $x - $c -# $binary = 'binary . $c -# $binary = $binary . ' -# $nchar = 'nchar . $c -# $nchar = $nchar . ' -# -# $t1 = $t + $ms -# sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) -# $x = $x + 1 -# endw -# -# $i = $i + 1 -#endw -# -#sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) -# -#$j = 0 -#$t = 1578203484000 -#$rowNum = 1000 -#$tbNum = 5 -#$i = 0 -# -#while $i < $tbNum -# $tb1 = $tbPrefix1 . $j -# sql create table $tb1 using $mt1 tags( $i ) -# -# $x = 0 -# while $x < $rowNum -# $ms = $x * 1000 -# $ms = $ms * 60 -# -# $c = $x / 100 -# $c = $c * 100 -# $c = $x - $c -# $binary = 'binary . $c -# $binary = $binary . ' -# $nchar = 'nchar . $c -# $nchar = $nchar . ' -# -# $t1 = $t + $ms -# sql insert into $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $j = $j + 1 -#endw -# -#print sleep 1sec. -#sleep 1000 +sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) + +$i = 0 +$t = 1578203484000 + +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $ms = $x * 1000 + $ms = $ms * 60 + + $c = $x / 100 + $c = $c * 100 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + + $t1 = $t + $ms + sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) + +$j = 0 +$t = 1578203484000 +$rowNum = 1000 +$tbNum = 5 +$i = 0 + +while $i < $tbNum + $tb1 = $tbPrefix1 . $j + sql create table $tb1 using $mt1 tags( $i ) + + $x = 0 + while $x < $rowNum + $ms = $x * 1000 + $ms = $ms * 60 + + $c = $x / 100 + $c = $c * 100 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + + $t1 = $t + $ms + sql insert into $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + $x = $x + 1 + endw + + $i = $i + 1 + $j = $j + 1 +endw + +print sleep 1sec. +sleep 1000 $i = 1 $tb = $tbPrefix . $i @@ -412,4 +412,8 @@ if $data10 != @union_db0@ then return -1 endi +sql_error show tables union all show tables +sql_error show stables union all show stables +sql_error show databases union all show databases + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file